98 lines
2.2 KiB
Markdown
98 lines
2.2 KiB
Markdown
|
### 本地文件
|
|||
|
|
Sample Logstash configuration for creating a simple
|
|||
|
|
```
|
|||
|
|
Beats -> Logstash -> Elasticsearch pipeline.
|
|||
|
|
input {
|
|||
|
|
tcp {
|
|||
|
|
port => 5000
|
|||
|
|
}
|
|||
|
|
file {
|
|||
|
|
path => "D:/projects/beijing/migu_music/migu-wiki/logs/server.log"
|
|||
|
|
start_position => beginning
|
|||
|
|
# sincedb_path => "D:\projects\beijing\migu_music\migu-wiki\logs"
|
|||
|
|
codec => multiline {
|
|||
|
|
pattern => "^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}"
|
|||
|
|
negate => true
|
|||
|
|
auto_flush_interval => 3
|
|||
|
|
what => previous
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
filter {
|
|||
|
|
mutate { replace => { type => "random_logs" } }
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
output {
|
|||
|
|
elasticsearch {
|
|||
|
|
hosts => "http://cqmg.860001.xyz:26017/"
|
|||
|
|
user => "elastic"
|
|||
|
|
password => "changeme"
|
|||
|
|
index=>"random_logs"
|
|||
|
|
}
|
|||
|
|
stdout { codec => rubydebug }
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
```
|
|||
|
|
F:\Users\ASUS\Downloads\logstash-8.1.3-windows-x86_64\logstash-8.1.3
|
|||
|
|
|
|||
|
|
|
|||
|
|
### 同步mysql
|
|||
|
|
```
|
|||
|
|
input{
|
|||
|
|
stdin {
|
|||
|
|
}
|
|||
|
|
jdbc {
|
|||
|
|
jdbc_driver_library => "/data/app/logstash-7.13.4/mysql-connector-java-5.1.47.jar"
|
|||
|
|
jdbc_driver_class => "com.mysql.jdbc.Driver"
|
|||
|
|
jdbc_connection_string => "jdbc:mysql://10.19.83.209:12306/dmp_dev?characterEncoding=utf8&useSSL=false"
|
|||
|
|
jdbc_user => "root"
|
|||
|
|
jdbc_password => "My%209admin"
|
|||
|
|
jdbc_paging_enabled => "true"
|
|||
|
|
jdbc_page_size => "50000"
|
|||
|
|
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含>义为每分钟都更新
|
|||
|
|
schedule => "* * * * * "
|
|||
|
|
statement => "select * from res_terminal"
|
|||
|
|
# tracking_column => "updatetime"
|
|||
|
|
# 保存上一次运行>的信息(tracking_column)
|
|||
|
|
# last_run_metadata_path => "./logstash_jdbc_last_run"
|
|||
|
|
type => "res_terminal"
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
filter{
|
|||
|
|
json{
|
|||
|
|
source => "message"
|
|||
|
|
remove_field => ["message"]
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
#output插件配置
|
|||
|
|
output{
|
|||
|
|
if[type] == "res_terminal" {
|
|||
|
|
elasticsearch {
|
|||
|
|
hosts => ["127.0.0.1:9200"]
|
|||
|
|
index => "res_terminal"
|
|||
|
|
document_type => "res_terminal"
|
|||
|
|
document_id => "%{resource_id}"
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
#控制台打印json
|
|||
|
|
stdout {
|
|||
|
|
codec => json_lines
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
```
|
|||
|
|
|
|||
|
|
bin/logstash -f logstash.conf
|
|||
|
|
|
|||
|
|
bin/logstash -f logstash.conf -t 检测配置文件
|
|||
|
|
|
|||
|
|
|
|||
|
|
### document_id 大小写敏感
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|