obsidian_note/技术探究/logstash.md

本地文件

Sample Logstash configuration for creating a simple

Beats -> Logstash -> Elasticsearch pipeline.
input {
  tcp {
    port => 5000
  }
  file {
    path => "D:/projects/beijing/migu_music/migu-wiki/logs/server.log"
    start_position => beginning
    # sincedb_path => "D:\projects\beijing\migu_music\migu-wiki\logs"
    codec => multiline {
      pattern => "^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}"
      negate => true
      auto_flush_interval => 3
      what => previous
    }
  }
}

filter {
  mutate { replace => { type => "random_logs" } }
}

output {
  elasticsearch {
    hosts => "http://cqmg.860001.xyz:26017/"
    user => "elastic"
    password => "changeme"
    index=>"random_logs"
  }
  stdout { codec => rubydebug }

}

F:\Users\ASUS\Downloads\logstash-8.1.3-windows-x86_64\logstash-8.1.3

同步mysql

input{
    stdin {
    }
    jdbc {
        jdbc_driver_library => "/data/app/logstash-7.13.4/mysql-connector-java-5.1.47.jar"
        jdbc_driver_class => "com.mysql.jdbc.Driver"
        jdbc_connection_string => "jdbc:mysql://10.19.83.209:12306/dmp_dev?characterEncoding=utf8&useSSL=false"
        jdbc_user => "root"
        jdbc_password => "My%209admin"
        jdbc_paging_enabled => "true"
        jdbc_page_size => "50000" 
        # 设置监听间隔  各字段含义（由左至右）分、时、天、月、年，全部为*默认含>义为每分钟都更新
        schedule => "* * * * * "
        statement => "select * from res_terminal"
        # tracking_column => "updatetime"
        # 保存上一次运行>的信息(tracking_column)
        # last_run_metadata_path => "./logstash_jdbc_last_run"
        type => "res_terminal" 
    }
}

filter{
    json{
        source => "message"
        remove_field => ["message"]
    }
}
#output插件配置
output{
     if[type] == "res_terminal" {
		elasticsearch {
			hosts => ["127.0.0.1:9200"]
			index => "res_terminal"
            document_type => "res_terminal"
			document_id => "%{resource_id}"
		}
	}
    #控制台打印json
    stdout {
        codec => json_lines
    }
}

bin/logstash -f logstash.conf

bin/logstash -f logstash.conf -t 检测配置文件

document_id 大小写敏感