vault backup: 2025-12-04 09:12:56

技术探究/logstash.md

@@ -0,0 +1,97 @@
+### 本地文件
+Sample Logstash configuration for creating a simple
+```
+Beats -> Logstash -> Elasticsearch pipeline.
+input {
+  tcp {
+    port => 5000
+  }
+  file {
+    path => "D:/projects/beijing/migu_music/migu-wiki/logs/server.log"
+    start_position => beginning
+    # sincedb_path => "D:\projects\beijing\migu_music\migu-wiki\logs"
+    codec => multiline {
+      pattern => "^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}"
+      negate => true
+      auto_flush_interval => 3
+      what => previous
+    }
+  }
+}
+
+filter {
+  mutate { replace => { type => "random_logs" } }
+}
+
+output {
+  elasticsearch {
+    hosts => "http://cqmg.860001.xyz:26017/"
+    user => "elastic"
+    password => "changeme"
+    index=>"random_logs"
+  }
+  stdout { codec => rubydebug }
+
+}
+```
+F:\Users\ASUS\Downloads\logstash-8.1.3-windows-x86_64\logstash-8.1.3
+
+
+### 同步mysql
+```
+input{
+    stdin {
+    }
+    jdbc {
+        jdbc_driver_library => "/data/app/logstash-7.13.4/mysql-connector-java-5.1.47.jar"
+        jdbc_driver_class => "com.mysql.jdbc.Driver"
+        jdbc_connection_string => "jdbc:mysql://10.19.83.209:12306/dmp_dev?characterEncoding=utf8&useSSL=false"
+        jdbc_user => "root"
+        jdbc_password => "My%209admin"
+        jdbc_paging_enabled => "true"
+        jdbc_page_size => "50000" 
+        # 设置监听间隔  各字段含义（由左至右）分、时、天、月、年，全部为*默认含>义为每分钟都更新
+        schedule => "* * * * * "
+        statement => "select * from res_terminal"
+        # tracking_column => "updatetime"
+        # 保存上一次运行>的信息(tracking_column)
+        # last_run_metadata_path => "./logstash_jdbc_last_run"
+        type => "res_terminal" 
+    }
+}
+
+filter{
+    json{
+        source => "message"
+        remove_field => ["message"]
+    }
+}
+#output插件配置
+output{
+     if[type] == "res_terminal" {
+		elasticsearch {
+			hosts => ["127.0.0.1:9200"]
+			index => "res_terminal"
+            document_type => "res_terminal"
+			document_id => "%{resource_id}"
+		}
+	}
+    #控制台打印json
+    stdout {
+        codec => json_lines
+    }
+}
+
+```
+
+bin/logstash -f logstash.conf
+
+bin/logstash -f logstash.conf  -t 检测配置文件
+
+
+### document_id 大小写敏感
+
+
+
+
+