curl -L https://toolbelt.treasuredata.com/sh/install-amazon1-td-agent3.sh | sh
/etc/td-agent/
[root@ip td-agent]# cat td-agent.conf
@include /etc/td-agent/conf.d/*.conf
/etc/td-agent/conf.d
[root@ip conf.d]# cat td-gamex.conf
<source>
@type forward
port 24224
</source>
##########################################################################################
<source>
@type tail
format json
time_key time
time_format %N
pos_file /var/log/td-agent/a.log.pos
path /opt/supervisor/log/log*.%d.%m.%Y.log
tag log.106
</source>
##########################################################################################
<match debug.**>
@type stdout
</match>
##########################################################################################
<match log.**>
@type forward # forward模式，转发给其他服务器处理
send_timeout 60s
recover_wait 10s
transport
phi_threshold 16
hard_timeout 60s
<server>
name log.shard
host 【IP】
port 24224
weight 1
</server>
<secondary>
@type file
path /var/log/td-agent/log-forward-failed
</secondary>
</match>
##########################################################################################
例如添加一个 client log
[root@ip conf.d]# cat client.conf
<source>
@type tail
format json
time_key time
time_format %N
pos_file /var/log/td-agent/client_150.log.pos
path /opt/supervisor/log/client*.%d.%m.%Y.log
tag client.150
</source>
##########################################################################################
<match debug.**>
@type stdout
</match>
##########################################################################################
<match client_log.**>
@type forward # forward模式，转发给其他服务器处理
send_timeout 60s
recover_wait 10s
transport
phi_threshold 16
hard_timeout 60s
<server>
name client_log
host 【IP】
port 24224
weight 1
</server>
<secondary>
@type file
path /var/log/td-agent/logics-forward-failed
</secondary>
</match>
##########################################################################################
中转机器上添加配置
[root@ip conf.d]# cat td-client.conf
<match debug.**>
@type stdout
</match>
##################################################################################
<match client_log.**>
@type copy
<store>
@type elasticsearch_dynamic
host 【IP】
port 9200
request_timeout 15s #defaults to 5s
reload_connections false
reload_on_failure true # defaults to false
logstash_format true
logstash_prefix client_logiclog
logstash_dateformat %Y.%m.%d
time_key time
<buffer>
@type file
path /var/log/td-agent/buffer/td-client_log-buffer
chunk_limit_size 512MB #Default: 8MB (memory) / 256MB (file)
total_limit_size 32GB #Default: 512MB (memory) / 64GB (file)
chunk_full_threshold 0.9 #output plugin will flush the chunk when actual size reaches chunk_limit_size *
chunk_full_threshold
compress text #The option to specify compression of each chunks, during events are buffered
flush_mode default
flush_interval 15s #Default: 60s
flush_thread_count 1 #Default: 1 The number of threads of output plugins, which is used to write chunks in parallel
delayed_commit_timeout 60 #The timeout seconds until output plugin decides that async write operation fails
overflow_action throw_exception
retry_timeout 10m
</buffer>
</store>
</match>
