环境准备
| 主机名称 | IP地址 |
|---|---|
| flume | 192.168.200.81 |
其他信息：
[root@flume ~]# uname -r
2.6.32-358.el6.x86_64
[root@flume ~]# uname -m
x86_64
[root@flume ~]# getenforce
Disabled
echo "192.168.200.81 flume">>/etc/hosts
一、安装JDK
请参考：Linux下快速安装JDK
二、安装flume
2.1 安装配置
下载
wget http://mirrors.tuna.tsinghua.edu.cn/apache/flume/1.8.0/apache-flume-1.8.0-bin.tar.gz
解压
tar zxf /usr/local/src/apache-flume-1.8.0-bin.tar.gz -C /usr/local/
创建软链接
ln -s /usr/local/apache-flume-1.8.0-bin /usr/local/flume
创建测试配置文件
- mkdir /usr/local/flume/mytest
- vim /usr/local/flume/mytest/1-example.conf
# example.conf: A single-node Flume configuration
# Name the components on this agent
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# Describe/configure the source
a1.sources.r1.type = netcat
a1.sources.r1.bind = localhost
a1.sources.r1.port = 44444
# Describe the sink
a1.sinks.k1.type = logger
# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# Bind the source and sink to the channel
a1.sources.r1.channels = c1a1.sinks.k1.channel = c1
2.2 启动
cd /usr/local/flume/
bin/flume-ng agent --conf conf --conf-file mytest/1-example.conf --name a1 -Dflume.root.logger=INFO,console
2.3 测试
另开启一个ssh连接端，执行telnet localhost 44444，成功后，输入hello flume
[root@flume ~]# telnet localhost 44444
Trying ::1...
telnet: connect to address ::1: Connection refused
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
hello flume
OK
在启动端可以查看到对应的hello flume，表示成功
2017-12-03 14:05:17,752 (SinkRunner-PollingRunner-DefaultSinkProcessor) [INFO - org.apache.flume.sink.LoggerSink.process(LoggerSink.java:95)] Event: { headers:{} body: 68 65 6C 6C 6F 20 66 6C 75 6D 65 0D hello flume. }
三、flume从日志文件中收集数据
3.1 编写配置文件
vim mytest/2-sourceFromFile.conf
# Name the components on this agent
a2.sources = r1
a2.sinks = k1
a2.channels = c1
# Describe/configure the source
a2.sources.r1.type = exec
a2.sources.r1.command = tail -F /tmp/temp.log
a2.sources.r1.channels = c1
# Describe the sink
a2.sinks.k1.type = logger
# Use a channel which buffers events in memory
a2.channels.c1.type = memory
a2.channels.c1.capacity = 1000
a2.channels.c1.transactionCapacity = 100
# Bind the source and sink to the channel
a2.sources.r1.channels = c1
a2.sinks.k1.channel = c1
创建测试日志文件
echo `date +%F\ %T` >> /tmp/temp.log
3.2 启动
bin/flume-ng agent --conf conf --conf-file mytest/2-sourceFromFile.conf --name a2 -Dflume.root.logger=INFO,console
3.3 测试观察
执行以下命令，并观察控制台
echo `date +%F\ %T` >> /tmp/temp.log
