1. HDFS로 로그를 write할 Agent
agentM.sources = avro-collection-source
agentM.channels = mem-channel
agentM.sinks = hdfs-sink
# For each one of the sources, the type is defined
agentM.sources.avro-collection-source.channels = mem-channel
agentM.sinks.hdfs-sink.channel = mem-channel
agentM.sinks.hdfs-sink.type = hdfs
agentM.sinks.hdfs-sink.hdfs.path = hdfs://sm06/flumetest/events/
agentM.sinks.hdfs-sink.hdfs.filePrefix = events-
agentM.sinks.hdfs-sink.hdfs.round = true
agentM.sinks.hdfs-sink.hdfs.roundValue = 10
agentM.sinks.hdfs-sink.hdfs.roundUnit = 1
# Each sink's type must be defined
agentM.sources.avro-collection-source.type = avro
agentM.sources.avro-collection-source.bind = 0.0.0.0
agentM.sources.avro-collection-source.port = 41414
#Specify the channel the sink should use
# Each channel's type is defined.
agentM.channels.mem-channel.type = memory
# Other config values specific to each type of channel(sink or source)
# can be defined as well
# In this case, it specifies the capacity of the memory channel
agentM.channels.mem-channel.capacity = 100
sudo -u hdfs flume-ng agent --conf /etc/flume-ng/conf --conf-file /etc/flume-ng/conf/flume.conf --name agentM -Dflume.root.logger=INFO,console
2. HDFS Agent로 로그를 보내줄 Agent
agent.sources = avro-AppSrv-source
agent.channels = file-channel
agent.sinks = avro-forward-sink
agent.sources.avro-AppSrv-source.type = exec
agent.sources.avro-AppSrv-source.command = tail -F /var/log/httpd/access_log
agent.sources.avro-AppSrv-source.channels = file-channel
agent.sinks.avro-forward-sink.channel = file-channel
agent.sinks.avro-forward-sink.type = avro
agent.sinks.avro-forward-sink.hostname = 172.21.81.106
agent.sinks.avro-forward-sink.port = 41414
agent.channels.file-channel.type = memory
agent.channels.file-channel.capacity = 100
sudo -u hdfs flume-ng agent --conf /etc/flume-ng/conf --conf-file /etc/flume-ng/conf/flume.conf --name agent -Dflume.root.logger=INFO,console