Anda di halaman 1dari 2

flume.

conf

#name of source seqGenSrc


#name of our channel memoryChannel
#name of our destination/sink loggerSink

agent.sources = seqGenSrc
agent.channels = memoryChannel
agent.sinks = loggerSink

#Telling Flume that on our every update continuously (exec) run a command
#tail -F /home/training/log.txt

agent.sources.seqGenSrc.type = exec
agent.sources.seqGenSrc.command = tail -F /home/cloudera/log.txt

#channel is memoryChannel
agent.sources.seqGenSrc.channels = memoryChannel
agent.sinks.loggerSink.type = logger
agent.sinks.loggerSink.channel = memoryChannel

#memory channel is buffering all data in memory


agent.channels.memoryChannel.type = memory

#capacity of memory channel is 100 bytes


agent.channels.memoryChannel.capacity = 100

#as soon as 100 bytes are full it should send the data to destination hdfs
#at hdfs destination hdfs://localhost/user/cloudera/logs
#and its a data stream

agent.sinks.loggerSink.type = hdfs
agent.sinks.loggerSink.hdfs.path = hdfs://localhost/user/cloudera/logs
agent.sinks.loggerSink.hdfs.fileType = DataStream

----------------------------------------------------------------------------------------
******************************************************************
STEP1:
copy the flume.conf file to flume library
******************************************************************
sudo cp /home/cloudera/Desktop/flume.conf /usr/lib/flume-ng/conf/

******************************************************************
STEP2:
start flume service
******************************************************************
flume-ng agent --conf-file /usr/lib/flume-ng/conf/flume.conf --name agent -
Dflume.root.logger=INFO,console

# --conf-file mentions the file that flume has to run


# -D flag is to override or change the default behaviour
# INFO,console means log all the messages starting from INFO level and I would like to see all of
them on console

******************************************************************
STEP3:
create file log.txt
******************************************************************
pwd /home/cloudera/
vi log.txt

esc + :wq

ls -ltr

******************************************************************
STEP4:
modify log.txt and check it on hdfs log folder
******************************************************************

Anda mungkin juga menyukai