Kinesis_firehose_example

sudo yum install –y aws-kinesis-agent

cd /etc/aws-kinesis/

sudo vi agent.json

sudo service aws-kinesis-agent start
sudo chkconfig aws-kinesis-agent on

python3 LogGenerator.py 1000

cd /var/log/aws-kinesis-agent/

tail -f aws-kinesis-agent.log
[ec2-user@ip-172-31-24-247 aws-kinesis]$ cat agent.json
{
“cloudwatch.emitMetrics”: true,
“kinesis.endpoint”: “”,
“firehose.endpoint”: “”,

“flows”: [
{
“filePattern”: “/home/ec2-user/*.log*”,
“deliveryStream”: “kinesis_log_s3”
}
]
}

#########

import names
import random
import time
import sys
import pandas as pd
data=[]
company = [‘TCS’, ‘Wipro’, ‘Infosys’, ‘CTS’, ‘ABC’,’BCD’,’XYZ’]
location = [‘Banglore’, ‘Chennai’, ‘Patna’, ‘Pune’, ‘Mumbai’,’Tokyo’,’Hyderabad’,’Ranchi’,’Kolkata’]
numLines = 10000
if (len(sys.argv) > 1):
numLines = int(sys.argv[1])
for i in range(numLines):
rand_name = names.get_first_name(gender=’male’)
rand_last_name = names.get_last_name()
rand_salary=random.randint(20000,90000)
rand_age=random.randint(28,60)
company_name=random.choice(company)
location=random.choice(company)
z=[rand_name,rand_last_name,rand_age,rand_salary,company_name,location]
data.append(z)

destData = time.strftime(“log_%Y%m%d-%H%M%S.log”)
col=[“First_name”,’Last_name’,’age’,’salary’,’company’,’location’]

df=pd.DataFrame(data,columns=col)
df.to_csv(destData,index=False)

Note:- please take care of indentation.