Logentries is a very good tool to store and analyze logs in central location but when we are storing logs from a Nginx/Apache shared hosting environment it gets complex as we need to tag each log to which host it belong. I am using rsyslog to forward to send my logs to Logentries as it gives me more flexibility as all rsyslog functionality works.
First need to create a main configuration file which we will use to create corresponding rsyslog files. It will have your Logentries secret key to whom log need to be followed.
For access log:
[root@ip-10-0-1-220 ravi]# cat access-vanila
$Modload imfile
$InputFileName access-log-location
$InputFileTag access-tag
$InputFileStateFile filestate-tag
$InputFileSeverity info
$InputFileFacility local7
$InputRunFileMonitor
# Only entered once in case of following multiple files
$InputFilePollInterval 1
$template filestate-tag,"6fb8xxxxxxxxxxxxxxxxxxxxxxe8ed %HOSTNAME% %syslogtag% %msg%\n"
if $programname == 'access-tag' then @@data.logentries.com:10000;filestate-tag
& ~
For error log:
[root@ip-10-0-1-220 ravi]# cat error-vanila
$Modload imfile
$InputFileName error-log-location
$InputFileTag error-tag
$InputFileStateFile filestate-tag
$InputFileSeverity info
$InputFileFacility local7
$InputRunFileMonitor
# Only entered once in case of following multiple files
$InputFilePollInterval 1
$template filestate-tag,"4f71xxxxxxxxxxxxxxxxxxxxxxx138 %HOSTNAME% %syslogtag% %msg%\n"
if $programname == 'error-tag' then @@data.logentries.com:10000;filestate-tag
& ~
Now use the following script to create rsyslog configurations to send logs to Logentries.
[root@ip-10-0-1-220 ravi]# cat logscript.sh
#!/bin/bash
#Script to add hosts to Logentries
#By Ravi Gadgil
#Path where nginx hosts configurations are placed /etc/nginx/sites-enabled/sites/
for j in `ls /etc/nginx/sites-enabled/sites/`; do cat /etc/nginx/sites-enabled/sites/$j | grep -m 1 access; cat /etc/nginx/sites-enabled/sites/$j | grep -m 1 error ; done | uniq | grep 'teamieapp\|theteamie' > /home/ravi/data.txt
#Creating files having entries of access and error log, if want to avoid any log place instead of avoid in grep command.
cat data.txt | grep access.log | grep -v 'avoid' | awk '{ print $2 }' | cut -d';' -f1 > /home/ravi/accessfiles.txt
cat data.txt | grep error.log | grep -v 'avoid' | awk '{ print $2 }' | cut -d';' -f1 > /home/ravi/errorfiles.txt
#To create unique Filestate Tag in rsyslog.
COUNTER=50
#To create access log entries.
for i in `cat /home/ravi/accessfiles.txt`
do
logfileaccess=$( echo -e "$i" )
echo -e "Access Log file is : $logfileaccess"
accesstag=$(echo -e $logfileaccess | cut -d'/' -f5 | sed 's/.log//g' | sed "s/_/-/g" | sed "s/\./-/g")
echo -e "Access Tag is : $accesstag"
COUNTER=$[$COUNTER +1]
filestate=$(echo -e "nginx$COUNTER")
echo -e "File state is : $filestate"
cp /home/ravi/access-vanila /home/ravi/tempconf
sed -i "s#access-log-location#$logfileaccess#g" /home/ravi/tempconf
sed -i "s/access-tag/$accesstag/g" /home/ravi/tempconf
sed -i "s/filestate-tag/$filestate/g" /home/ravi/tempconf
mv /home/ravi/tempconf /home/ravi/conf/$accesstag.conf
done
#To create error log entries.
for i in `cat /home/ravi/errorfiles.txt`
do
logfileerror=$( echo -e "$i" )
echo -e "Error Log file is : $logfileerror"
errortag=$(echo -e $logfileerror | cut -d'/' -f5 | sed 's/.log//g' | sed "s/_/-/g" | sed "s/\./-/g")
echo -e "Error Tag is : $errortag"
COUNTER=$[$COUNTER +1]
filestate=$(echo -e "nginx$COUNTER")
echo -e "File state is : $filestate"
cp /home/ravi/error-vanila /home/ravi/tempconf
sed -i "s#error-log-location#$logfileerror#g" /home/ravi/tempconf
sed -i "s/error-tag/$errortag/g" /home/ravi/tempconf
sed -i "s/filestate-tag/$filestate/g" /home/ravi/tempconf
mv /home/ravi/tempconf /home/ravi/conf/$errortag.conf
done
Note:
1. Your log format should be in site_access.log and site_error.log if not in this format do adjust in script.
2. Script was working in /home/ravi directory so adjust script according to your paths.
3. Once all configurations are created add them to your rsyslog directory and restart service.
4. Rsyslog can only forward 100 files so do keep that in mind and check /var/log/messages for
any errors if any.
