path: root/suspicious

#!/usr/bin/python 
#Author: Marc Jones <mjones@softwarefreedom.org>
#Date: Feb 26, 2014
#Version 0.1.2

#Add weight score function to remove scores from files that otherwise would not have scores

from optparse import OptionParser
import os
import re
import sys
import datetime
import string

report = {}
wordscore = {}
filescore = {}
filelist = list()
skipped = 0
opened = 0
datasize = 0
progresstext = "" 

	
def sortscore(score, reverse=True):
	sortedscore = sorted(score.items(), key=lambda score: score[1], reverse=reverse)
	returnscore = []
	for s in sortedscore:
		if s[1] > 0:
			returnscore.append(s)
	
	return returnscore

def printscore(report):
	for i in report:
		print i[0] + ':' + str(i[1])

def scorewords(report):
	for file in report.keys():
		for word in report[file].keys():
			if not word in wordscore:
				wordscore[word] = 0
			if not file in filescore:
				filescore[file] = 0
			wordscore[word] += report[file][word]
	return wordscore

def weightreport(report, commonwords):
	notsuspiciousfiles = []
	for file in report:
		suspicious = False
		for word in report[file]:
			if not word in commonwords:
				if report[file][word] > 0:
					suspicious = True
		
		if not suspicious:
			notsuspiciousfiles.append(file)
	for file in notsuspiciousfiles:
		report.pop(file)

	return report

def scorefile(report):
	for file in report.keys():
		for word in report[file].keys():
			if not word in wordscore:
				wordscore[word] = 0
			if not file in filescore:
				filescore[file] = 0
			filescore[file] += report[file][word]
	return filescore

def summary(report):
	filescore = scorefile(report)
	text = ""
	for file in sortscore(filescore):
		text += file[0] + '(' + str(file[1]) + '):'
		for word in report[file[0]].keys():
			if report[file[0]][word] > 0:
				text += word + '(' + str(report[file[0]][word]) + ');' 
		text += '\n'
	return text

def wholeword(word, string):
	re.purge()
	matches = []
	
	if word.isdigit():
		int(word)
		regexNum = r'([^0-9]|\b)(' + word + r')([^0-9]|\b)'
		mN = re.search(regexNum, string)
		if "groups" in dir(mN):
			matches.append(mN.groups())
	
	else:
		regexU = r'([A-Z]|[^a-zA-Z]|\b)(' + word.lower() + r')([A-Z]|[^a-zA-Z]|\b)'
		regexL = r'([a-z]|[^a-zA-Z]|\b)(' + word.upper() + r')([a-z]|[^a-zA-Z]|\b)'
		mU = re.search(regexU, string)
		if "groups" in dir(mU):
			matches.append(mU.groups())
		re.purge()
		mL = re.search(regexL, string)
		if "groups" in dir(mL):
			matches.append(mL.groups())
	return matches

def skipfile(filename,skippedexts):
	if not isinstance(skippedexts, list):
		return False
	for skip in skippedexts:
		if filename.endswith(skip):
			return True
	return False

def scoretext(wordlist, text, maxwholewordlen = -1):
	score = {}
	ltext = text.lower()
	for word in wordlist:
		wordreg = word.replace('-', ' ')
		wordreg = wordreg.replace(' ', '['+string.punctuation+' ]?')
		if int(len(word)) > int(maxwholewordlen):
			matches = [] 
			m = re.search(wordreg.lower(),ltext)
			if "groups" in dir(m):
				matches.append(m.groups())
			score[word] = len(matches)			
		else:
			score[word] = len(wholeword(wordreg,text))
	return score

usage = "%prog [options] DIRECTORY ... DIRECTORYN"
epilog = "example: ./suspicious ../git.lf/janitor -s .tar -s .gz -s .bmp -s .zip -s .ppt -s .docx -s .pdf -s .xls -s .xlsx -s .gif -s .png -s .jpg -s .css -r fw -w cryptology.txt -c -p -l 3"
parser = OptionParser(usage = usage, epilog = epilog)
parser.add_option("-f", "--file", 
		dest="suspiciousfilename", 
		help="specify file to scan", action="append")
parser.add_option("-w", "--wordlist", 
		dest="wordlistfilename", 
		help="file containing all of the words to look for")
parser.add_option("-s", "--skip", 
		dest="skipfileextensions", 
		help="file extensions to skip", 
		action="append")
parser.add_option("-v", "--verbose", 
		dest="verbose", 
		help="print verberose information", 
		default=False, 
		action="store_true")
parser.add_option("-r", "--report", 
		dest="printreport", 
		default="wf", 
		help="print score")
parser.add_option("--show-wordlist", 
		dest="show_wordlist", 
		default=False, 
		help="print list of words to detect", 
		action="store_true")
parser.add_option("-c", "--display-counts", 
		dest="display_counts", 
		default=False, 
		help="Show the number of files processed", 
		action="store_true")
parser.add_option("-p", "--display_progress", 
		dest="display_progress", 
		default=False, 
		help="show percentage complete", 
		action="store_true")
parser.add_option("-l", "--max-wholeword-length", 
		dest="maxwholewordlength", 
		type="int", 
		default=-1, 
		help="maximun length of a word allowed to only find matches on whole word")
parser.add_option("-o", "--summary-file", 
		dest="summaryfile", 
		help="name of the file to store the summary in")
parser.add_option("-x", "--display-summary", 
		dest="displaysummary", 
		default=False, 
		help="Display a summary from the summary file", 
		action="store_true")
parser.add_option("-X", "--dont-display-summary", 
		dest="dontdisplaysummary", 
		default=False, 
		help="Dont Display a summary after running a scan", 
		action="store_true")
parser.add_option("-k", "--commonwords", 
		dest="commonwordfilename", 
		help="file containing commmon words that allow do not indicate a suspicious file")
parser.add_option("-t", "--test", 
		dest="test", 
		default=False, 
		help="Run internal tests on pattern matching", 
		action="store_true")

(options, args) = parser.parse_args()

if options.commonwordfilename:
	commonwords = list(set(open(options.commonwordfilename).read().lower().strip().split('\n')))
if options.wordlistfilename:
	wordlist = list(set(open(options.wordlistfilename).read().lower().strip().split('\n')))
#	uncommonwordlist = wordlist
#	if options.commonwordfilename:
#		for word in commonwords:
#			if word in uncommonwordlist:
#				uncommonwordlist.remove(word)
	
#	uncommonwordlist = optimizewordlist(uncommonwordlist, options.maxwholewordlength)

#	if options.commonwordfilename:
#		wordlist = list(set(uncommonwordlist + commonwords))
#	else:
#		wordlist = uncommonwordlist

if options.show_wordlist: print wordlist; exit()

if options.displaysummary and options.summaryfile:
	report = dict()
	try:
		summaryfile = open(options.summaryfile)
	except:
		print "no summary file: " + options.summaryfile
		exit()
	#sample input
	#../bzr.lf/lsb/devel/build_env/headers/x86-64/4.1/glib-2.0/gio/gmenuexporter.h.defs(1): export(1);
	for line in summaryfile:
		#find the file name which is before the matching parathsis before the last colon on the line
		filename = line[:line[:line.rfind(':')].rfind('(')]
		#find the total number of words found by locating the end of the filename and taking the number in parathesis right before the :
		totalfilecount = line[line[:line.rfind(':')].rfind('(')+1:line[:line.rfind(':')].rfind(')')]
		#find the list of words following the :, and split them by the ;, and then drop the last item on the list which is always a \n
		foundwords = line[line.rfind(':')+1:].split(';')[:-1]
		report[filename] = dict()		
		for w in foundwords:
			w = w.strip()
			word = w[:w.find('(')]
			wcount = w[w.find('(')+1:w.find(')')]		
			report[filename][word] = int(wcount)
	if options.commonwordfilename:
		report = weightreport(report, commonwords)

	if options.printreport:
		if options.printreport == "f":
			printscore(sortscore(scorefile(report)))
		elif options.printreport == "w":
			printscore(sortscore(scorewords(report)))
		elif options.printreport == "wf" or options.printreport == "fw":
			print summary(report)			
	else:
		print summary(report)
	
	exit()
if len(args) > 0:
	for a in args:
		for (path, dirs, files) in os.walk(a):
			if 'CVS' in dirs:
				dirs.remove('CVS')
			if '.git' in dirs:
				dirs.remove('.git')
			if '.bzr' in dirs:
				dirs.remove('.bzr')
			if '.hg' in dirs:
				dirs.remove('.hg')
			if '.svn' in dirs:
				dirs.remove('.svn')
	
			for file in files:
				filelist.append(path + '/' + file)
	
if options.suspiciousfilename:
	filelist += options.suspiciousfilename

start = datetime.datetime.now()
for file in filelist:
	if skipfile(file, options.skipfileextensions):
		skipped += 1
		continue
	try:
		f = open(file)
	except:
		print "failed to open: " + file
		continue
	opened +=1
	now = datetime.datetime.now()
	estimate = (((now - start) / (opened + skipped)) * len(filelist)) 
	if options.display_progress: 
		if len(file)> 60:
			prog_file = file.split('/')[0] + "/.../" + file.split('/')[-1]
		else:
			prog_file = file
		print '\r' + " " * len(progresstext) + '\r',
		progresstext = str(((opened + skipped)*1.0/len(filelist))*100)[:5] + '% '+ " time left:" + str(estimate).split('.')[0] + ' ' + prog_file + '\r'
		print progresstext,
	sys.stdout.flush()
	filecontents = f.read()
	datasize += len(filecontents)		
	filenamescore = scoretext(wordlist, file, options.maxwholewordlength)
	filecontentsscore = scoretext(wordlist, filecontents, options.maxwholewordlength)
	report[file] = {}
	for k in filecontentsscore.keys():
		report[file][k] = filenamescore[k] + filecontentsscore[k]

if options.display_progress: 
	print '\r' + " " * len(progresstext) + '\r',

if options.summaryfile and len(filelist) > 0 and not options.displaysummary:
	summaryfilename = options.summaryfile	
	counter = 0
	while os.path.isfile(summaryfilename):
		counter +=1
		summaryfilename = options.summaryfile + '.' + str(counter)
	try:
		if counter > 1: print "saving as " + summaryfilename + "...."	
		summaryfile = open(summaryfilename, 'w+')
		summaryfile.write(summary(report))
		summaryfile.close()		
	except:
		print report
		print "error saving summary as " + summaryfilename

if options.printreport and not options.dontdisplaysummary:
	if options.printreport == "f":
		printscore(sortscore(scorefile(report)))
	elif options.printreport == "wf" or options.printreport == "fw":
		print summary(report)
	else:
		printscore(sortscore(scorewords(report)))

if options.display_counts:
	print "total files:" + str(len(filelist)) ,
	print "suspicious files:" + str(len(sortscore(scorefile(report)))) ,
	print "skipped files:" + str(skipped) ,
	print "searched:" + str(datasize) + 'B', 
	print "time:" + str(datetime.datetime.now() - start).split('.')[0]