110 lines
2.7 KiB
Python
110 lines
2.7 KiB
Python
|
|
import os
|
|
import sys
|
|
import re
|
|
import json
|
|
import hashlib
|
|
import gzip
|
|
|
|
line_pattern = re.compile(
|
|
r'(\d{4}\/\d{2}\/\d{2}) (\d{2}:\d{2}:\d{2}) \[([A-Za-z]+)\] (\d+#\d+): ([*]\d+)( \[client [^]]+\])? ModSecurity: ([^[]+) (.*)')
|
|
headers_pattern = re.compile(r', ([^:]+): ([^,]+)')
|
|
request_pattern = re.compile(r'"([^ ]+) ([^ ]+) ([^ ]+)"')
|
|
|
|
|
|
def pivot_tags(tags):
|
|
results = {}
|
|
|
|
for x in tags:
|
|
key, value = x
|
|
|
|
if key in results:
|
|
results[key].append(value)
|
|
else:
|
|
results[key] = [value]
|
|
|
|
return results
|
|
|
|
|
|
def parse_line(log_name, line):
|
|
m = line_pattern.match(line)
|
|
if not m:
|
|
return line
|
|
|
|
fields = {'date': m[1],
|
|
'time': m[2],
|
|
'level': m[3],
|
|
'process': m[4],
|
|
'request': m[5],
|
|
'msg': m[7],
|
|
'tags': pivot_tags(re.findall(r'\[([^ ]+) "?([^"]+)"?\] ', m[8])),
|
|
'headers': {k:v for k, v in headers_pattern.findall(m[8])},
|
|
'hash': hashlib.md5(line.encode('utf-8')).hexdigest(),
|
|
'log_name': log_name}
|
|
|
|
if 'request' in fields['headers']:
|
|
rm = request_pattern.findall(fields['headers']['request'])
|
|
fields['headers']['request'] = {
|
|
'method': rm[0][0],
|
|
'path': rm[0][1],
|
|
'version': rm[0][2]}
|
|
|
|
return fields
|
|
|
|
|
|
def read_log(path):
|
|
if path.endswith('.gz'):
|
|
f = gzip.open(path, 'rb')
|
|
else:
|
|
f = open(path, 'rb')
|
|
|
|
lines = (x.decode('latin-1') for x in f.readlines())
|
|
f.close()
|
|
|
|
log_name = os.path.basename(path)
|
|
return [parse_line(log_name, x.strip()) for x in lines if 'ModSecurity' in x]
|
|
|
|
|
|
def partition_results(events):
|
|
parsed = []
|
|
failed = []
|
|
|
|
for x in events:
|
|
if isinstance(x, str):
|
|
failed.append(x)
|
|
else:
|
|
parsed.append(x)
|
|
|
|
return parsed, failed
|
|
|
|
|
|
def write_results(events, out_file, err_file):
|
|
parsed, failed = partition_results(events)
|
|
|
|
with gzip.open(out_file, 'w') as f:
|
|
f.write(json.dumps(parsed).encode('utf-8'))
|
|
|
|
if len(failed):
|
|
with open(err_file, 'w') as f:
|
|
for x in failed:
|
|
f.write(x + '\n')
|
|
|
|
print(f'Wrote {out_file}, Parsed: {len(parsed)}, Failed: {len(failed)}')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
if len(sys.argv) != 3:
|
|
print('USAGE: python parse.py <log file> <out dir>')
|
|
sys.exit(1)
|
|
|
|
try:
|
|
os.mkdir(sys.argv[2])
|
|
except FileExistsError:
|
|
pass
|
|
|
|
out_file = os.path.join(sys.argv[2], sys.argv[1] + '.json.gz')
|
|
err_file = os.path.join(sys.argv[2], sys.argv[1] + '.failed')
|
|
|
|
events = read_log(sys.argv[1])
|
|
write_results(events, out_file, err_file)
|