I have created a package that outputs JSON formatted lines to a log file. It can make use of the standard logging parameters and/or take custom input. The use of JSON in the log file allows for easy filtering and processing.
Python, 167 lines
Download
Copy to clipboard1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 | """
logjson
This package was created to leverage the very flexible standard Python logging package to produce JSON logs. JSON allows for combining multiple types of records in one stream which can easily be filtered or viewed together as needed.
The uses of good logging data include debugging, user support, auditing, reporting, and more.
For some really advanced uses of logs see: http://shop.oreilly.com/product/0636920034339.do?sortby=publicationDate
Python Versions Tested: 2.7,3.4
Usage:
- Output data to the console
mylog = streamlogger('test',['levelname','asctime'])
mylog.info(['John',3.0,'science fiction'])
{"levelname": "INFO", "asctime": "2015-03-21 14:56:04,431", "msg": ["John", 3.0, "science fiction"]}
- A more complicated example, with log filtering.
import os
from collections import OrderedDict
record1 = OrderedDict((('user','Fred'),('query','Bujold'),('time',0.5),('results',5)))
record2 = OrderedDict((('user','Bill'),('query','Heinlein'),('time',0.7),('results',3)))
record3 = OrderedDict((('user','Mary'),('query','Asimov'),('time',0.2),('results',1)))
record4 = OrderedDict((('user','Johan'),('query','Niven'),('time',0.9),('results',2)))
fname = "testfilelogger.txt"
currentdirectory = os.getcwd()
fullpath = os.path.join(currentdirectory,fname)
flog = filelogger('testfilelog',[],fullpath)
flog.info(record1)
flog.info(record2)
flog.info(record3)
flog.info(record4)
logrecs = readJSONlog(open(fullpath,'r'),(lambda x: x['results'] > 2))
flog.removeHandler(flog.handlers[0])
os.remove(fullpath)
logrecs
[{u'query': u'Bujold', u'results': 5, u'time': 0.5, u'user': u'Fred'},
{u'query': u'Heinlein', u'results': 3, u'time': 0.7, u'user': u'Bill'}]
"""
from collections import OrderedDict
from logging import Formatter,FileHandler,StreamHandler,getLogger,INFO
from json import loads,dumps
def logger(name,handler,recordfields = [],level = INFO):
log = getLogger(name)
textformatter = JSONFormatter(recordfields)
handler.setFormatter(textformatter)
log.addHandler(handler)
log.setLevel(level)
return log
def filelogger(logname,recordfields =[],filename='json.log',level = INFO):
"""A convenience function to return a JSON file logger for simple situations.
Args:
logname : The name of the logger - to allow for multiple logs, and levels of logs in an application
recordfields : The metadata fields to add to the JSON record created by the logger
filename : The name of the file to be used in the logger
Returns:
A JSON file logger.
"""
handler = FileHandler(filename,'w')
return logger(logname,handler,recordfields,level)
def streamlogger(logname,recordfields=[],outputstream=None, level = INFO):
"""A convenience function to return a JSON stream logger for simple situations.
Args:
logname : The name of the logger - to allow for multiple logs, and levels of logs in an application
recordfields : The metadata fields to add to the JSON record created by the logger
outputstream : The outputstream to be used by the logger. sys.stderr is used when outputstream is None.
Returns:
A JSON stream logger.
"""
handler = StreamHandler(outputstream)
return logger(logname,handler,recordfields,level)
def readJSONlog(logfile,filterfunction = (lambda x: True),customjson = None):
"""Iterate through a log file of JSON records and return a list of JSON records that meet the filterfunction.
Args:
logfile : A file like object consisting of JSON records.
filterfunction : A function that returns True if the JSON record should be included in the output and False otherwise.
customjson : A decoder function to enable the loading of custom json objects
Returns:
A list of Python objects built from JSON records that passed the filterfunction.
"""
JSONrecords = []
for x in logfile:
#if the record in the logfile returns true from the filter function convert it to JSON and add it the records to return
rec = loads(x[:-1],object_hook = customjson)
if filterfunction(rec): JSONrecords.append(rec)
return JSONrecords
class JSONFormatter(Formatter):
"""The JSONFormatter class outputs Python log records in JSON format.
JSONFormatter assumes that log record metadata fields are specified at the fomatter level as opposed to the
record level. The specification of matadata fields at the formatter level allows for multiple handles to display
differing levels of detail. For example, console log output might specify less detail to allow for quick problem
triage while file log output generated from the same data may contain more detail for in-depth investigations.
Attributes:
recordfields : A list of strings containing the names of metadata fields (see Python log record documentation
for details) to add to the JSON output. Metadata fields will be added to the JSON record in
the order specified in the recordfields list.
customjson : A JSONEncoder subclass to enable writing of custom JSON objects.
"""
def __init__(self,recordfields = [], datefmt=None, customjson=None):
"""__init__ overrides the default constructor to accept a formatter specific list of metadata fields
Args:
recordfields : A list of strings referring to metadata fields on the record object. It can be empty.
The list of fields will be added to the JSON record created by the formatter.
"""
Formatter.__init__(self,None,datefmt)
self.recordfields = recordfields
self.customjson = customjson
def usesTime(self):
""" Overridden from the ancestor to look for the asctime attribute in the recordfields attribute.
The override is needed because of the change in design assumptions from the documentation for the logging module. The implementation in this object could be brittle if a new release changes the name or adds another time attribute.
Returns:
boolean : True if asctime is in self.recordfields, False otherwise.
"""
return 'asctime' in self.recordfields
def _formattime(self,record):
if self.usesTime():
record.asctime = self.formatTime(record, self.datefmt)
def _getjsondata(self,record):
""" combines any supplied recordfields with the log record msg field into an object to convert to JSON
Args:
record : log record to output to JSON log
Returns:
An object to convert to JSON - either an ordered dict if recordfields are supplied or the record.msg attribute
"""
if (len(self.recordfields)>0):
fields = []
for x in self.recordfields:
fields.append((x,getattr(record,x)))
fields.append(('msg',record.msg))
# An OrderedDict is used to ensure that the converted data appears in the same order for every record
return OrderedDict(fields)
else:
return record.msg
def format(self,record):
"""overridden from the ancestor class to take a log record and output a JSON formatted string.
Args:
record : log record to output to JSON log
Returns:
A JSON formatted string
"""
self._formattime(record)
jsondata = self._getjsondata(record)
formattedjson = dumps(jsondata, cls=self.customjson)
return formattedjson
|
The package provided here gives a flexible and powerful way to use JSON with the Python logging framework. I believe this solution is more flexible than the solution in https://docs.python.org/3/howto/logging-cookbook.html under Implementing structured logging.
