python找寻合适的日志库logging Handler——Handler自定义实现

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2012 Ethan Zhang<http://github.com/Ethan-Zhang> 
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.


import time
import os
from logging.handlers import TimedRotatingFileHandler

class MultiProcessTimedRotatingFileHandler(TimedRotatingFileHandler):

    def doRollover(self):
        """ 
        do a rollover; in this case, a date/time stamp is appended to the filename
        when the rollover happens.  However, you want the file to be named for the
        start of the interval, not the current time.  If there is a backup count,
        then we have to get a list of matching filenames, sort them and remove
        the one with the oldest suffix.
        """
        if self.stream:
            self.stream.close()
        # get the time that this sequence started at and make it a TimeTuple
        t = self.rolloverAt - self.interval
        if self.utc:
            timeTuple = time.gmtime(t)
        else:
            timeTuple = time.localtime(t)
        dfn = self.baseFilename + "." + time.strftime(self.suffix, timeTuple)
        #if os.path.exists(dfn):
        #    os.remove(dfn)
        if not os.path.exists(dfn):
            os.rename(self.baseFilename, dfn)
        if self.backupCount > 0:
            # find the oldest log file and delete it
            #s = glob.glob(self.baseFilename + ".20*")
            #if len(s) > self.backupCount:
            #    s.sort()
            #    os.remove(s[0])
            for s in self.getFilesToDelete():
                os.remove(s)
        #print "%s -> %s" % (self.baseFilename, dfn)
        self.mode = ‘a‘
        self.stream = self._open()
        currentTime = int(time.time())
        newRolloverAt = self.computeRollover(currentTime)
        while newRolloverAt <= currentTime:
            newRolloverAt = newRolloverAt + self.interval
        #If DST changes and midnight or weekly rollover, adjust for this.
        if (self.when == ‘MIDNIGHT‘ or self.when.startswith(‘W‘)) and not self.utc:
            dstNow = time.localtime(currentTime)[-1]
            dstAtRollover = time.localtime(newRolloverAt)[-1]
            if dstNow != dstAtRollover:
                if not dstNow:  # DST kicks in before next rollover, so we need to deduct an hour
                    newRolloverAt = newRolloverAt - 3600
                else:           # DST bows out before next rollover, so we need to add an hour
                    newRolloverAt = newRolloverAt + 3600
        self.rolloverAt = newRolloverAt

import logging

from bson.timestamp import Timestamp
from pymongo import Connection
from pymongo.collection import Collection
from pymongo.errors import OperationFailure, PyMongoError


"""
Example format of generated bson document:
{
    ‘thread‘: -1216977216,
    ‘threadName‘: ‘MainThread‘,
    ‘level‘: ‘ERROR‘,
    ‘timestamp‘: Timestamp(1290895671, 63),
    ‘message‘: ‘test message‘,
    ‘module‘: ‘test_module‘,
    ‘fileName‘: ‘/var/projects/python/log4mongo-python/tests/test_handlers.py‘,
    ‘lineNumber‘: 38,
    ‘method‘: ‘test_emit_exception‘,
    ‘loggerName‘:  ‘testLogger‘,
    ‘exception‘: {
        ‘stackTrace‘: ‘Traceback (most recent call last):
                       File "/var/projects/python/log4mongo-python/tests/test_handlers.py", line 36, in test_emit_exception
                       raise Exception(\‘exc1\‘)
                       Exception: exc1‘,
        ‘message‘: ‘exc1‘,
        ‘code‘: 0
    }
}
"""


class MongoFormatter(logging.Formatter):

    DEFAULT_PROPERTIES = logging.LogRecord(‘‘, ‘‘, ‘‘, ‘‘, ‘‘, ‘‘, ‘‘, ‘‘).__dict__.keys()

    def format(self, record):
        """Formats LogRecord into python dictionary."""
        # Standard document
        document = {
            ‘timestamp‘: Timestamp(int(record.created), int(record.msecs)),
            ‘level‘: record.levelname,
            ‘thread‘: record.thread,
            ‘threadName‘: record.threadName,
            ‘message‘: record.getMessage(),
            ‘loggerName‘: record.name,
            ‘fileName‘: record.pathname,
            ‘module‘: record.module,
            ‘method‘: record.funcName,
            ‘lineNumber‘: record.lineno
        }
        # Standard document decorated with exception info
        if record.exc_info is not None:
            document.update({
                ‘exception‘: {
                    ‘message‘: str(record.exc_info[1]),
                    ‘code‘: 0,
                    ‘stackTrace‘: self.formatException(record.exc_info)
                }
            })
        # Standard document decorated with extra contextual information
        if len(self.DEFAULT_PROPERTIES) != len(record.__dict__):
            contextual_extra = set(record.__dict__).difference(set(self.DEFAULT_PROPERTIES))
            if contextual_extra:
                for key in contextual_extra:
                    document[key] = record.__dict__[key]
        return document


class MongoHandler(logging.Handler):

    def __init__(self, level=logging.NOTSET, host=‘localhost‘, port=27017, database_name=‘logs‘, collection=‘logs‘,
                 username=None, password=None, fail_silently=False, formatter=None, capped=False,
                 capped_max=1000, capped_size=1000000, **options):
        """Setting up mongo handler, initializing mongo database connection via pymongo."""
        logging.Handler.__init__(self, level)
        self.host = host
        self.port = port
        self.database_name = database_name
        self.collection_name = collection
        self.username = username
        self.password = password
        self.fail_silently = fail_silently
        self.connection = None
        self.db = None
        self.collection = None
        self.authenticated = False
        self.formatter = formatter or MongoFormatter()
        self.capped = capped
        self.capped_max = capped_max
        self.capped_size = capped_size
        self.options = options
        self._connect()

    def _connect(self):
        """Connecting to mongo database."""

        try:
            self.connection = Connection(host=self.host, port=self.port, **self.options)
        except PyMongoError:
            if self.fail_silently:
                return
            else:
                raise

        self.db = self.connection[self.database_name]
        if self.username is not None and self.password is not None:
            self.authenticated = self.db.authenticate(self.username, self.password)

        if self.capped:
            try:  # We don‘t want to override the capped collection (and it throws an error anyway)
                self.collection = Collection(self.db, self.collection_name, capped=True, max=self.capped_max, size=self.capped_size)
            except OperationFailure:
                # Capped collection exists, so get it.
                self.collection = self.db[self.collection_name]
        else:
            self.collection = self.db[self.collection_name]

    def close(self):
        """If authenticated, logging out and closing mongo database connection."""
        if self.authenticated:
            self.db.logout()
        if self.connection is not None:
            self.connection.disconnect()

    def emit(self, record):
        """Inserting new logging record to mongo database."""
        if self.collection is not None:
            try:
                self.collection.save(self.format(record))
            except Exception:
                if not self.fail_silently:
                    self.handleError(record)