logging_utils.py

Go to the documentation of this file.

#!/usr/bin/env python
from __future__ import print_function
import errno
import sys
import os
import json
import logging
import logging.config
import re
import socket
import platform

log_level = {
    'CRITICAL': logging.CRITICAL,
    'critical': logging.CRITICAL,
    'DEBUG': logging.DEBUG,
    'debug': logging.DEBUG,
    'ERROR': logging.ERROR,
    'error': logging.ERROR,
    'FATAL': logging.FATAL,
    'fatal': logging.FATAL,
    'INFO': logging.INFO,
    'info': logging.INFO,
    'WARN': logging.WARNING,
    'warn': logging.WARNING,
    'WARNING': logging.WARNING,
    'warning': logging.WARNING
}

log_file_path = os.path.join(os.path.expanduser("~"), '.log')


def get_log_config_file_path():
    directory = os.path.dirname(os.path.realpath(__file__))
    path = os.path.join(directory, 'python_logging.conf')
    return path


def configure_common_logging(level='INFO', handlers=["syslog", "console"], file_path=None):
    """
    Configure the default logging scheme, with just a few override options.

    @type level string
    @param level log level of the root handler as a string.

    @type handlers list-of-strings
    @param handlers List of named handlers.  Valid handlers are "syslog", "console", "file", or "socket".

    @type file_path string
    @param file_path Path to the log file, if the 'file' appender is used.  If file_path is just
        a file name, the logfile will be placed in the default location (/home/$USER/.log/<file_path>).
        Otherwise the entire path will be used.
    """
    global log_file_path

    config = get_common_logging_configuration()
    config['root']['level'] = level

    config['root']['handlers'] = []
    myplatform = platform.system()
    for handler in handlers:
        # syslog is not supported on windows
        if myplatform != 'Linux' and handler == 'syslog':
            print('Warning: syslog is not a valid handler on {}'.format(myplatform), file=sys.stderr)
            continue
        config['root']['handlers'].append(handler)

    if file_path is not None:
        # If the file_path is just a filename, put the log file in the default directory
        if os.path.dirname(file_path) == '':
            file_path = os.path.join(log_file_path, file_path)

        config['handlers']['file']['filename'] = file_path

    logging.config.dictConfig(config)
    logging.info("Logging to {}".format(log_file_path))
    return config


def get_common_logging_configuration():
    """
    Return the common logging configuration as a dictionary.

    When invoked, this method will create the folder ~/.log, if it does not exist
    """
    log_file_path = os.path.expanduser(os.path.join('~', '.log'))

    hostname = socket.gethostname()

    try:
        os.makedirs(log_file_path)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise e

    config = {
        "version": 1,
        "disable_existing_loggers": False,
        "formatters": {
            "brief": {
                "format": "[%(name)s][%(levelname)-8s] %(message)s"
            },
            "precise": {
                "format": "%(asctime)s {} ncl_py: [%(levelname)-8s] [%(name)-15s] %(message)s".format(hostname),
                "datefmt": "%Y-%m-%d %H:%M:%S"
            },
            "syslog": {
                "format": "ncl_py: [%(levelname)-8s] [%(name)-15s] %(message)s"
            }
        },
        "handlers": {
            "console": {
                "class": "logging.StreamHandler",
                "formatter": "precise",
                "stream": "ext://sys.stdout"
            },
            "file": {
                "class": "logging.handlers.RotatingFileHandler",
                "formatter": "precise",
                "filename": "{}".format(os.path.join(log_file_path, "nasa_common_logging.log")),
                "maxBytes": 1000000,
                "backupCount": 3
            },
            "socket": {
                "class": "logging.handlers.SocketHandler",
                "formatter": "precise",
                "host": "localhost",
                "port": 9020
            },
        },
        "root": {
            "level": "NOTSET",
            "handlers": ["console"],
            "propagate": "no"
        }
    }

    # the syslog handler is only valid in Linux, not in macOS or Windows
    if platform.system() == 'Linux':
        config['handlers']['syslog'] = {
                "class": "logging.handlers.SysLogHandler",
                "formatter": "syslog",
                "address": "/dev/log",
                "facility": "local5"
            }
    return config


def configure_custom_logging(config):
    """
    Configure the root logger with a custom config.

    NOTE: One could call get_common_logging_configuration(), modify the contents, and simply
    call this method as an easy means of changing the logging configuration
    """
    logging.config.dictConfig(config)


def configure_custom_logging_file(config_file):
    """
    Configure the root logger from a JSON-format logging config file.
    """
    with open(config_file, 'r') as fp:
        config = json.load(fp)
        fp.close()
    logging.config.dictConfig(config)


"""
The following lines contain the regular expressions needed to parse from the syslog
"""
re_fill  = '.*?'
re_syslog_time_stamp = '[A-Z][a-z]{2,}\s+\d{1,2} \d{2}:\d{2}:\d{2}'
re_ncl_time_stamp    =  '\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}'

re_timestamp = '({}|{})'.format(re_syslog_time_stamp,re_ncl_time_stamp)
re_host      = '\s*([^\s]+)\s*'
re_source    = '\s*([^\s]+)\s*\\:'
re_level     = '\\[(.*?)\s*\\]'
re_category  = '\\[(.*?)\s*\\]'
re_msg       = '((?:.*\n?)(?:^(?!{}).*\n?)*)'.format(re_timestamp)

#   to explain:
#      ".*\n"- match everything up to and including a new-line
#      "(?: <expr>)" - match <expr>, but don't put it in a group
#
#   Therefore:
#      1) "(?:.*\n)" - match the rest of the first line, but don't put it in it's own group
#      2) "(?:^(?!{}).*\n)*" - match 0 or more other lines that DON'T start with a timestamp
#
#   Then, by enclosing 1) and 2) in a set of parentheses, the entire message will be grouped together.

re_ncl_log_entry = '^' + re_timestamp + re_fill + re_host + re_fill + re_source + re_fill + re_level + re_fill + re_category + '\s*' + re_msg
rg_ncl_log_entry = re.compile(re_ncl_log_entry,re.MULTILINE)

re_generic_log_entry = '^' + re_timestamp + re_fill + re_host + re_fill + re_source + '\s*' + re_msg
rg_generic_log_entry = re.compile(re_generic_log_entry,re.MULTILINE)

def parse_log_buffer(buffer):
    """
    Parse log entries from a buffer.  This buffer could be a portion, or an entire log file.  This method will only
    detect and properly parse log messages that are formatted using the nasa_common_logging default log format.  For
    syslog, this is logs using the "syslog" formatter or the "precise" formatter.  For formatter definitions, see
    "get_common_logging_configuration"

    Args:
        buffer: A single string from a nasa_common_logging formatted log file.

    Returns:
        List of log entries.  Each entry is a tuple: (timestamp, source, tag, loglevel, message)
    """
    entries = []
    matched = False

    matches = rg_ncl_log_entry.finditer(buffer)
    for m in matches:
        if len(m.groups()) == 7:
            t = m.group(1)
            host = m.group(2)
            src = m.group(3)
            lvl = m.group(4)
            cat = m.group(5)
            msg = m.group(6).rstrip('\n')
            entries.append((t, host, src, lvl, cat, msg))
            matched = True
    if matched:
        return entries

    matches = rg_generic_log_entry.finditer(buffer)
    for m in matches:
        if len(m.groups()) == 5:
            t = m.group(1)
            host = m.group(2)
            src = m.group(3)
            lvl = None
            cat = None
            msg = m.group(4).rstrip('\n')
            entries.append((t, host, src, lvl, cat, msg))

    return entries

class PreciseFormatter(logging.Formatter):

    def __init__(self, fmt=None, datefmt=None):

        hostname = socket.gethostname()
        format = "%(asctime)s {} ncl_py: [%(levelname)-8s] [%(name)-15s] %(message)s".format(hostname)
        datefmt= "%Y-%m-%d %H:%M:%S"
        super(PreciseFormatter, self).__init__(format,datefmt)