@@ -1409,13 +1420,16 @@
switch ($(this).val()) {
case 'xld':
$("#xldproperties").slideDown();
- break;
+ break;
case 'ffmpeg':
$("#lameffmpegproperties").slideDown();
- break;
+ break;
+ case 'libav':
+ $("#lameffmpegproperties").slideDown();
+ break;
case 'lame':
$("#lameffmpegproperties").slideDown();
- break;
+ break;
}
};
diff --git a/headphones/logger.py b/headphones/logger.py
index 4a21f3f8..3c384643 100644
--- a/headphones/logger.py
+++ b/headphones/logger.py
@@ -13,24 +13,30 @@
# You should have received a copy of the GNU General Public License
# along with Headphones. If not, see .
-import os
-import sys
-import logging
-import traceback
-import threading
-import headphones
-
-from logging import handlers
-
from headphones import helpers
+from logutils.queue import QueueHandler, QueueListener
+from logging import handlers
+
+import multiprocessing
+import contextlib
+import headphones
+import threading
+import traceback
+import logging
+import sys
+import os
+
# These settings are for file logging only
-FILENAME = 'headphones.log'
+FILENAME = "headphones.log"
MAX_SIZE = 1000000 # 1 MB
MAX_FILES = 5
# Headphones logger
-logger = logging.getLogger('headphones')
+logger = logging.getLogger("headphones")
+
+# Global queue of multiprocessing logging
+queue = multiprocessing.Queue()
class LogListHandler(logging.Handler):
"""
@@ -43,6 +49,39 @@ class LogListHandler(logging.Handler):
headphones.LOG_LIST.insert(0, (helpers.now(), message, record.levelname, record.threadName))
+@contextlib.contextmanager
+def listener():
+ """
+ Wrapper that create a QueueListener, starts it and automatically stops it.
+ To be used in a with statement in the main process, for multiprocessing.
+ """
+
+ queue_listener = QueueListener(queue, *logger.handlers)
+
+ try:
+ queue_listener.start()
+ yield
+ finally:
+ queue_listener.stop()
+
+def initMultiprocessing():
+ """
+ Remove all handlers and add QueueHandler on top. This should only be called
+ inside a multiprocessing worker process, since it changes the logger
+ completely.
+ """
+
+ for handler in logger.handlers[:]:
+ logger.removeHandler(handler)
+
+ queue_handler = QueueHandler(queue)
+ queue_handler.setLevel(logging.DEBUG)
+
+ logger.addHandler(queue_handler)
+
+ # Change current thread name for log record
+ threading.current_thread().name = multiprocessing.current_process().name
+
def initLogger(console=False, verbose=False):
"""
Setup logging for Headphones. It uses the logger instance with the name
diff --git a/headphones/music_encoder.py b/headphones/music_encoder.py
index be7f705f..48c93a50 100644
--- a/headphones/music_encoder.py
+++ b/headphones/music_encoder.py
@@ -53,7 +53,7 @@ def encode(albumPath):
time.sleep(1)
os.mkdir(tempDirEncode)
- except Exception, e:
+ except Exception as e:
logger.exception("Unable to create temporary directory")
return None
@@ -96,13 +96,18 @@ def encode(albumPath):
encoder = "C:/Program Files/ffmpeg/bin/ffmpeg.exe"
else:
encoder="ffmpeg"
+ elif headphones.ENCODER == 'libav':
+ if headphones.SYS_PLATFORM == "win32":
+ encoder = "C:/Program Files/libav/bin/avconv.exe"
+ else:
+ encoder="avconv"
i=0
encoder_failed = False
jobs = []
for music in musicFiles:
- infoMusic=MediaFile(music)
+ infoMusic = MediaFile(music)
encode = False
if XLD:
@@ -141,29 +146,32 @@ def encode(albumPath):
# Encode music files
if len(jobs) > 0:
+ processes = 1
+
+ # Use multicore if enabled
if headphones.ENCODER_MULTICORE:
if headphones.ENCODER_MULTICORE_COUNT == 0:
processes = multiprocessing.cpu_count()
else:
processes = headphones.ENCODER_MULTICORE_COUNT
- logger.debug("Multi-core encoding enabled, %d processes", processes)
- else:
- processes = 1
+ logger.debug("Multi-core encoding enabled, spawning %d processes",
+ processes)
# Use multiprocessing only if it's worth the overhead. and if it is
# enabled. If not, then use the old fashioned way.
if processes > 1:
- pool = multiprocessing.Pool(processes=processes)
- results = pool.map_async(command_map, jobs)
+ with logger.listener():
+ pool = multiprocessing.Pool(processes=processes)
+ results = pool.map_async(command_map, jobs)
- # No new processes will be created, so close it and wait for all
- # processes to finish
- pool.close()
- pool.join()
+ # No new processes will be created, so close it and wait for all
+ # processes to finish
+ pool.close()
+ pool.join()
- # Retrieve the results
- results = results.get()
+ # Retrieve the results
+ results = results.get()
else:
results = map(command_map, jobs)
@@ -178,7 +186,7 @@ def encode(albumPath):
for dest in musicTempFiles:
if not os.path.exists(dest):
encoder_failed = True
- logger.error('Encoded file \'%s\' does not exist in the destination temp directory', dest)
+ logger.error("Encoded file '%s' does not exist in the destination temp directory", dest)
# No errors, move from temp to parent
if not encoder_failed and musicTempFiles:
@@ -204,7 +212,7 @@ def encode(albumPath):
# Return with error if any encoding errors
if encoder_failed:
- logger.error('One or more files failed to encode, check debuglog and ensure you have the latest version of %s installed', headphones.ENCODER)
+ logger.error("One or more files failed to encode. Ensure you have the latest version of %s installed.", headphones.ENCODER)
return None
time.sleep(1)
@@ -220,19 +228,31 @@ def encode(albumPath):
def command_map(args):
"""
- This method is used for the multiprocessing.map() method as a wrapper.
+ Wrapper for the '[multiprocessing.]map()' method, to unpack the arguments
+ and wrap exceptions.
"""
+ # Initialize multiprocessing logger
+ if multiprocessing.current_process().name != "MainProcess":
+ logger.initMultiprocessing()
+
+ # Start encoding
try:
return command(*args)
- except Exception, e:
- logger.exception("Encoder exception, will return failed")
+ except Exception as e:
+ logger.exception("Encoder raised an exception.")
return False
-def command(encoder, musicSource ,musicDest, albumPath):
- cmd=[]
- startMusicTime=time.time()
+def command(encoder, musicSource, musicDest, albumPath):
+ """
+ Encode a given music file with a certain encoder. Returns True on success,
+ or False otherwise.
+ """
+ startMusicTime = time.time()
+ cmd = []
+
+ # XLD
if XLD:
xldDestDir = os.path.split(musicDest)[0]
cmd = [encoder]
@@ -242,10 +262,11 @@ def command(encoder, musicSource ,musicDest, albumPath):
cmd.extend(['-o'])
cmd.extend([xldDestDir])
+ # Lame
elif headphones.ENCODER == 'lame':
cmd = [encoder]
opts = []
- if headphones.ADVANCEDENCODER =='':
+ if not headphones.ADVANCEDENCODER:
opts.extend(['-h'])
if headphones.ENCODERVBRCBR=='cbr':
opts.extend(['--resample', str(headphones.SAMPLINGFREQUENCY), '-b', str(headphones.BITRATE)])
@@ -259,10 +280,11 @@ def command(encoder, musicSource ,musicDest, albumPath):
opts.extend([musicDest])
cmd.extend(opts)
+ # FFmpeg
elif headphones.ENCODER == 'ffmpeg':
cmd = [encoder, '-i', musicSource]
opts = []
- if headphones.ADVANCEDENCODER =='':
+ if not headphones.ADVANCEDENCODER:
if headphones.ENCODEROUTPUTFORMAT=='ogg':
opts.extend(['-acodec', 'libvorbis'])
if headphones.ENCODEROUTPUTFORMAT=='m4a':
@@ -279,13 +301,30 @@ def command(encoder, musicSource ,musicDest, albumPath):
opts.extend([musicDest])
cmd.extend(opts)
- # Encode
+ # Libav
+ elif headphones.ENCODER == "libav":
+ cmd = [encoder, '-i', musicSource]
+ opts = []
+ if not headphones.ADVANCEDENCODER:
+ if headphones.ENCODEROUTPUTFORMAT=='ogg':
+ opts.extend(['-acodec', 'libvorbis'])
+ if headphones.ENCODEROUTPUTFORMAT=='m4a':
+ opts.extend(['-strict', 'experimental'])
+ if headphones.ENCODERVBRCBR=='cbr':
+ opts.extend(['-ar', str(headphones.SAMPLINGFREQUENCY), '-ab', str(headphones.BITRATE) + 'k'])
+ elif headphones.ENCODERVBRCBR=='vbr':
+ opts.extend(['-aq', str(headphones.ENCODERQUALITY)])
+ opts.extend(['-y', '-ac', '2', '-vn'])
+ else:
+ advanced = (headphones.ADVANCEDENCODER.split())
+ for tok in advanced:
+ opts.extend([tok.encode(headphones.SYS_ENCODING)])
+ opts.extend([musicDest])
+ cmd.extend(opts)
- logger.info('Encoding %s...' % (musicSource.decode(headphones.SYS_ENCODING, 'replace')))
- logger.debug(subprocess.list2cmdline(cmd))
-
- # stop windows opening the cmd
+ # Prevent Windows from opening a terminal window
startupinfo = None
+
if headphones.SYS_PLATFORM == "win32":
startupinfo = subprocess.STARTUPINFO()
try:
@@ -293,12 +332,17 @@ def command(encoder, musicSource ,musicDest, albumPath):
except AttributeError:
startupinfo.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW
- p = subprocess.Popen(cmd, startupinfo=startupinfo, stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ # Encode
+ logger.info('Encoding %s...' % (musicSource.decode(headphones.SYS_ENCODING, 'replace')))
+ logger.debug(subprocess.list2cmdline(cmd))
- stdout, stderr = p.communicate(headphones.ENCODER)
+ process = subprocess.Popen(cmd, startupinfo=startupinfo,
+ stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ stdout, stderr = process.communicate(headphones.ENCODER)
- # error if return code not zero
- if p.returncode:
+ # Error if return code not zero
+ if process.returncode:
logger.error('Encoding failed for %s' % (musicSource.decode(headphones.SYS_ENCODING, 'replace')))
out = stdout if stdout else stderr
out = out.decode(headphones.SYS_ENCODING, 'replace')
@@ -319,5 +363,4 @@ def getTimeEncode(start):
seconds -= 3600*hours
minutes = seconds / 60
seconds -= 60*minutes
- return "%02d:%02d:%02d" % (hours, minutes, seconds)
-
+ return "%02d:%02d:%02d" % (hours, minutes, seconds)
\ No newline at end of file
diff --git a/lib/logutils/__init__.py b/lib/logutils/__init__.py
new file mode 100644
index 00000000..44e261f8
--- /dev/null
+++ b/lib/logutils/__init__.py
@@ -0,0 +1,195 @@
+#
+# Copyright (C) 2010-2013 Vinay Sajip. See LICENSE.txt for details.
+#
+"""
+The logutils package provides a set of handlers for the Python standard
+library's logging package.
+
+Some of these handlers are out-of-scope for the standard library, and
+so they are packaged here. Others are updated versions which have
+appeared in recent Python releases, but are usable with older versions
+of Python, and so are packaged here.
+"""
+import logging
+from string import Template
+
+__version__ = '0.3.3'
+
+class NullHandler(logging.Handler):
+ """
+ This handler does nothing. It's intended to be used to avoid the
+ "No handlers could be found for logger XXX" one-off warning. This is
+ important for library code, which may contain code to log events. If a user
+ of the library does not configure logging, the one-off warning might be
+ produced; to avoid this, the library developer simply needs to instantiate
+ a NullHandler and add it to the top-level logger of the library module or
+ package.
+ """
+
+ def handle(self, record):
+ """
+ Handle a record. Does nothing in this class, but in other
+ handlers it typically filters and then emits the record in a
+ thread-safe way.
+ """
+ pass
+
+ def emit(self, record):
+ """
+ Emit a record. This does nothing and shouldn't be called during normal
+ processing, unless you redefine :meth:`~logutils.NullHandler.handle`.
+ """
+ pass
+
+ def createLock(self):
+ """
+ Since this handler does nothing, it has no underlying I/O to protect
+ against multi-threaded access, so this method returns `None`.
+ """
+ self.lock = None
+
+class PercentStyle(object):
+
+ default_format = '%(message)s'
+ asctime_format = '%(asctime)s'
+
+ def __init__(self, fmt):
+ self._fmt = fmt or self.default_format
+
+ def usesTime(self):
+ return self._fmt.find(self.asctime_format) >= 0
+
+ def format(self, record):
+ return self._fmt % record.__dict__
+
+class StrFormatStyle(PercentStyle):
+ default_format = '{message}'
+ asctime_format = '{asctime}'
+
+ def format(self, record):
+ return self._fmt.format(**record.__dict__)
+
+
+class StringTemplateStyle(PercentStyle):
+ default_format = '${message}'
+ asctime_format = '${asctime}'
+
+ def __init__(self, fmt):
+ self._fmt = fmt or self.default_format
+ self._tpl = Template(self._fmt)
+
+ def usesTime(self):
+ fmt = self._fmt
+ return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_format) >= 0
+
+ def format(self, record):
+ return self._tpl.substitute(**record.__dict__)
+
+_STYLES = {
+ '%': PercentStyle,
+ '{': StrFormatStyle,
+ '$': StringTemplateStyle
+}
+
+class Formatter(logging.Formatter):
+ """
+ Subclasses Formatter in Pythons earlier than 3.2 in order to give
+ 3.2 Formatter behaviour with respect to allowing %-, {} or $-
+ formatting.
+ """
+ def __init__(self, fmt=None, datefmt=None, style='%'):
+ """
+ Initialize the formatter with specified format strings.
+
+ Initialize the formatter either with the specified format string, or a
+ default as described above. Allow for specialized date formatting with
+ the optional datefmt argument (if omitted, you get the ISO8601 format).
+
+ Use a style parameter of '%', '{' or '$' to specify that you want to
+ use one of %-formatting, :meth:`str.format` (``{}``) formatting or
+ :class:`string.Template` formatting in your format string.
+ """
+ if style not in _STYLES:
+ raise ValueError('Style must be one of: %s' % ','.join(
+ _STYLES.keys()))
+ self._style = _STYLES[style](fmt)
+ self._fmt = self._style._fmt
+ self.datefmt = datefmt
+
+ def usesTime(self):
+ """
+ Check if the format uses the creation time of the record.
+ """
+ return self._style.usesTime()
+
+ def formatMessage(self, record):
+ return self._style.format(record)
+
+ def format(self, record):
+ """
+ Format the specified record as text.
+
+ The record's attribute dictionary is used as the operand to a
+ string formatting operation which yields the returned string.
+ Before formatting the dictionary, a couple of preparatory steps
+ are carried out. The message attribute of the record is computed
+ using LogRecord.getMessage(). If the formatting string uses the
+ time (as determined by a call to usesTime(), formatTime() is
+ called to format the event time. If there is exception information,
+ it is formatted using formatException() and appended to the message.
+ """
+ record.message = record.getMessage()
+ if self.usesTime():
+ record.asctime = self.formatTime(record, self.datefmt)
+ s = self.formatMessage(record)
+ if record.exc_info:
+ # Cache the traceback text to avoid converting it multiple times
+ # (it's constant anyway)
+ if not record.exc_text:
+ record.exc_text = self.formatException(record.exc_info)
+ if record.exc_text:
+ if s[-1:] != "\n":
+ s = s + "\n"
+ s = s + record.exc_text
+ return s
+
+
+class BraceMessage(object):
+ def __init__(self, fmt, *args, **kwargs):
+ self.fmt = fmt
+ self.args = args
+ self.kwargs = kwargs
+ self.str = None
+
+ def __str__(self):
+ if self.str is None:
+ self.str = self.fmt.format(*self.args, **self.kwargs)
+ return self.str
+
+class DollarMessage(object):
+ def __init__(self, fmt, **kwargs):
+ self.fmt = fmt
+ self.kwargs = kwargs
+ self.str = None
+
+ def __str__(self):
+ if self.str is None:
+ self.str = Template(self.fmt).substitute(**self.kwargs)
+ return self.str
+
+
+def hasHandlers(logger):
+ """
+ See if a logger has any handlers.
+ """
+ rv = False
+ while logger:
+ if logger.handlers:
+ rv = True
+ break
+ elif not logger.propagate:
+ break
+ else:
+ logger = logger.parent
+ return rv
+
diff --git a/lib/logutils/adapter.py b/lib/logutils/adapter.py
new file mode 100644
index 00000000..399e1eed
--- /dev/null
+++ b/lib/logutils/adapter.py
@@ -0,0 +1,116 @@
+#
+# Copyright (C) 2010-2013 Vinay Sajip. See LICENSE.txt for details.
+#
+import logging
+import logutils
+
+class LoggerAdapter(object):
+ """
+ An adapter for loggers which makes it easier to specify contextual
+ information in logging output.
+ """
+
+ def __init__(self, logger, extra):
+ """
+ Initialize the adapter with a logger and a dict-like object which
+ provides contextual information. This constructor signature allows
+ easy stacking of LoggerAdapters, if so desired.
+
+ You can effectively pass keyword arguments as shown in the
+ following example:
+
+ adapter = LoggerAdapter(someLogger, dict(p1=v1, p2="v2"))
+ """
+ self.logger = logger
+ self.extra = extra
+
+ def process(self, msg, kwargs):
+ """
+ Process the logging message and keyword arguments passed in to
+ a logging call to insert contextual information. You can either
+ manipulate the message itself, the keyword args or both. Return
+ the message and kwargs modified (or not) to suit your needs.
+
+ Normally, you'll only need to override this one method in a
+ LoggerAdapter subclass for your specific needs.
+ """
+ kwargs["extra"] = self.extra
+ return msg, kwargs
+
+ #
+ # Boilerplate convenience methods
+ #
+ def debug(self, msg, *args, **kwargs):
+ """
+ Delegate a debug call to the underlying logger.
+ """
+ self.log(logging.DEBUG, msg, *args, **kwargs)
+
+ def info(self, msg, *args, **kwargs):
+ """
+ Delegate an info call to the underlying logger.
+ """
+ self.log(logging.INFO, msg, *args, **kwargs)
+
+ def warning(self, msg, *args, **kwargs):
+ """
+ Delegate a warning call to the underlying logger.
+ """
+ self.log(logging.WARNING, msg, *args, **kwargs)
+
+ warn = warning
+
+ def error(self, msg, *args, **kwargs):
+ """
+ Delegate an error call to the underlying logger.
+ """
+ self.log(logging.ERROR, msg, *args, **kwargs)
+
+ def exception(self, msg, *args, **kwargs):
+ """
+ Delegate an exception call to the underlying logger.
+ """
+ kwargs["exc_info"] = 1
+ self.log(logging.ERROR, msg, *args, **kwargs)
+
+ def critical(self, msg, *args, **kwargs):
+ """
+ Delegate a critical call to the underlying logger.
+ """
+ self.log(logging.CRITICAL, msg, *args, **kwargs)
+
+ def log(self, level, msg, *args, **kwargs):
+ """
+ Delegate a log call to the underlying logger, after adding
+ contextual information from this adapter instance.
+ """
+ if self.isEnabledFor(level):
+ msg, kwargs = self.process(msg, kwargs)
+ self.logger._log(level, msg, args, **kwargs)
+
+ def isEnabledFor(self, level):
+ """
+ Is this logger enabled for level 'level'?
+ """
+ if self.logger.manager.disable >= level:
+ return False
+ return level >= self.getEffectiveLevel()
+
+ def setLevel(self, level):
+ """
+ Set the specified level on the underlying logger.
+ """
+ self.logger.setLevel(level)
+
+ def getEffectiveLevel(self):
+ """
+ Get the effective level for the underlying logger.
+ """
+ return self.logger.getEffectiveLevel()
+
+ def hasHandlers(self):
+ """
+ See if the underlying logger has any handlers.
+ """
+ return logutils.hasHandlers(self.logger)
+
diff --git a/lib/logutils/colorize.py b/lib/logutils/colorize.py
new file mode 100644
index 00000000..2c396394
--- /dev/null
+++ b/lib/logutils/colorize.py
@@ -0,0 +1,194 @@
+#
+# Copyright (C) 2010-2013 Vinay Sajip. All rights reserved.
+#
+import ctypes
+import logging
+import os
+
+try:
+ unicode
+except NameError:
+ unicode = None
+
+class ColorizingStreamHandler(logging.StreamHandler):
+ """
+ A stream handler which supports colorizing of console streams
+ under Windows, Linux and Mac OS X.
+
+ :param strm: The stream to colorize - typically ``sys.stdout``
+ or ``sys.stderr``.
+ """
+
+ # color names to indices
+ color_map = {
+ 'black': 0,
+ 'red': 1,
+ 'green': 2,
+ 'yellow': 3,
+ 'blue': 4,
+ 'magenta': 5,
+ 'cyan': 6,
+ 'white': 7,
+ }
+
+ #levels to (background, foreground, bold/intense)
+ if os.name == 'nt':
+ level_map = {
+ logging.DEBUG: (None, 'blue', True),
+ logging.INFO: (None, 'white', False),
+ logging.WARNING: (None, 'yellow', True),
+ logging.ERROR: (None, 'red', True),
+ logging.CRITICAL: ('red', 'white', True),
+ }
+ else:
+ "Maps levels to colour/intensity settings."
+ level_map = {
+ logging.DEBUG: (None, 'blue', False),
+ logging.INFO: (None, 'black', False),
+ logging.WARNING: (None, 'yellow', False),
+ logging.ERROR: (None, 'red', False),
+ logging.CRITICAL: ('red', 'white', True),
+ }
+
+ csi = '\x1b['
+ reset = '\x1b[0m'
+
+ @property
+ def is_tty(self):
+ "Returns true if the handler's stream is a terminal."
+ isatty = getattr(self.stream, 'isatty', None)
+ return isatty and isatty()
+
+ def emit(self, record):
+ try:
+ message = self.format(record)
+ stream = self.stream
+ if unicode and isinstance(message, unicode):
+ enc = getattr(stream, 'encoding', 'utf-8')
+ message = message.encode(enc, 'replace')
+ if not self.is_tty:
+ stream.write(message)
+ else:
+ self.output_colorized(message)
+ stream.write(getattr(self, 'terminator', '\n'))
+ self.flush()
+ except (KeyboardInterrupt, SystemExit):
+ raise
+ except:
+ self.handleError(record)
+
+ if os.name != 'nt':
+ def output_colorized(self, message):
+ """
+ Output a colorized message.
+
+ On Linux and Mac OS X, this method just writes the
+ already-colorized message to the stream, since on these
+ platforms console streams accept ANSI escape sequences
+ for colorization. On Windows, this handler implements a
+ subset of ANSI escape sequence handling by parsing the
+ message, extracting the sequences and making Win32 API
+ calls to colorize the output.
+
+ :param message: The message to colorize and output.
+ """
+ self.stream.write(message)
+ else:
+ import re
+ ansi_esc = re.compile(r'\x1b\[((?:\d+)(?:;(?:\d+))*)m')
+
+ nt_color_map = {
+ 0: 0x00, # black
+ 1: 0x04, # red
+ 2: 0x02, # green
+ 3: 0x06, # yellow
+ 4: 0x01, # blue
+ 5: 0x05, # magenta
+ 6: 0x03, # cyan
+ 7: 0x07, # white
+ }
+
+ def output_colorized(self, message):
+ """
+ Output a colorized message.
+
+ On Linux and Mac OS X, this method just writes the
+ already-colorized message to the stream, since on these
+ platforms console streams accept ANSI escape sequences
+ for colorization. On Windows, this handler implements a
+ subset of ANSI escape sequence handling by parsing the
+ message, extracting the sequences and making Win32 API
+ calls to colorize the output.
+
+ :param message: The message to colorize and output.
+ """
+ parts = self.ansi_esc.split(message)
+ write = self.stream.write
+ h = None
+ fd = getattr(self.stream, 'fileno', None)
+ if fd is not None:
+ fd = fd()
+ if fd in (1, 2): # stdout or stderr
+ h = ctypes.windll.kernel32.GetStdHandle(-10 - fd)
+ while parts:
+ text = parts.pop(0)
+ if text:
+ write(text)
+ if parts:
+ params = parts.pop(0)
+ if h is not None:
+ params = [int(p) for p in params.split(';')]
+ color = 0
+ for p in params:
+ if 40 <= p <= 47:
+ color |= self.nt_color_map[p - 40] << 4
+ elif 30 <= p <= 37:
+ color |= self.nt_color_map[p - 30]
+ elif p == 1:
+ color |= 0x08 # foreground intensity on
+ elif p == 0: # reset to default color
+ color = 0x07
+ else:
+ pass # error condition ignored
+ ctypes.windll.kernel32.SetConsoleTextAttribute(h, color)
+
+ def colorize(self, message, record):
+ """
+ Colorize a message for a logging event.
+
+ This implementation uses the ``level_map`` class attribute to
+ map the LogRecord's level to a colour/intensity setting, which is
+ then applied to the whole message.
+
+ :param message: The message to colorize.
+ :param record: The ``LogRecord`` for the message.
+ """
+ if record.levelno in self.level_map:
+ bg, fg, bold = self.level_map[record.levelno]
+ params = []
+ if bg in self.color_map:
+ params.append(str(self.color_map[bg] + 40))
+ if fg in self.color_map:
+ params.append(str(self.color_map[fg] + 30))
+ if bold:
+ params.append('1')
+ if params:
+ message = ''.join((self.csi, ';'.join(params),
+ 'm', message, self.reset))
+ return message
+
+ def format(self, record):
+ """
+ Formats a record for output.
+
+ This implementation colorizes the message line, but leaves
+ any traceback unolorized.
+ """
+ message = logging.StreamHandler.format(self, record)
+ if self.is_tty:
+ # Don't colorize any traceback
+ parts = message.split('\n', 1)
+ parts[0] = self.colorize(parts[0], record)
+ message = '\n'.join(parts)
+ return message
+
diff --git a/lib/logutils/dictconfig.py b/lib/logutils/dictconfig.py
new file mode 100644
index 00000000..4a2281f3
--- /dev/null
+++ b/lib/logutils/dictconfig.py
@@ -0,0 +1,573 @@
+#
+# Copyright (C) 2009-2013 Vinay Sajip. See LICENSE.txt for details.
+#
+import logging.handlers
+import re
+import sys
+import types
+
+try:
+ basestring
+except NameError:
+ basestring = str
+try:
+ StandardError
+except NameError:
+ StandardError = Exception
+
+IDENTIFIER = re.compile('^[a-z_][a-z0-9_]*$', re.I)
+
+def valid_ident(s):
+ m = IDENTIFIER.match(s)
+ if not m:
+ raise ValueError('Not a valid Python identifier: %r' % s)
+ return True
+
+#
+# This function is defined in logging only in recent versions of Python
+#
+try:
+ from logging import _checkLevel
+except ImportError:
+ def _checkLevel(level):
+ if isinstance(level, int):
+ rv = level
+ elif str(level) == level:
+ if level not in logging._levelNames:
+ raise ValueError('Unknown level: %r' % level)
+ rv = logging._levelNames[level]
+ else:
+ raise TypeError('Level not an integer or a '
+ 'valid string: %r' % level)
+ return rv
+
+# The ConvertingXXX classes are wrappers around standard Python containers,
+# and they serve to convert any suitable values in the container. The
+# conversion converts base dicts, lists and tuples to their wrapped
+# equivalents, whereas strings which match a conversion format are converted
+# appropriately.
+#
+# Each wrapper should have a configurator attribute holding the actual
+# configurator to use for conversion.
+
+class ConvertingDict(dict):
+ """A converting dictionary wrapper."""
+
+ def __getitem__(self, key):
+ value = dict.__getitem__(self, key)
+ result = self.configurator.convert(value)
+ #If the converted value is different, save for next time
+ if value is not result:
+ self[key] = result
+ if type(result) in (ConvertingDict, ConvertingList,
+ ConvertingTuple):
+ result.parent = self
+ result.key = key
+ return result
+
+ def get(self, key, default=None):
+ value = dict.get(self, key, default)
+ result = self.configurator.convert(value)
+ #If the converted value is different, save for next time
+ if value is not result:
+ self[key] = result
+ if type(result) in (ConvertingDict, ConvertingList,
+ ConvertingTuple):
+ result.parent = self
+ result.key = key
+ return result
+
+ def pop(self, key, default=None):
+ value = dict.pop(self, key, default)
+ result = self.configurator.convert(value)
+ if value is not result:
+ if type(result) in (ConvertingDict, ConvertingList,
+ ConvertingTuple):
+ result.parent = self
+ result.key = key
+ return result
+
+class ConvertingList(list):
+ """A converting list wrapper."""
+ def __getitem__(self, key):
+ value = list.__getitem__(self, key)
+ result = self.configurator.convert(value)
+ #If the converted value is different, save for next time
+ if value is not result:
+ self[key] = result
+ if type(result) in (ConvertingDict, ConvertingList,
+ ConvertingTuple):
+ result.parent = self
+ result.key = key
+ return result
+
+ def pop(self, idx=-1):
+ value = list.pop(self, idx)
+ result = self.configurator.convert(value)
+ if value is not result:
+ if type(result) in (ConvertingDict, ConvertingList,
+ ConvertingTuple):
+ result.parent = self
+ return result
+
+class ConvertingTuple(tuple):
+ """A converting tuple wrapper."""
+ def __getitem__(self, key):
+ value = tuple.__getitem__(self, key)
+ result = self.configurator.convert(value)
+ if value is not result:
+ if type(result) in (ConvertingDict, ConvertingList,
+ ConvertingTuple):
+ result.parent = self
+ result.key = key
+ return result
+
+class BaseConfigurator(object):
+ """
+ The configurator base class which defines some useful defaults.
+ """
+
+ CONVERT_PATTERN = re.compile(r'^(?P[a-z]+)://(?P.*)$')
+
+ WORD_PATTERN = re.compile(r'^\s*(\w+)\s*')
+ DOT_PATTERN = re.compile(r'^\.\s*(\w+)\s*')
+ INDEX_PATTERN = re.compile(r'^\[\s*(\w+)\s*\]\s*')
+ DIGIT_PATTERN = re.compile(r'^\d+$')
+
+ value_converters = {
+ 'ext' : 'ext_convert',
+ 'cfg' : 'cfg_convert',
+ }
+
+ # We might want to use a different one, e.g. importlib
+ importer = __import__
+ "Allows the importer to be redefined."
+
+ def __init__(self, config):
+ """
+ Initialise an instance with the specified configuration
+ dictionary.
+ """
+ self.config = ConvertingDict(config)
+ self.config.configurator = self
+
+ def resolve(self, s):
+ """
+ Resolve strings to objects using standard import and attribute
+ syntax.
+ """
+ name = s.split('.')
+ used = name.pop(0)
+ try:
+ found = self.importer(used)
+ for frag in name:
+ used += '.' + frag
+ try:
+ found = getattr(found, frag)
+ except AttributeError:
+ self.importer(used)
+ found = getattr(found, frag)
+ return found
+ except ImportError:
+ e, tb = sys.exc_info()[1:]
+ v = ValueError('Cannot resolve %r: %s' % (s, e))
+ v.__cause__, v.__traceback__ = e, tb
+ raise v
+
+ def ext_convert(self, value):
+ """Default converter for the ext:// protocol."""
+ return self.resolve(value)
+
+ def cfg_convert(self, value):
+ """Default converter for the cfg:// protocol."""
+ rest = value
+ m = self.WORD_PATTERN.match(rest)
+ if m is None:
+ raise ValueError("Unable to convert %r" % value)
+ else:
+ rest = rest[m.end():]
+ d = self.config[m.groups()[0]]
+ #print d, rest
+ while rest:
+ m = self.DOT_PATTERN.match(rest)
+ if m:
+ d = d[m.groups()[0]]
+ else:
+ m = self.INDEX_PATTERN.match(rest)
+ if m:
+ idx = m.groups()[0]
+ if not self.DIGIT_PATTERN.match(idx):
+ d = d[idx]
+ else:
+ try:
+ n = int(idx) # try as number first (most likely)
+ d = d[n]
+ except TypeError:
+ d = d[idx]
+ if m:
+ rest = rest[m.end():]
+ else:
+ raise ValueError('Unable to convert '
+ '%r at %r' % (value, rest))
+ #rest should be empty
+ return d
+
+ def convert(self, value):
+ """
+ Convert values to an appropriate type. dicts, lists and tuples are
+ replaced by their converting alternatives. Strings are checked to
+ see if they have a conversion format and are converted if they do.
+ """
+ if not isinstance(value, ConvertingDict) and isinstance(value, dict):
+ value = ConvertingDict(value)
+ value.configurator = self
+ elif not isinstance(value, ConvertingList) and isinstance(value, list):
+ value = ConvertingList(value)
+ value.configurator = self
+ elif not isinstance(value, ConvertingTuple) and\
+ isinstance(value, tuple):
+ value = ConvertingTuple(value)
+ value.configurator = self
+ elif isinstance(value, basestring):
+ m = self.CONVERT_PATTERN.match(value)
+ if m:
+ d = m.groupdict()
+ prefix = d['prefix']
+ converter = self.value_converters.get(prefix, None)
+ if converter:
+ suffix = d['suffix']
+ converter = getattr(self, converter)
+ value = converter(suffix)
+ return value
+
+ def configure_custom(self, config):
+ """Configure an object with a user-supplied factory."""
+ c = config.pop('()')
+ if isinstance(c, basestring):
+ c = self.resolve(c)
+ props = config.pop('.', None)
+ # Check for valid identifiers
+ kwargs = dict([(k, config[k]) for k in config if valid_ident(k)])
+ result = c(**kwargs)
+ if props:
+ for name, value in props.items():
+ setattr(result, name, value)
+ return result
+
+ def as_tuple(self, value):
+ """Utility function which converts lists to tuples."""
+ if isinstance(value, list):
+ value = tuple(value)
+ return value
+
+def named_handlers_supported():
+ major, minor = sys.version_info[:2]
+ if major == 2:
+ result = minor >= 7
+ elif major == 3:
+ result = minor >= 2
+ else:
+ result = (major > 3)
+ return result
+
+class DictConfigurator(BaseConfigurator):
+ """
+ Configure logging using a dictionary-like object to describe the
+ configuration.
+ """
+
+ def configure(self):
+ """Do the configuration."""
+
+ config = self.config
+ if 'version' not in config:
+ raise ValueError("dictionary doesn't specify a version")
+ if config['version'] != 1:
+ raise ValueError("Unsupported version: %s" % config['version'])
+ incremental = config.pop('incremental', False)
+ EMPTY_DICT = {}
+ logging._acquireLock()
+ try:
+ if incremental:
+ handlers = config.get('handlers', EMPTY_DICT)
+ # incremental handler config only if handler name
+ # ties in to logging._handlers (Python 2.7, 3.2+)
+ if named_handlers_supported():
+ for name in handlers:
+ if name not in logging._handlers:
+ raise ValueError('No handler found with '
+ 'name %r' % name)
+ else:
+ try:
+ handler = logging._handlers[name]
+ handler_config = handlers[name]
+ level = handler_config.get('level', None)
+ if level:
+ handler.setLevel(_checkLevel(level))
+ except StandardError:
+ e = sys.exc_info()[1]
+ raise ValueError('Unable to configure handler '
+ '%r: %s' % (name, e))
+ loggers = config.get('loggers', EMPTY_DICT)
+ for name in loggers:
+ try:
+ self.configure_logger(name, loggers[name], True)
+ except StandardError:
+ e = sys.exc_info()[1]
+ raise ValueError('Unable to configure logger '
+ '%r: %s' % (name, e))
+ root = config.get('root', None)
+ if root:
+ try:
+ self.configure_root(root, True)
+ except StandardError:
+ e = sys.exc_info()[1]
+ raise ValueError('Unable to configure root '
+ 'logger: %s' % e)
+ else:
+ disable_existing = config.pop('disable_existing_loggers', True)
+
+ logging._handlers.clear()
+ del logging._handlerList[:]
+
+ # Do formatters first - they don't refer to anything else
+ formatters = config.get('formatters', EMPTY_DICT)
+ for name in formatters:
+ try:
+ formatters[name] = self.configure_formatter(
+ formatters[name])
+ except StandardError:
+ e = sys.exc_info()[1]
+ raise ValueError('Unable to configure '
+ 'formatter %r: %s' % (name, e))
+ # Next, do filters - they don't refer to anything else, either
+ filters = config.get('filters', EMPTY_DICT)
+ for name in filters:
+ try:
+ filters[name] = self.configure_filter(filters[name])
+ except StandardError:
+ e = sys.exc_info()[1]
+ raise ValueError('Unable to configure '
+ 'filter %r: %s' % (name, e))
+
+ # Next, do handlers - they refer to formatters and filters
+ # As handlers can refer to other handlers, sort the keys
+ # to allow a deterministic order of configuration
+ handlers = config.get('handlers', EMPTY_DICT)
+ for name in sorted(handlers):
+ try:
+ handler = self.configure_handler(handlers[name])
+ handler.name = name
+ handlers[name] = handler
+ except StandardError:
+ e = sys.exc_info()[1]
+ raise ValueError('Unable to configure handler '
+ '%r: %s' % (name, e))
+ # Next, do loggers - they refer to handlers and filters
+
+ #we don't want to lose the existing loggers,
+ #since other threads may have pointers to them.
+ #existing is set to contain all existing loggers,
+ #and as we go through the new configuration we
+ #remove any which are configured. At the end,
+ #what's left in existing is the set of loggers
+ #which were in the previous configuration but
+ #which are not in the new configuration.
+ root = logging.root
+ existing = sorted(root.manager.loggerDict.keys())
+ #The list needs to be sorted so that we can
+ #avoid disabling child loggers of explicitly
+ #named loggers. With a sorted list it is easier
+ #to find the child loggers.
+ #We'll keep the list of existing loggers
+ #which are children of named loggers here...
+ child_loggers = []
+ #now set up the new ones...
+ loggers = config.get('loggers', EMPTY_DICT)
+ for name in loggers:
+ if name in existing:
+ i = existing.index(name)
+ prefixed = name + "."
+ pflen = len(prefixed)
+ num_existing = len(existing)
+ i = i + 1 # look at the entry after name
+ while (i < num_existing) and\
+ (existing[i][:pflen] == prefixed):
+ child_loggers.append(existing[i])
+ i = i + 1
+ existing.remove(name)
+ try:
+ self.configure_logger(name, loggers[name])
+ except StandardError:
+ e = sys.exc_info()[1]
+ raise ValueError('Unable to configure logger '
+ '%r: %s' % (name, e))
+
+ #Disable any old loggers. There's no point deleting
+ #them as other threads may continue to hold references
+ #and by disabling them, you stop them doing any logging.
+ #However, don't disable children of named loggers, as that's
+ #probably not what was intended by the user.
+ for log in existing:
+ logger = root.manager.loggerDict[log]
+ if log in child_loggers:
+ logger.level = logging.NOTSET
+ logger.handlers = []
+ logger.propagate = True
+ elif disable_existing:
+ logger.disabled = True
+
+ # And finally, do the root logger
+ root = config.get('root', None)
+ if root:
+ try:
+ self.configure_root(root)
+ except StandardError:
+ e = sys.exc_info()[1]
+ raise ValueError('Unable to configure root '
+ 'logger: %s' % e)
+ finally:
+ logging._releaseLock()
+
+ def configure_formatter(self, config):
+ """Configure a formatter from a dictionary."""
+ if '()' in config:
+ factory = config['()'] # for use in exception handler
+ try:
+ result = self.configure_custom(config)
+ except TypeError:
+ te = sys.exc_info()[1]
+ if "'format'" not in str(te):
+ raise
+ #Name of parameter changed from fmt to format.
+ #Retry with old name.
+ #This is so that code can be used with older Python versions
+ #(e.g. by Django)
+ config['fmt'] = config.pop('format')
+ config['()'] = factory
+ result = self.configure_custom(config)
+ else:
+ fmt = config.get('format', None)
+ dfmt = config.get('datefmt', None)
+ result = logging.Formatter(fmt, dfmt)
+ return result
+
+ def configure_filter(self, config):
+ """Configure a filter from a dictionary."""
+ if '()' in config:
+ result = self.configure_custom(config)
+ else:
+ name = config.get('name', '')
+ result = logging.Filter(name)
+ return result
+
+ def add_filters(self, filterer, filters):
+ """Add filters to a filterer from a list of names."""
+ for f in filters:
+ try:
+ filterer.addFilter(self.config['filters'][f])
+ except StandardError:
+ e = sys.exc_info()[1]
+ raise ValueError('Unable to add filter %r: %s' % (f, e))
+
+ def configure_handler(self, config):
+ """Configure a handler from a dictionary."""
+ formatter = config.pop('formatter', None)
+ if formatter:
+ try:
+ formatter = self.config['formatters'][formatter]
+ except StandardError:
+ e = sys.exc_info()[1]
+ raise ValueError('Unable to set formatter '
+ '%r: %s' % (formatter, e))
+ level = config.pop('level', None)
+ filters = config.pop('filters', None)
+ if '()' in config:
+ c = config.pop('()')
+ if isinstance(c, basestring):
+ c = self.resolve(c)
+ factory = c
+ else:
+ klass = self.resolve(config.pop('class'))
+ #Special case for handler which refers to another handler
+ if issubclass(klass, logging.handlers.MemoryHandler) and\
+ 'target' in config:
+ try:
+ config['target'] = self.config['handlers'][config['target']]
+ except StandardError:
+ e = sys.exc_info()[1]
+ raise ValueError('Unable to set target handler '
+ '%r: %s' % (config['target'], e))
+ elif issubclass(klass, logging.handlers.SMTPHandler) and\
+ 'mailhost' in config:
+ config['mailhost'] = self.as_tuple(config['mailhost'])
+ elif issubclass(klass, logging.handlers.SysLogHandler) and\
+ 'address' in config:
+ config['address'] = self.as_tuple(config['address'])
+ factory = klass
+ kwargs = dict([(k, config[k]) for k in config if valid_ident(k)])
+ try:
+ result = factory(**kwargs)
+ except TypeError:
+ te = sys.exc_info()[1]
+ if "'stream'" not in str(te):
+ raise
+ #The argument name changed from strm to stream
+ #Retry with old name.
+ #This is so that code can be used with older Python versions
+ #(e.g. by Django)
+ kwargs['strm'] = kwargs.pop('stream')
+ result = factory(**kwargs)
+ if formatter:
+ result.setFormatter(formatter)
+ if level is not None:
+ result.setLevel(_checkLevel(level))
+ if filters:
+ self.add_filters(result, filters)
+ return result
+
+ def add_handlers(self, logger, handlers):
+ """Add handlers to a logger from a list of names."""
+ for h in handlers:
+ try:
+ logger.addHandler(self.config['handlers'][h])
+ except StandardError:
+ e = sys.exc_info()[1]
+ raise ValueError('Unable to add handler %r: %s' % (h, e))
+
+ def common_logger_config(self, logger, config, incremental=False):
+ """
+ Perform configuration which is common to root and non-root loggers.
+ """
+ level = config.get('level', None)
+ if level is not None:
+ logger.setLevel(_checkLevel(level))
+ if not incremental:
+ #Remove any existing handlers
+ for h in logger.handlers[:]:
+ logger.removeHandler(h)
+ handlers = config.get('handlers', None)
+ if handlers:
+ self.add_handlers(logger, handlers)
+ filters = config.get('filters', None)
+ if filters:
+ self.add_filters(logger, filters)
+
+ def configure_logger(self, name, config, incremental=False):
+ """Configure a non-root logger from a dictionary."""
+ logger = logging.getLogger(name)
+ self.common_logger_config(logger, config, incremental)
+ propagate = config.get('propagate', None)
+ if propagate is not None:
+ logger.propagate = propagate
+
+ def configure_root(self, config, incremental=False):
+ """Configure a root logger from a dictionary."""
+ root = logging.getLogger()
+ self.common_logger_config(root, config, incremental)
+
+dictConfigClass = DictConfigurator
+
+def dictConfig(config):
+ """Configure logging using a dictionary."""
+ dictConfigClass(config).configure()
diff --git a/lib/logutils/http.py b/lib/logutils/http.py
new file mode 100644
index 00000000..5d145c37
--- /dev/null
+++ b/lib/logutils/http.py
@@ -0,0 +1,90 @@
+#
+# Copyright (C) 2010-2013 Vinay Sajip. See LICENSE.txt for details.
+#
+import logging
+
+class HTTPHandler(logging.Handler):
+ """
+ A class which sends records to a Web server, using either GET or
+ POST semantics.
+
+ :param host: The Web server to connect to.
+ :param url: The URL to use for the connection.
+ :param method: The HTTP method to use. GET and POST are supported.
+ :param secure: set to True if HTTPS is to be used.
+ :param credentials: Set to a username/password tuple if desired. If
+ set, a Basic authentication header is sent. WARNING:
+ if using credentials, make sure `secure` is `True`
+ to avoid sending usernames and passwords in
+ cleartext over the wire.
+ """
+ def __init__(self, host, url, method="GET", secure=False, credentials=None):
+ """
+ Initialize an instance.
+ """
+ logging.Handler.__init__(self)
+ method = method.upper()
+ if method not in ["GET", "POST"]:
+ raise ValueError("method must be GET or POST")
+ self.host = host
+ self.url = url
+ self.method = method
+ self.secure = secure
+ self.credentials = credentials
+
+ def mapLogRecord(self, record):
+ """
+ Default implementation of mapping the log record into a dict
+ that is sent as the CGI data. Overwrite in your class.
+ Contributed by Franz Glasner.
+
+ :param record: The record to be mapped.
+ """
+ return record.__dict__
+
+ def emit(self, record):
+ """
+ Emit a record.
+
+ Send the record to the Web server as a percent-encoded dictionary
+
+ :param record: The record to be emitted.
+ """
+ try:
+ import http.client, urllib.parse
+ host = self.host
+ if self.secure:
+ h = http.client.HTTPSConnection(host)
+ else:
+ h = http.client.HTTPConnection(host)
+ url = self.url
+ data = urllib.parse.urlencode(self.mapLogRecord(record))
+ if self.method == "GET":
+ if (url.find('?') >= 0):
+ sep = '&'
+ else:
+ sep = '?'
+ url = url + "%c%s" % (sep, data)
+ h.putrequest(self.method, url)
+ # support multiple hosts on one IP address...
+ # need to strip optional :port from host, if present
+ i = host.find(":")
+ if i >= 0:
+ host = host[:i]
+ h.putheader("Host", host)
+ if self.method == "POST":
+ h.putheader("Content-type",
+ "application/x-www-form-urlencoded")
+ h.putheader("Content-length", str(len(data)))
+ if self.credentials:
+ import base64
+ s = ('u%s:%s' % self.credentials).encode('utf-8')
+ s = 'Basic ' + base64.b64encode(s).strip()
+ h.putheader('Authorization', s)
+ h.endheaders(data if self.method == "POST" else None)
+ h.getresponse() #can't do anything with the result
+ except (KeyboardInterrupt, SystemExit):
+ raise
+ except:
+ self.handleError(record)
+
diff --git a/lib/logutils/queue.py b/lib/logutils/queue.py
new file mode 100644
index 00000000..cced8c55
--- /dev/null
+++ b/lib/logutils/queue.py
@@ -0,0 +1,225 @@
+#
+# Copyright (C) 2010-2013 Vinay Sajip. See LICENSE.txt for details.
+#
+"""
+This module contains classes which help you work with queues. A typical
+application is when you want to log from performance-critical threads, but
+where the handlers you want to use are slow (for example,
+:class:`~logging.handlers.SMTPHandler`). In that case, you can create a queue,
+pass it to a :class:`QueueHandler` instance and use that instance with your
+loggers. Elsewhere, you can instantiate a :class:`QueueListener` with the same
+queue and some slow handlers, and call :meth:`~QueueListener.start` on it.
+This will start monitoring the queue on a separate thread and call all the
+configured handlers *on that thread*, so that your logging thread is not held
+up by the slow handlers.
+
+Note that as well as in-process queues, you can use these classes with queues
+from the :mod:`multiprocessing` module.
+
+**N.B.** This is part of the standard library since Python 3.2, so the
+version here is for use with earlier Python versions.
+"""
+import logging
+try:
+ import Queue as queue
+except ImportError:
+ import queue
+import threading
+
+class QueueHandler(logging.Handler):
+ """
+ This handler sends events to a queue. Typically, it would be used together
+ with a multiprocessing Queue to centralise logging to file in one process
+ (in a multi-process application), so as to avoid file write contention
+ between processes.
+
+ :param queue: The queue to send `LogRecords` to.
+ """
+
+ def __init__(self, queue):
+ """
+ Initialise an instance, using the passed queue.
+ """
+ logging.Handler.__init__(self)
+ self.queue = queue
+
+ def enqueue(self, record):
+ """
+ Enqueue a record.
+
+ The base implementation uses :meth:`~queue.Queue.put_nowait`. You may
+ want to override this method if you want to use blocking, timeouts or
+ custom queue implementations.
+
+ :param record: The record to enqueue.
+ """
+ self.queue.put_nowait(record)
+
+ def prepare(self, record):
+ """
+ Prepares a record for queuing. The object returned by this method is
+ enqueued.
+
+ The base implementation formats the record to merge the message
+ and arguments, and removes unpickleable items from the record
+ in-place.
+
+ You might want to override this method if you want to convert
+ the record to a dict or JSON string, or send a modified copy
+ of the record while leaving the original intact.
+
+ :param record: The record to prepare.
+ """
+ # The format operation gets traceback text into record.exc_text
+ # (if there's exception data), and also puts the message into
+ # record.message. We can then use this to replace the original
+ # msg + args, as these might be unpickleable. We also zap the
+ # exc_info attribute, as it's no longer needed and, if not None,
+ # will typically not be pickleable.
+ self.format(record)
+ record.msg = record.message
+ record.args = None
+ record.exc_info = None
+ return record
+
+ def emit(self, record):
+ """
+ Emit a record.
+
+ Writes the LogRecord to the queue, preparing it for pickling first.
+
+ :param record: The record to emit.
+ """
+ try:
+ self.enqueue(self.prepare(record))
+ except (KeyboardInterrupt, SystemExit):
+ raise
+ except:
+ self.handleError(record)
+
+class QueueListener(object):
+ """
+ This class implements an internal threaded listener which watches for
+ LogRecords being added to a queue, removes them and passes them to a
+ list of handlers for processing.
+
+ :param record: The queue to listen to.
+ :param handlers: The handlers to invoke on everything received from
+ the queue.
+ """
+ _sentinel = None
+
+ def __init__(self, queue, *handlers):
+ """
+ Initialise an instance with the specified queue and
+ handlers.
+ """
+ self.queue = queue
+ self.handlers = handlers
+ self._stop = threading.Event()
+ self._thread = None
+
+ def dequeue(self, block):
+ """
+ Dequeue a record and return it, optionally blocking.
+
+ The base implementation uses :meth:`~queue.Queue.get`. You may want to
+ override this method if you want to use timeouts or work with custom
+ queue implementations.
+
+ :param block: Whether to block if the queue is empty. If `False` and
+ the queue is empty, an :class:`~queue.Empty` exception
+ will be thrown.
+ """
+ return self.queue.get(block)
+
+ def start(self):
+ """
+ Start the listener.
+
+ This starts up a background thread to monitor the queue for
+ LogRecords to process.
+ """
+ self._thread = t = threading.Thread(target=self._monitor)
+ t.setDaemon(True)
+ t.start()
+
+ def prepare(self , record):
+ """
+ Prepare a record for handling.
+
+ This method just returns the passed-in record. You may want to
+ override this method if you need to do any custom marshalling or
+ manipulation of the record before passing it to the handlers.
+
+ :param record: The record to prepare.
+ """
+ return record
+
+ def handle(self, record):
+ """
+ Handle a record.
+
+ This just loops through the handlers offering them the record
+ to handle.
+
+ :param record: The record to handle.
+ """
+ record = self.prepare(record)
+ for handler in self.handlers:
+ handler.handle(record)
+
+ def _monitor(self):
+ """
+ Monitor the queue for records, and ask the handler
+ to deal with them.
+
+ This method runs on a separate, internal thread.
+ The thread will terminate if it sees a sentinel object in the queue.
+ """
+ q = self.queue
+ has_task_done = hasattr(q, 'task_done')
+ while not self._stop.isSet():
+ try:
+ record = self.dequeue(True)
+ if record is self._sentinel:
+ break
+ self.handle(record)
+ if has_task_done:
+ q.task_done()
+ except queue.Empty:
+ pass
+ # There might still be records in the queue.
+ while True:
+ try:
+ record = self.dequeue(False)
+ if record is self._sentinel:
+ break
+ self.handle(record)
+ if has_task_done:
+ q.task_done()
+ except queue.Empty:
+ break
+
+ def enqueue_sentinel(self):
+ """
+ Writes a sentinel to the queue to tell the listener to quit. This
+ implementation uses ``put_nowait()``. You may want to override this
+ method if you want to use timeouts or work with custom queue
+ implementations.
+ """
+ self.queue.put_nowait(self._sentinel)
+
+ def stop(self):
+ """
+ Stop the listener.
+
+ This asks the thread to terminate, and then waits for it to do so.
+ Note that if you don't call this before your application exits, there
+ may be some records still left on the queue, which won't be processed.
+ """
+ self._stop.set()
+ self.enqueue_sentinel()
+ self._thread.join()
+ self._thread = None
+
diff --git a/lib/logutils/redis.py b/lib/logutils/redis.py
new file mode 100644
index 00000000..0fea2d1e
--- /dev/null
+++ b/lib/logutils/redis.py
@@ -0,0 +1,75 @@
+#
+# Copyright (C) 2011-2013 Vinay Sajip. See LICENSE.txt for details.
+#
+"""
+This module contains classes which help you work with Redis queues.
+"""
+
+from logutils.queue import QueueHandler, QueueListener
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
+
+class RedisQueueHandler(QueueHandler):
+ """
+ A QueueHandler implementation which pushes pickled
+ records to a Redis queue using a specified key.
+
+ :param key: The key to use for the queue. Defaults to
+ "python.logging".
+ :param redis: If specified, this instance is used to
+ communicate with a Redis instance.
+ :param limit: If specified, the queue is restricted to
+ have only this many elements.
+ """
+ def __init__(self, key='python.logging', redis=None, limit=0):
+ if redis is None:
+ from redis import Redis
+ redis = Redis()
+ self.key = key
+ assert limit >= 0
+ self.limit = limit
+ QueueHandler.__init__(self, redis)
+
+ def enqueue(self, record):
+ s = pickle.dumps(vars(record))
+ self.queue.rpush(self.key, s)
+ if self.limit:
+ self.queue.ltrim(self.key, -self.limit, -1)
+
+class RedisQueueListener(QueueListener):
+ """
+ A QueueListener implementation which fetches pickled
+ records from a Redis queue using a specified key.
+
+ :param key: The key to use for the queue. Defaults to
+ "python.logging".
+ :param redis: If specified, this instance is used to
+ communicate with a Redis instance.
+ """
+ def __init__(self, *handlers, **kwargs):
+ redis = kwargs.get('redis')
+ if redis is None:
+ from redis import Redis
+ redis = Redis()
+ self.key = kwargs.get('key', 'python.logging')
+ QueueListener.__init__(self, redis, *handlers)
+
+ def dequeue(self, block):
+ """
+ Dequeue and return a record.
+ """
+ if block:
+ s = self.queue.blpop(self.key)[1]
+ else:
+ s = self.queue.lpop(self.key)
+ if not s:
+ record = None
+ else:
+ record = pickle.loads(s)
+ return record
+
+ def enqueue_sentinel(self):
+ self.queue.rpush(self.key, '')
+
diff --git a/lib/logutils/testing.py b/lib/logutils/testing.py
new file mode 100644
index 00000000..dfc8d212
--- /dev/null
+++ b/lib/logutils/testing.py
@@ -0,0 +1,156 @@
+#
+# Copyright (C) 2010-2013 Vinay Sajip. See LICENSE.txt for details.
+#
+import logging
+from logging.handlers import BufferingHandler
+
+class TestHandler(BufferingHandler):
+ """
+ This handler collects records in a buffer for later inspection by
+ your unit test code.
+
+ :param matcher: The :class:`~logutils.testing.Matcher` instance to
+ use for matching.
+ """
+ def __init__(self, matcher):
+ # BufferingHandler takes a "capacity" argument
+ # so as to know when to flush. As we're overriding
+ # shouldFlush anyway, we can set a capacity of zero.
+ # You can call flush() manually to clear out the
+ # buffer.
+ BufferingHandler.__init__(self, 0)
+ self.formatted = []
+ self.matcher = matcher
+
+ def shouldFlush(self):
+ """
+ Should the buffer be flushed?
+
+ This returns `False` - you'll need to flush manually, usually after
+ your unit test code checks the buffer contents against your
+ expectations.
+ """
+ return False
+
+ def emit(self, record):
+ """
+ Saves the `__dict__` of the record in the `buffer` attribute,
+ and the formatted records in the `formatted` attribute.
+
+ :param record: The record to emit.
+ """
+ self.formatted.append(self.format(record))
+ self.buffer.append(record.__dict__)
+
+ def flush(self):
+ """
+ Clears out the `buffer` and `formatted` attributes.
+ """
+ BufferingHandler.flush(self)
+ self.formatted = []
+
+ def matches(self, **kwargs):
+ """
+ Look for a saved dict whose keys/values match the supplied arguments.
+
+ Return `True` if found, else `False`.
+
+ :param kwargs: A set of keyword arguments whose names are LogRecord
+ attributes and whose values are what you want to
+ match in a stored LogRecord.
+ """
+ result = False
+ for d in self.buffer:
+ if self.matcher.matches(d, **kwargs):
+ result = True
+ break
+ #if not result:
+ # print('*** matcher failed completely on %d records' % len(self.buffer))
+ return result
+
+ def matchall(self, kwarglist):
+ """
+ Accept a list of keyword argument values and ensure that the handler's
+ buffer of stored records matches the list one-for-one.
+
+ Return `True` if exactly matched, else `False`.
+
+ :param kwarglist: A list of keyword-argument dictionaries, each of
+ which will be passed to :meth:`matches` with the
+ corresponding record from the buffer.
+ """
+ if self.count != len(kwarglist):
+ result = False
+ else:
+ result = True
+ for d, kwargs in zip(self.buffer, kwarglist):
+ if not self.matcher.matches(d, **kwargs):
+ result = False
+ break
+ return result
+
+ @property
+ def count(self):
+ """
+ The number of records in the buffer.
+ """
+ return len(self.buffer)
+
+class Matcher(object):
+ """
+ This utility class matches a stored dictionary of
+ :class:`logging.LogRecord` attributes with keyword arguments
+ passed to its :meth:`~logutils.testing.Matcher.matches` method.
+ """
+
+ _partial_matches = ('msg', 'message')
+ """
+ A list of :class:`logging.LogRecord` attribute names which
+ will be checked for partial matches. If not in this list,
+ an exact match will be attempted.
+ """
+
+ def matches(self, d, **kwargs):
+ """
+ Try to match a single dict with the supplied arguments.
+
+ Keys whose values are strings and which are in self._partial_matches
+ will be checked for partial (i.e. substring) matches. You can extend
+ this scheme to (for example) do regular expression matching, etc.
+
+ Return `True` if found, else `False`.
+
+ :param kwargs: A set of keyword arguments whose names are LogRecord
+ attributes and whose values are what you want to
+ match in a stored LogRecord.
+ """
+ result = True
+ for k in kwargs:
+ v = kwargs[k]
+ dv = d.get(k)
+ if not self.match_value(k, dv, v):
+ #print('*** matcher failed: %s, %r, %r' % (k, dv, v))
+ result = False
+ break
+ return result
+
+ def match_value(self, k, dv, v):
+ """
+ Try to match a single stored value (dv) with a supplied value (v).
+
+ Return `True` if found, else `False`.
+
+ :param k: The key value (LogRecord attribute name).
+ :param dv: The stored value to match against.
+ :param v: The value to compare with the stored value.
+ """
+ if type(v) != type(dv):
+ result = False
+ elif type(dv) is not str or k not in self._partial_matches:
+ result = (v == dv)
+ else:
+ result = dv.find(v) >= 0
+ #if not result:
+ # print('*** matcher failed on %s: %r vs. %r' % (k, dv, v))
+ return result
+