mutagen, musicbrainzngs, beets lib updates

lib/beets/dbcore/query.py

@@ -1,5 +1,6 @@
+# -*- coding: utf-8 -*-
 # This file is part of beets.
-# Copyright 2014, Adrian Sampson.
+# Copyright 2016, Adrian Sampson.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -14,10 +15,49 @@
 
 """The Query type hierarchy for DBCore.
 """
+from __future__ import division, absolute_import, print_function
+
 import re
-from operator import attrgetter
+from operator import mul
 from beets import util
 from datetime import datetime, timedelta
+import unicodedata
+from functools import reduce
+import six
+
+if not six.PY2:
+    buffer = memoryview  # sqlite won't accept memoryview in python 2
+
+
+class ParsingError(ValueError):
+    """Abstract class for any unparseable user-requested album/query
+    specification.
+    """
+
+
+class InvalidQueryError(ParsingError):
+    """Represent any kind of invalid query.
+
+    The query should be a unicode string or a list, which will be space-joined.
+    """
+    def __init__(self, query, explanation):
+        if isinstance(query, list):
+            query = " ".join(query)
+        message = u"'{0}': {1}".format(query, explanation)
+        super(InvalidQueryError, self).__init__(message)
+
+
+class InvalidQueryArgumentTypeError(ParsingError):
+    """Represent a query argument that could not be converted as expected.
+
+    It exists to be caught in upper stack levels so a meaningful (i.e. with the
+    query) InvalidQueryError can be raised.
+    """
+    def __init__(self, what, expected, detail=None):
+        message = u"'{0}' is not {1}".format(what, expected)
+        if detail:
+            message = u"{0}: {1}".format(message, detail)
+        super(InvalidQueryArgumentTypeError, self).__init__(message)
 
 
 class Query(object):
@@ -25,9 +65,8 @@ class Query(object):
     """
     def clause(self):
         """Generate an SQLite expression implementing the query.
-        Return a clause string, a sequence of substitution values for
-        the clause, and a Query object representing the "remainder"
-        Returns (clause, subvals) where clause is a valid sqlite
+
+        Return (clause, subvals) where clause is a valid sqlite
         WHERE clause implementing the query and subvals is a list of
         items to be substituted for ?s in the clause.
         """
@@ -39,6 +78,15 @@ class Query(object):
         """
         raise NotImplementedError
 
+    def __repr__(self):
+        return "{0.__class__.__name__}()".format(self)
+
+    def __eq__(self, other):
+        return type(self) == type(other)
+
+    def __hash__(self):
+        return 0
+
 
 class FieldQuery(Query):
     """An abstract query that searches in a specific field for a
@@ -72,6 +120,17 @@ class FieldQuery(Query):
     def match(self, item):
         return self.value_match(self.pattern, item.get(self.field))
 
+    def __repr__(self):
+        return ("{0.__class__.__name__}({0.field!r}, {0.pattern!r}, "
+                "{0.fast})".format(self))
+
+    def __eq__(self, other):
+        return super(FieldQuery, self).__eq__(other) and \
+            self.field == other.field and self.pattern == other.pattern
+
+    def __hash__(self):
+        return hash((self.field, hash(self.pattern)))
+
 
 class MatchQuery(FieldQuery):
     """A query that looks for exact matches in an item field."""
@@ -86,19 +145,21 @@ class MatchQuery(FieldQuery):
 class NoneQuery(FieldQuery):
 
     def __init__(self, field, fast=True):
-        self.field = field
-        self.fast = fast
+        super(NoneQuery, self).__init__(field, None, fast)
 
     def col_clause(self):
         return self.field + " IS NULL", ()
 
     @classmethod
-    def match(self, item):
+    def match(cls, item):
         try:
-            return item[self.field] is None
+            return item[cls.field] is None
         except KeyError:
             return True
 
+    def __repr__(self):
+        return "{0.__class__.__name__}({0.field!r}, {0.fast})".format(self)
+
 
 class StringFieldQuery(FieldQuery):
     """A FieldQuery that converts values to strings before matching
@@ -139,15 +200,31 @@ class SubstringQuery(StringFieldQuery):
 class RegexpQuery(StringFieldQuery):
     """A query that matches a regular expression in a specific item
     field.
+
+    Raises InvalidQueryError when the pattern is not a valid regular
+    expression.
     """
+    def __init__(self, field, pattern, fast=True):
+        super(RegexpQuery, self).__init__(field, pattern, fast)
+        pattern = self._normalize(pattern)
+        try:
+            self.pattern = re.compile(self.pattern)
+        except re.error as exc:
+            # Invalid regular expression.
+            raise InvalidQueryArgumentTypeError(pattern,
+                                                u"a regular expression",
+                                                format(exc))
+
+    @staticmethod
+    def _normalize(s):
+        """Normalize a Unicode string's representation (used on both
+        patterns and matched values).
+        """
+        return unicodedata.normalize('NFC', s)
+
     @classmethod
     def string_match(cls, pattern, value):
-        try:
-            res = re.search(pattern, value)
-        except re.error:
-            # Invalid regular expression.
-            return False
-        return res is not None
+        return pattern.search(cls._normalize(value)) is not None
 
 
 class BooleanQuery(MatchQuery):
@@ -156,28 +233,26 @@ class BooleanQuery(MatchQuery):
     """
     def __init__(self, field, pattern, fast=True):
         super(BooleanQuery, self).__init__(field, pattern, fast)
-        if isinstance(pattern, basestring):
+        if isinstance(pattern, six.string_types):
             self.pattern = util.str2bool(pattern)
         self.pattern = int(self.pattern)
 
 
 class BytesQuery(MatchQuery):
     """Match a raw bytes field (i.e., a path). This is a necessary hack
-    to work around the `sqlite3` module's desire to treat `str` and
+    to work around the `sqlite3` module's desire to treat `bytes` and
     `unicode` equivalently in Python 2. Always use this query instead of
     `MatchQuery` when matching on BLOB values.
     """
     def __init__(self, field, pattern):
         super(BytesQuery, self).__init__(field, pattern)
 
-        # Use a buffer representation of the pattern for SQLite
+        # Use a buffer/memoryview representation of the pattern for SQLite
         # matching. This instructs SQLite to treat the blob as binary
         # rather than encoded Unicode.
-        if isinstance(self.pattern, basestring):
-            # Implicitly coerce Unicode strings to their bytes
-            # equivalents.
-            if isinstance(self.pattern, unicode):
-                self.pattern = self.pattern.encode('utf8')
+        if isinstance(self.pattern, (six.text_type, bytes)):
+            if isinstance(self.pattern, six.text_type):
+                self.pattern = self.pattern.encode('utf-8')
             self.buf_pattern = buffer(self.pattern)
         elif isinstance(self.pattern, buffer):
             self.buf_pattern = self.pattern
@@ -191,19 +266,26 @@ class NumericQuery(FieldQuery):
     """Matches numeric fields. A syntax using Ruby-style range ellipses
     (``..``) lets users specify one- or two-sided ranges. For example,
     ``year:2001..`` finds music released since the turn of the century.
+
+    Raises InvalidQueryError when the pattern does not represent an int or
+    a float.
     """
     def _convert(self, s):
-        """Convert a string to a numeric type (float or int). If the
-        string cannot be converted, return None.
+        """Convert a string to a numeric type (float or int).
+
+        Return None if `s` is empty.
+        Raise an InvalidQueryError if the string cannot be converted.
         """
         # This is really just a bit of fun premature optimization.
+        if not s:
+            return None
         try:
             return int(s)
         except ValueError:
             try:
                 return float(s)
             except ValueError:
-                return None
+                raise InvalidQueryArgumentTypeError(s, u"an int or a float")
 
     def __init__(self, field, pattern, fast=True):
         super(NumericQuery, self).__init__(field, pattern, fast)
@@ -224,7 +306,7 @@ class NumericQuery(FieldQuery):
         if self.field not in item:
             return False
         value = item[self.field]
-        if isinstance(value, basestring):
+        if isinstance(value, six.string_types):
             value = self._convert(value)
 
         if self.point is not None:
@@ -248,7 +330,7 @@ class NumericQuery(FieldQuery):
             elif self.rangemax is not None:
                 return u'{0} <= ?'.format(self.field), (self.rangemax,)
             else:
-                return '1', ()
+                return u'1', ()
 
 
 class CollectionQuery(Query):
@@ -273,7 +355,7 @@ class CollectionQuery(Query):
         return item in self.subqueries
 
     def clause_with_joiner(self, joiner):
-        """Returns a clause created by joining together the clauses of
+        """Return a clause created by joining together the clauses of
         all subqueries with the string joiner (padded by spaces).
         """
         clause_parts = []
@@ -288,6 +370,19 @@ class CollectionQuery(Query):
         clause = (' ' + joiner + ' ').join(clause_parts)
         return clause, subvals
 
+    def __repr__(self):
+        return "{0.__class__.__name__}({0.subqueries!r})".format(self)
+
+    def __eq__(self, other):
+        return super(CollectionQuery, self).__eq__(other) and \
+            self.subqueries == other.subqueries
+
+    def __hash__(self):
+        """Since subqueries are mutable, this object should not be hashable.
+        However and for conveniences purposes, it can be hashed.
+        """
+        return reduce(mul, map(hash, self.subqueries), 1)
+
 
 class AnyFieldQuery(CollectionQuery):
     """A query that matches if a given FieldQuery subclass matches in
@@ -313,6 +408,17 @@ class AnyFieldQuery(CollectionQuery):
                 return True
         return False
 
+    def __repr__(self):
+        return ("{0.__class__.__name__}({0.pattern!r}, {0.fields!r}, "
+                "{0.query_class.__name__})".format(self))
+
+    def __eq__(self, other):
+        return super(AnyFieldQuery, self).__eq__(other) and \
+            self.query_class == other.query_class
+
+    def __hash__(self):
+        return hash((self.pattern, tuple(self.fields), self.query_class))
+
 
 class MutableCollectionQuery(CollectionQuery):
     """A collection query whose subqueries may be modified after the
@@ -343,6 +449,36 @@ class OrQuery(MutableCollectionQuery):
         return any([q.match(item) for q in self.subqueries])
 
 
+class NotQuery(Query):
+    """A query that matches the negation of its `subquery`, as a shorcut for
+    performing `not(subquery)` without using regular expressions.
+    """
+    def __init__(self, subquery):
+        self.subquery = subquery
+
+    def clause(self):
+        clause, subvals = self.subquery.clause()
+        if clause:
+            return 'not ({0})'.format(clause), subvals
+        else:
+            # If there is no clause, there is nothing to negate. All the logic
+            # is handled by match() for slow queries.
+            return clause, subvals
+
+    def match(self, item):
+        return not self.subquery.match(item)
+
+    def __repr__(self):
+        return "{0.__class__.__name__}({0.subquery!r})".format(self)
+
+    def __eq__(self, other):
+        return super(NotQuery, self).__eq__(other) and \
+            self.subquery == other.subquery
+
+    def __hash__(self):
+        return hash(('not', hash(self.subquery)))
+
+
 class TrueQuery(Query):
     """A query that always matches."""
     def clause(self):
@@ -367,13 +503,13 @@ def _to_epoch_time(date):
     """Convert a `datetime` object to an integer number of seconds since
     the (local) Unix epoch.
     """
-    epoch = datetime.fromtimestamp(0)
-    delta = date - epoch
-    try:
+    if hasattr(date, 'timestamp'):
+        # The `timestamp` method exists on Python 3.3+.
+        return int(date.timestamp())
+    else:
+        epoch = datetime.fromtimestamp(0)
+        delta = date - epoch
         return int(delta.total_seconds())
-    except AttributeError:
-        # datetime.timedelta.total_seconds() is not available on Python 2.6
-        return delta.seconds + delta.days * 24 * 3600
 
 
 def _parse_periods(pattern):
@@ -405,7 +541,7 @@ class Period(object):
         precision (a string, one of "year", "month", or "day").
         """
         if precision not in Period.precisions:
-            raise ValueError('Invalid precision ' + str(precision))
+            raise ValueError(u'Invalid precision {0}'.format(precision))
         self.date = date
         self.precision = precision
 
@@ -445,7 +581,7 @@ class Period(object):
         elif 'day' == precision:
             return date + timedelta(days=1)
         else:
-            raise ValueError('unhandled precision ' + str(precision))
+            raise ValueError(u'unhandled precision {0}'.format(precision))
 
 
 class DateInterval(object):
@@ -457,7 +593,7 @@ class DateInterval(object):
 
     def __init__(self, start, end):
         if start is not None and end is not None and not start < end:
-            raise ValueError("start date {0} is not before end date {1}"
+            raise ValueError(u"start date {0} is not before end date {1}"
                              .format(start, end))
         self.start = start
         self.end = end
@@ -478,7 +614,7 @@ class DateInterval(object):
         return True
 
     def __str__(self):
-        return'[{0}, {1})'.format(self.start, self.end)
+        return '[{0}, {1})'.format(self.start, self.end)
 
 
 class DateQuery(FieldQuery):
@@ -496,6 +632,8 @@ class DateQuery(FieldQuery):
         self.interval = DateInterval.from_periods(start, end)
 
     def match(self, item):
+        if self.field not in item:
+            return False
         timestamp = float(item[self.field])
         date = datetime.utcfromtimestamp(timestamp)
         return self.interval.contains(date)
@@ -523,6 +661,33 @@ class DateQuery(FieldQuery):
         return clause, subvals
 
 
+class DurationQuery(NumericQuery):
+    """NumericQuery that allow human-friendly (M:SS) time interval formats.
+
+    Converts the range(s) to a float value, and delegates on NumericQuery.
+
+    Raises InvalidQueryError when the pattern does not represent an int, float
+    or M:SS time interval.
+    """
+    def _convert(self, s):
+        """Convert a M:SS or numeric string to a float.
+
+        Return None if `s` is empty.
+        Raise an InvalidQueryError if the string cannot be converted.
+        """
+        if not s:
+            return None
+        try:
+            return util.raw_seconds_short(s)
+        except ValueError:
+            try:
+                return float(s)
+            except ValueError:
+                raise InvalidQueryArgumentTypeError(
+                    s,
+                    u"a M:SS string or a float")
+
+
 # Sorting.
 
 class Sort(object):
@@ -547,6 +712,12 @@ class Sort(object):
         """
         return False
 
+    def __hash__(self):
+        return 0
+
+    def __eq__(self, other):
+        return type(self) == type(other)
+
 
 class MultipleSort(Sort):
     """Sort that encapsulates multiple sub-sorts.
@@ -606,38 +777,67 @@ class MultipleSort(Sort):
         return items
 
     def __repr__(self):
-        return u'MultipleSort({0})'.format(repr(self.sorts))
+        return 'MultipleSort({!r})'.format(self.sorts)
+
+    def __hash__(self):
+        return hash(tuple(self.sorts))
+
+    def __eq__(self, other):
+        return super(MultipleSort, self).__eq__(other) and \
+            self.sorts == other.sorts
 
 
 class FieldSort(Sort):
     """An abstract sort criterion that orders by a specific field (of
     any kind).
     """
-    def __init__(self, field, ascending=True):
+    def __init__(self, field, ascending=True, case_insensitive=True):
         self.field = field
         self.ascending = ascending
+        self.case_insensitive = case_insensitive
 
     def sort(self, objs):
         # TODO: Conversion and null-detection here. In Python 3,
         # comparisons with None fail. We should also support flexible
         # attributes with different types without falling over.
-        return sorted(objs, key=attrgetter(self.field),
-                      reverse=not self.ascending)
+
+        def key(item):
+            field_val = item.get(self.field, '')
+            if self.case_insensitive and isinstance(field_val, six.text_type):
+                field_val = field_val.lower()
+            return field_val
+
+        return sorted(objs, key=key, reverse=not self.ascending)
 
     def __repr__(self):
-        return u'<{0}: {1}{2}>'.format(
+        return '<{0}: {1}{2}>'.format(
             type(self).__name__,
             self.field,
             '+' if self.ascending else '-',
         )
 
+    def __hash__(self):
+        return hash((self.field, self.ascending))
+
+    def __eq__(self, other):
+        return super(FieldSort, self).__eq__(other) and \
+            self.field == other.field and \
+            self.ascending == other.ascending
+
 
 class FixedFieldSort(FieldSort):
     """Sort object to sort on a fixed field.
     """
     def order_clause(self):
         order = "ASC" if self.ascending else "DESC"
-        return "{0} {1}".format(self.field, order)
+        if self.case_insensitive:
+            field = '(CASE ' \
+                    'WHEN TYPEOF({0})="text" THEN LOWER({0}) ' \
+                    'WHEN TYPEOF({0})="blob" THEN LOWER({0}) ' \
+                    'ELSE {0} END)'.format(self.field)
+        else:
+            field = self.field
+        return "{0} {1}".format(field, order)
 
 
 class SlowFieldSort(FieldSort):
@@ -650,5 +850,17 @@ class SlowFieldSort(FieldSort):
 
 class NullSort(Sort):
     """No sorting. Leave results unsorted."""
-    def sort(items):
+    def sort(self, items):
         return items
+
+    def __nonzero__(self):
+        return self.__bool__()
+
+    def __bool__(self):
+        return False
+
+    def __eq__(self, other):
+        return type(self) == type(other) or other is None
+
+    def __hash__(self):
+        return 0