Merge with rembo10/develop

beets/__init__.py

@@ -0,0 +1,26 @@
+# This file is part of beets.
+# Copyright 2013, Adrian Sampson.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+
+# This particular version has been slightly modified to work with headphones
+# https://github.com/rembo10/headphones
+
+__version__ = '1.3.4'
+__author__ = 'Adrian Sampson <adrian@radbox.org>'
+
+import beets.library
+from beets.util import confit
+
+Library = beets.library.Library
+
+config = confit.LazyConfig('beets', __name__)

beets/autotag/__init__.py

@@ -0,0 +1,247 @@
+# This file is part of beets.
+# Copyright 2013, Adrian Sampson.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+
+"""Facilities for automatically determining files' correct metadata.
+"""
+import os
+import logging
+import re
+
+from beets import library, mediafile, config
+from beets.util import sorted_walk, ancestry, displayable_path
+
+# Parts of external interface.
+from .hooks import AlbumInfo, TrackInfo, AlbumMatch, TrackMatch
+from .match import tag_item, tag_album
+from .match import recommendation
+
+# Global logger.
+log = logging.getLogger('beets')
+
+# Constants for directory walker.
+MULTIDISC_MARKERS = (r'dis[ck]', r'cd')
+MULTIDISC_PAT_FMT = r'^(.*%s[\W_]*)\d'
+
+
+# Additional utilities for the main interface.
+
+def albums_in_dir(path):
+    """Recursively searches the given directory and returns an iterable
+    of (paths, items) where paths is a list of directories and items is
+    a list of Items that is probably an album. Specifically, any folder
+    containing any media files is an album.
+    """
+    collapse_pat = collapse_paths = collapse_items = None
+
+    for root, dirs, files in sorted_walk(path,
+                                         ignore=config['ignore'].as_str_seq(),
+                                         logger=log):
+        # Get a list of items in the directory.
+        items = []
+        for filename in files:
+            try:
+                i = library.Item.from_path(os.path.join(root, filename))
+            except mediafile.FileTypeError:
+                pass
+            except mediafile.UnreadableFileError:
+                log.warn(u'unreadable file: {0}'.format(
+                    displayable_path(filename))
+                )
+            else:
+                items.append(i)
+
+        # If we're currently collapsing the constituent directories in a
+        # multi-disc album, check whether we should continue collapsing
+        # and add the current directory. If so, just add the directory
+        # and move on to the next directory. If not, stop collapsing.
+        if collapse_paths:
+            if (not collapse_pat and collapse_paths[0] in ancestry(root)) or \
+                    (collapse_pat and
+                     collapse_pat.match(os.path.basename(root))):
+                # Still collapsing.
+                collapse_paths.append(root)
+                collapse_items += items
+                continue
+            else:
+                # Collapse finished. Yield the collapsed directory and
+                # proceed to process the current one.
+                if collapse_items:
+                    yield collapse_paths, collapse_items
+                collapse_pat = collapse_paths = collapse_items = None
+
+        # Check whether this directory looks like the *first* directory
+        # in a multi-disc sequence. There are two indicators: the file
+        # is named like part of a multi-disc sequence (e.g., "Title Disc
+        # 1") or it contains no items but only directories that are
+        # named in this way.
+        start_collapsing = False
+        for marker in MULTIDISC_MARKERS:
+            marker_pat = re.compile(MULTIDISC_PAT_FMT % marker, re.I)
+            match = marker_pat.match(os.path.basename(root))
+
+            # Is this directory the root of a nested multi-disc album?
+            if dirs and not items:
+                # Check whether all subdirectories have the same prefix.
+                start_collapsing = True
+                subdir_pat = None
+                for subdir in dirs:
+                    # The first directory dictates the pattern for
+                    # the remaining directories.
+                    if not subdir_pat:
+                        match = marker_pat.match(subdir)
+                        if match:
+                            subdir_pat = re.compile(r'^%s\d' %
+                                re.escape(match.group(1)), re.I)
+                        else:
+                            start_collapsing = False
+                            break
+
+                    # Subsequent directories must match the pattern.
+                    elif not subdir_pat.match(subdir):
+                        start_collapsing = False
+                        break
+
+                # If all subdirectories match, don't check other
+                # markers.
+                if start_collapsing:
+                    break
+
+            # Is this directory the first in a flattened multi-disc album?
+            elif match:
+                start_collapsing = True
+                # Set the current pattern to match directories with the same
+                # prefix as this one, followed by a digit.
+                collapse_pat = re.compile(r'^%s\d' %
+                    re.escape(match.group(1)), re.I)
+                break
+
+        # If either of the above heuristics indicated that this is the
+        # beginning of a multi-disc album, initialize the collapsed
+        # directory and item lists and check the next directory.
+        if start_collapsing:
+            # Start collapsing; continue to the next iteration.
+            collapse_paths = [root]
+            collapse_items = items
+            continue
+
+        # If it's nonempty, yield it.
+        if items:
+            yield [root], items
+
+    # Clear out any unfinished collapse.
+    if collapse_paths and collapse_items:
+        yield collapse_paths, collapse_items
+
+def apply_item_metadata(item, track_info):
+    """Set an item's metadata from its matched TrackInfo object.
+    """
+    item.artist = track_info.artist
+    item.artist_sort = track_info.artist_sort
+    item.artist_credit = track_info.artist_credit
+    item.title = track_info.title
+    item.mb_trackid = track_info.track_id
+    if track_info.artist_id:
+        item.mb_artistid = track_info.artist_id
+    # At the moment, the other metadata is left intact (including album
+    # and track number). Perhaps these should be emptied?
+
+def apply_metadata(album_info, mapping):
+    """Set the items' metadata to match an AlbumInfo object using a
+    mapping from Items to TrackInfo objects.
+    """
+    for item, track_info in mapping.iteritems():
+        # Album, artist, track count.
+        if track_info.artist:
+            item.artist = track_info.artist
+        else:
+            item.artist = album_info.artist
+        item.albumartist = album_info.artist
+        item.album = album_info.album
+
+        # Artist sort and credit names.
+        item.artist_sort = track_info.artist_sort or album_info.artist_sort
+        item.artist_credit = track_info.artist_credit or \
+                             album_info.artist_credit
+        item.albumartist_sort = album_info.artist_sort
+        item.albumartist_credit = album_info.artist_credit
+
+        # Release date.
+        for prefix in '', 'original_':
+            if config['original_date'] and not prefix:
+                # Ignore specific release date.
+                continue
+
+            for suffix in 'year', 'month', 'day':
+                key = prefix + suffix
+                value = getattr(album_info, key) or 0
+
+                # If we don't even have a year, apply nothing.
+                if suffix == 'year' and not value:
+                    break
+
+                # Otherwise, set the fetched value (or 0 for the month
+                # and day if not available).
+                item[key] = value
+
+                # If we're using original release date for both fields,
+                # also set item.year = info.original_year, etc.
+                if config['original_date']:
+                    item[suffix] = value
+
+        # Title.
+        item.title = track_info.title
+
+        if config['per_disc_numbering']:
+            item.track = track_info.medium_index or track_info.index
+            item.tracktotal = track_info.medium_total or len(album_info.tracks)
+        else:
+            item.track = track_info.index
+            item.tracktotal = len(album_info.tracks)
+
+        # Disc and disc count.
+        item.disc = track_info.medium
+        item.disctotal = album_info.mediums
+
+        # MusicBrainz IDs.
+        item.mb_trackid = track_info.track_id
+        item.mb_albumid = album_info.album_id
+        if track_info.artist_id:
+            item.mb_artistid = track_info.artist_id
+        else:
+            item.mb_artistid = album_info.artist_id
+        item.mb_albumartistid = album_info.artist_id
+        item.mb_releasegroupid = album_info.releasegroup_id
+
+        # Compilation flag.
+        item.comp = album_info.va
+
+        # Miscellaneous metadata.
+        for field in ('albumtype',
+                      'label',
+                      'asin',
+                      'catalognum',
+                      'script',
+                      'language',
+                      'country',
+                      'albumstatus',
+                      'media',
+                      'albumdisambig'):
+            value = getattr(album_info, field)
+            if value is not None:
+                item[field] = value
+        if track_info.disctitle is not None:
+            item.disctitle = track_info.disctitle
+        
+        # Headphones seal of approval
+        item.comments = 'tagged by headphones/beets'

beets/autotag/hooks.py

@@ -0,0 +1,545 @@
+# This file is part of beets.
+# Copyright 2013, Adrian Sampson.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+
+"""Glue between metadata sources and the matching logic."""
+import logging
+from collections import namedtuple
+import re
+
+from beets import plugins
+from beets import config
+from beets.autotag import mb
+from beets.util import levenshtein
+from lib.unidecode import unidecode
+
+log = logging.getLogger('beets')
+
+
+# Classes used to represent candidate options.
+
+class AlbumInfo(object):
+    """Describes a canonical release that may be used to match a release
+    in the library. Consists of these data members:
+
+    - ``album``: the release title
+    - ``album_id``: MusicBrainz ID; UUID fragment only
+    - ``artist``: name of the release's primary artist
+    - ``artist_id``
+    - ``tracks``: list of TrackInfo objects making up the release
+    - ``asin``: Amazon ASIN
+    - ``albumtype``: string describing the kind of release
+    - ``va``: boolean: whether the release has "various artists"
+    - ``year``: release year
+    - ``month``: release month
+    - ``day``: release day
+    - ``label``: music label responsible for the release
+    - ``mediums``: the number of discs in this release
+    - ``artist_sort``: name of the release's artist for sorting
+    - ``releasegroup_id``: MBID for the album's release group
+    - ``catalognum``: the label's catalog number for the release
+    - ``script``: character set used for metadata
+    - ``language``: human language of the metadata
+    - ``country``: the release country
+    - ``albumstatus``: MusicBrainz release status (Official, etc.)
+    - ``media``: delivery mechanism (Vinyl, etc.)
+    - ``albumdisambig``: MusicBrainz release disambiguation comment
+    - ``artist_credit``: Release-specific artist name
+    - ``data_source``: The original data source (MusicBrainz, Discogs, etc.)
+    - ``data_url``: The data source release URL.
+
+    The fields up through ``tracks`` are required. The others are
+    optional and may be None.
+    """
+    def __init__(self, album, album_id, artist, artist_id, tracks, asin=None,
+                 albumtype=None, va=False, year=None, month=None, day=None,
+                 label=None, mediums=None, artist_sort=None,
+                 releasegroup_id=None, catalognum=None, script=None,
+                 language=None, country=None, albumstatus=None, media=None,
+                 albumdisambig=None, artist_credit=None, original_year=None,
+                 original_month=None, original_day=None, data_source=None,
+                 data_url=None):
+        self.album = album
+        self.album_id = album_id
+        self.artist = artist
+        self.artist_id = artist_id
+        self.tracks = tracks
+        self.asin = asin
+        self.albumtype = albumtype
+        self.va = va
+        self.year = year
+        self.month = month
+        self.day = day
+        self.label = label
+        self.mediums = mediums
+        self.artist_sort = artist_sort
+        self.releasegroup_id = releasegroup_id
+        self.catalognum = catalognum
+        self.script = script
+        self.language = language
+        self.country = country
+        self.albumstatus = albumstatus
+        self.media = media
+        self.albumdisambig = albumdisambig
+        self.artist_credit = artist_credit
+        self.original_year = original_year
+        self.original_month = original_month
+        self.original_day = original_day
+        self.data_source = data_source
+        self.data_url = data_url
+
+    # Work around a bug in python-musicbrainz-ngs that causes some
+    # strings to be bytes rather than Unicode.
+    # https://github.com/alastair/python-musicbrainz-ngs/issues/85
+    def decode(self, codec='utf8'):
+        """Ensure that all string attributes on this object, and the
+        constituent `TrackInfo` objects, are decoded to Unicode.
+        """
+        for fld in ['album', 'artist', 'albumtype', 'label', 'artist_sort',
+                    'catalognum', 'script', 'language', 'country',
+                    'albumstatus', 'albumdisambig', 'artist_credit', 'media']:
+            value = getattr(self, fld)
+            if isinstance(value, str):
+                setattr(self, fld, value.decode(codec, 'ignore'))
+
+        if self.tracks:
+            for track in self.tracks:
+                track.decode(codec)
+
+class TrackInfo(object):
+    """Describes a canonical track present on a release. Appears as part
+    of an AlbumInfo's ``tracks`` list. Consists of these data members:
+
+    - ``title``: name of the track
+    - ``track_id``: MusicBrainz ID; UUID fragment only
+    - ``artist``: individual track artist name
+    - ``artist_id``
+    - ``length``: float: duration of the track in seconds
+    - ``index``: position on the entire release
+    - ``medium``: the disc number this track appears on in the album
+    - ``medium_index``: the track's position on the disc
+    - ``medium_total``: the number of tracks on the item's disc
+    - ``artist_sort``: name of the track artist for sorting
+    - ``disctitle``: name of the individual medium (subtitle)
+    - ``artist_credit``: Recording-specific artist name
+
+    Only ``title`` and ``track_id`` are required. The rest of the fields
+    may be None. The indices ``index``, ``medium``, and ``medium_index``
+    are all 1-based.
+    """
+    def __init__(self, title, track_id, artist=None, artist_id=None,
+                 length=None, index=None, medium=None, medium_index=None,
+                 medium_total=None, artist_sort=None, disctitle=None,
+                 artist_credit=None, data_source=None, data_url=None):
+        self.title = title
+        self.track_id = track_id
+        self.artist = artist
+        self.artist_id = artist_id
+        self.length = length
+        self.index = index
+        self.medium = medium
+        self.medium_index = medium_index
+        self.medium_total = medium_total
+        self.artist_sort = artist_sort
+        self.disctitle = disctitle
+        self.artist_credit = artist_credit
+        self.data_source = data_source
+        self.data_url = data_url
+
+    # As above, work around a bug in python-musicbrainz-ngs.
+    def decode(self, codec='utf8'):
+        """Ensure that all string attributes on this object are decoded
+        to Unicode.
+        """
+        for fld in ['title', 'artist', 'medium', 'artist_sort', 'disctitle',
+                    'artist_credit']:
+            value = getattr(self, fld)
+            if isinstance(value, str):
+                setattr(self, fld, value.decode(codec, 'ignore'))
+
+
+# Candidate distance scoring.
+
+# Parameters for string distance function.
+# Words that can be moved to the end of a string using a comma.
+SD_END_WORDS = ['the', 'a', 'an']
+# Reduced weights for certain portions of the string.
+SD_PATTERNS = [
+    (r'^the ', 0.1),
+    (r'[\[\(]?(ep|single)[\]\)]?', 0.0),
+    (r'[\[\(]?(featuring|feat|ft)[\. :].+', 0.1),
+    (r'\(.*?\)', 0.3),
+    (r'\[.*?\]', 0.3),
+    (r'(, )?(pt\.|part) .+', 0.2),
+]
+# Replacements to use before testing distance.
+SD_REPLACE = [
+    (r'&', 'and'),
+]
+
+def _string_dist_basic(str1, str2):
+    """Basic edit distance between two strings, ignoring
+    non-alphanumeric characters and case. Comparisons are based on a
+    transliteration/lowering to ASCII characters. Normalized by string
+    length.
+    """
+    str1 = unidecode(str1)
+    str2 = unidecode(str2)
+    str1 = re.sub(r'[^a-z0-9]', '', str1.lower())
+    str2 = re.sub(r'[^a-z0-9]', '', str2.lower())
+    if not str1 and not str2:
+        return 0.0
+    return levenshtein(str1, str2) / float(max(len(str1), len(str2)))
+
+def string_dist(str1, str2):
+    """Gives an "intuitive" edit distance between two strings. This is
+    an edit distance, normalized by the string length, with a number of
+    tweaks that reflect intuition about text.
+    """
+    if str1 == None and str2 == None: return 0.0
+    if str1 == None or str2 == None:  return 1.0
+
+    str1 = str1.lower()
+    str2 = str2.lower()
+
+    # Don't penalize strings that move certain words to the end. For
+    # example, "the something" should be considered equal to
+    # "something, the".
+    for word in SD_END_WORDS:
+        if str1.endswith(', %s' % word):
+            str1 = '%s %s' % (word, str1[:-len(word)-2])
+        if str2.endswith(', %s' % word):
+            str2 = '%s %s' % (word, str2[:-len(word)-2])
+
+    # Perform a couple of basic normalizing substitutions.
+    for pat, repl in SD_REPLACE:
+        str1 = re.sub(pat, repl, str1)
+        str2 = re.sub(pat, repl, str2)
+
+    # Change the weight for certain string portions matched by a set
+    # of regular expressions. We gradually change the strings and build
+    # up penalties associated with parts of the string that were
+    # deleted.
+    base_dist = _string_dist_basic(str1, str2)
+    penalty = 0.0
+    for pat, weight in SD_PATTERNS:
+        # Get strings that drop the pattern.
+        case_str1 = re.sub(pat, '', str1)
+        case_str2 = re.sub(pat, '', str2)
+
+        if case_str1 != str1 or case_str2 != str2:
+            # If the pattern was present (i.e., it is deleted in the
+            # the current case), recalculate the distances for the
+            # modified strings.
+            case_dist = _string_dist_basic(case_str1, case_str2)
+            case_delta = max(0.0, base_dist - case_dist)
+            if case_delta == 0.0:
+                continue
+
+            # Shift our baseline strings down (to avoid rematching the
+            # same part of the string) and add a scaled distance
+            # amount to the penalties.
+            str1 = case_str1
+            str2 = case_str2
+            base_dist = case_dist
+            penalty += weight * case_delta
+
+    return base_dist + penalty
+
+class Distance(object):
+    """Keeps track of multiple distance penalties. Provides a single
+    weighted distance for all penalties as well as a weighted distance
+    for each individual penalty.
+    """
+    def __init__(self):
+        self._penalties = {}
+
+        weights_view = config['match']['distance_weights']
+        self._weights = {}
+        for key in weights_view.keys():
+            self._weights[key] = weights_view[key].as_number()
+
+
+    # Access the components and their aggregates.
+
+    @property
+    def distance(self):
+        """Return a weighted and normalized distance across all
+        penalties.
+        """
+        dist_max = self.max_distance
+        if dist_max:
+            return self.raw_distance / self.max_distance
+        return 0.0
+
+    @property
+    def max_distance(self):
+        """Return the maximum distance penalty (normalization factor).
+        """
+        dist_max = 0.0
+        for key, penalty in self._penalties.iteritems():
+            dist_max += len(penalty) * self._weights[key]
+        return dist_max
+
+    @property
+    def raw_distance(self):
+        """Return the raw (denormalized) distance.
+        """
+        dist_raw = 0.0
+        for key, penalty in self._penalties.iteritems():
+            dist_raw += sum(penalty) * self._weights[key]
+        return dist_raw
+
+    def items(self):
+        """Return a list of (key, dist) pairs, with `dist` being the
+        weighted distance, sorted from highest to lowest. Does not
+        include penalties with a zero value.
+        """
+        list_ = []
+        for key in self._penalties:
+            dist = self[key]
+            if dist:
+                list_.append((key, dist))
+        # Convert distance into a negative float we can sort items in
+        # ascending order (for keys, when the penalty is equal) and
+        # still get the items with the biggest distance first.
+        return sorted(list_, key=lambda (key, dist): (0-dist, key))
+
+
+    # Behave like a float.
+
+    def __cmp__(self, other):
+        return cmp(self.distance, other)
+
+    def __float__(self):
+        return self.distance
+    def __sub__(self, other):
+        return self.distance - other
+
+    def __rsub__(self, other):
+        return other - self.distance
+
+
+    # Behave like a dict.
+
+    def __getitem__(self, key):
+        """Returns the weighted distance for a named penalty.
+        """
+        dist = sum(self._penalties[key]) * self._weights[key]
+        dist_max = self.max_distance
+        if dist_max:
+            return dist / dist_max
+        return 0.0
+
+    def __iter__(self):
+        return iter(self.items())
+
+    def __len__(self):
+        return len(self.items())
+
+    def keys(self):
+        return [key for key, _ in self.items()]
+
+    def update(self, dist):
+        """Adds all the distance penalties from `dist`.
+        """
+        if not isinstance(dist, Distance):
+            raise ValueError(
+                    '`dist` must be a Distance object. It is: %r' % dist)
+        for key, penalties in dist._penalties.iteritems():
+            self._penalties.setdefault(key, []).extend(penalties)
+
+
+    # Adding components.
+
+    def _eq(self, value1, value2):
+        """Returns True if `value1` is equal to `value2`. `value1` may
+        be a compiled regular expression, in which case it will be
+        matched against `value2`.
+        """
+        if isinstance(value1, re._pattern_type):
+            return bool(value1.match(value2))
+        return value1 == value2
+
+    def add(self, key, dist):
+        """Adds a distance penalty. `key` must correspond with a
+        configured weight setting. `dist` must be a float between 0.0
+        and 1.0, and will be added to any existing distance penalties
+        for the same key.
+        """
+        if not 0.0 <= dist <= 1.0:
+            raise ValueError(
+                    '`dist` must be between 0.0 and 1.0. It is: %r' % dist)
+        self._penalties.setdefault(key, []).append(dist)
+
+    def add_equality(self, key, value, options):
+        """Adds a distance penalty of 1.0 if `value` doesn't match any
+        of the values in `options`. If an option is a compiled regular
+        expression, it will be considered equal if it matches against
+        `value`.
+        """
+        if not isinstance(options, (list, tuple)):
+            options = [options]
+        for opt in options:
+            if self._eq(opt, value):
+                dist = 0.0
+                break
+        else:
+            dist = 1.0
+        self.add(key, dist)
+
+    def add_expr(self, key, expr):
+        """Adds a distance penalty of 1.0 if `expr` evaluates to True,
+        or 0.0.
+        """
+        if expr:
+            self.add(key, 1.0)
+        else:
+            self.add(key, 0.0)
+
+    def add_number(self, key, number1, number2):
+        """Adds a distance penalty of 1.0 for each number of difference
+        between `number1` and `number2`, or 0.0 when there is no
+        difference. Use this when there is no upper limit on the
+        difference between the two numbers.
+        """
+        diff = abs(number1 - number2)
+        if diff:
+            for i in range(diff):
+                self.add(key, 1.0)
+        else:
+            self.add(key, 0.0)
+
+    def add_priority(self, key, value, options):
+        """Adds a distance penalty that corresponds to the position at
+        which `value` appears in `options`. A distance penalty of 0.0
+        for the first option, or 1.0 if there is no matching option. If
+        an option is a compiled regular expression, it will be
+        considered equal if it matches against `value`.
+        """
+        if not isinstance(options, (list, tuple)):
+            options = [options]
+        unit = 1.0 / (len(options) or 1)
+        for i, opt in enumerate(options):
+            if self._eq(opt, value):
+                dist = i * unit
+                break
+        else:
+            dist = 1.0
+        self.add(key, dist)
+
+    def add_ratio(self, key, number1, number2):
+        """Adds a distance penalty for `number1` as a ratio of `number2`.
+        `number1` is bound at 0 and `number2`.
+        """
+        number = float(max(min(number1, number2), 0))
+        if number2:
+            dist = number / number2
+        else:
+            dist = 0.0
+        self.add(key, dist)
+
+    def add_string(self, key, str1, str2):
+        """Adds a distance penalty based on the edit distance between
+        `str1` and `str2`.
+        """
+        dist = string_dist(str1, str2)
+        self.add(key, dist)
+
+
+# Structures that compose all the information for a candidate match.
+
+AlbumMatch = namedtuple('AlbumMatch', ['distance', 'info', 'mapping',
+                                       'extra_items', 'extra_tracks'])
+
+TrackMatch = namedtuple('TrackMatch', ['distance', 'info'])
+
+
+# Aggregation of sources.
+
+def album_for_mbid(release_id):
+    """Get an AlbumInfo object for a MusicBrainz release ID. Return None
+    if the ID is not found.
+    """
+    try:
+        return mb.album_for_id(release_id)
+    except mb.MusicBrainzAPIError as exc:
+        exc.log(log)
+
+def track_for_mbid(recording_id):
+    """Get a TrackInfo object for a MusicBrainz recording ID. Return None
+    if the ID is not found.
+    """
+    try:
+        return mb.track_for_id(recording_id)
+    except mb.MusicBrainzAPIError as exc:
+        exc.log(log)
+
+def albums_for_id(album_id):
+    """Get a list of albums for an ID."""
+    candidates = [album_for_mbid(album_id)]
+    candidates.extend(plugins.album_for_id(album_id))
+    return filter(None, candidates)
+
+def tracks_for_id(track_id):
+    """Get a list of tracks for an ID."""
+    candidates = [track_for_mbid(track_id)]
+    candidates.extend(plugins.track_for_id(track_id))
+    return filter(None, candidates)
+
+def album_candidates(items, artist, album, va_likely):
+    """Search for album matches. ``items`` is a list of Item objects
+    that make up the album. ``artist`` and ``album`` are the respective
+    names (strings), which may be derived from the item list or may be
+    entered by the user. ``va_likely`` is a boolean indicating whether
+    the album is likely to be a "various artists" release.
+    """
+    out = []
+
+    # Base candidates if we have album and artist to match.
+    if artist and album:
+        try:
+            out.extend(mb.match_album(artist, album, len(items)))
+        except mb.MusicBrainzAPIError as exc:
+            exc.log(log)
+
+    # Also add VA matches from MusicBrainz where appropriate.
+    if va_likely and album:
+        try:
+            out.extend(mb.match_album(None, album, len(items)))
+        except mb.MusicBrainzAPIError as exc:
+            exc.log(log)
+
+    # Candidates from plugins.
+    out.extend(plugins.candidates(items, artist, album, va_likely))
+
+    return out
+
+def item_candidates(item, artist, title):
+    """Search for item matches. ``item`` is the Item to be matched.
+    ``artist`` and ``title`` are strings and either reflect the item or
+    are specified by the user.
+    """
+    out = []
+
+    # MusicBrainz candidates.
+    if artist and title:
+        try:
+            out.extend(mb.match_track(artist, title))
+        except mb.MusicBrainzAPIError as exc:
+            exc.log(log)
+
+    # Plugin candidates.
+    out.extend(plugins.item_candidates(item, artist, title))
+
+    return out

beets/autotag/match.py

@@ -0,0 +1,456 @@
+# This file is part of beets.
+# Copyright 2013, Adrian Sampson.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+
+"""Matches existing metadata with canonical information to identify
+releases and tracks.
+"""
+from __future__ import division
+
+import datetime
+import logging
+import re
+from lib.munkres import Munkres
+
+from beets import plugins
+from beets import config
+from beets.util import plurality
+from beets.util.enumeration import enum
+from beets.autotag import hooks
+
+# Recommendation enumeration.
+recommendation = enum('none', 'low', 'medium', 'strong', name='recommendation')
+
+# Artist signals that indicate "various artists". These are used at the
+# album level to determine whether a given release is likely a VA
+# release and also on the track level to to remove the penalty for
+# differing artists.
+VA_ARTISTS = (u'', u'various artists', u'various', u'va', u'unknown')
+
+# Global logger.
+log = logging.getLogger('beets')
+
+
+# Primary matching functionality.
+
+def current_metadata(items):
+    """Extract the likely current metadata for an album given a list of its
+    items. Return two dictionaries:
+     - The most common value for each field.
+     - Whether each field's value was unanimous (values are booleans).
+    """
+    assert items  # Must be nonempty.
+
+    likelies = {}
+    consensus = {}
+    fields = ['artist', 'album', 'albumartist', 'year', 'disctotal',
+              'mb_albumid', 'label', 'catalognum', 'country', 'media',
+              'albumdisambig']
+    for key in fields:
+        values = [getattr(item, key) for item in items if item]
+        likelies[key], freq = plurality(values)
+        consensus[key] = (freq == len(values))
+
+    # If there's an album artist consensus, use this for the artist.
+    if consensus['albumartist'] and likelies['albumartist']:
+        likelies['artist'] = likelies['albumartist']
+
+    return likelies, consensus
+
+def assign_items(items, tracks):
+    """Given a list of Items and a list of TrackInfo objects, find the
+    best mapping between them. Returns a mapping from Items to TrackInfo
+    objects, a set of extra Items, and a set of extra TrackInfo
+    objects. These "extra" objects occur when there is an unequal number
+    of objects of the two types.
+    """
+    # Construct the cost matrix.
+    costs = []
+    for item in items:
+        row = []
+        for i, track in enumerate(tracks):
+            row.append(track_distance(item, track))
+        costs.append(row)
+
+    # Find a minimum-cost bipartite matching.
+    matching = Munkres().compute(costs)
+
+    # Produce the output matching.
+    mapping = dict((items[i], tracks[j]) for (i, j) in matching)
+    extra_items = list(set(items) - set(mapping.keys()))
+    extra_items.sort(key=lambda i: (i.disc, i.track, i.title))
+    extra_tracks = list(set(tracks) - set(mapping.values()))
+    extra_tracks.sort(key=lambda t: (t.index, t.title))
+    return mapping, extra_items, extra_tracks
+
+def track_index_changed(item, track_info):
+    """Returns True if the item and track info index is different. Tolerates
+    per disc and per release numbering.
+    """
+    return item.track not in (track_info.medium_index, track_info.index)
+
+def track_distance(item, track_info, incl_artist=False):
+    """Determines the significance of a track metadata change. Returns a
+    Distance object. `incl_artist` indicates that a distance component should
+    be included for the track artist (i.e., for various-artist releases).
+    """
+    dist = hooks.Distance()
+
+    # Length.
+    if track_info.length:
+        diff = abs(item.length - track_info.length) - \
+               config['match']['track_length_grace'].as_number()
+        dist.add_ratio('track_length', diff,
+                       config['match']['track_length_max'].as_number())
+
+    # Title.
+    dist.add_string('track_title', item.title, track_info.title)
+
+    # Artist. Only check if there is actually an artist in the track data.
+    if incl_artist and track_info.artist and \
+            item.artist.lower() not in VA_ARTISTS:
+        dist.add_string('track_artist', item.artist, track_info.artist)
+
+    # Track index.
+    if track_info.index and item.track:
+        dist.add_expr('track_index', track_index_changed(item, track_info))
+
+    # Track ID.
+    if item.mb_trackid:
+        dist.add_expr('track_id', item.mb_trackid != track_info.track_id)
+
+    # Plugins.
+    dist.update(plugins.track_distance(item, track_info))
+
+    return dist
+
+def distance(items, album_info, mapping):
+    """Determines how "significant" an album metadata change would be.
+    Returns a Distance object. `album_info` is an AlbumInfo object
+    reflecting the album to be compared. `items` is a sequence of all
+    Item objects that will be matched (order is not important).
+    `mapping` is a dictionary mapping Items to TrackInfo objects; the
+    keys are a subset of `items` and the values are a subset of
+    `album_info.tracks`.
+    """
+    likelies, _ = current_metadata(items)
+
+    dist = hooks.Distance()
+
+    # Artist, if not various.
+    if not album_info.va:
+        dist.add_string('artist', likelies['artist'], album_info.artist)
+
+    # Album.
+    dist.add_string('album', likelies['album'], album_info.album)
+
+    # Current or preferred media.
+    if album_info.media:
+        # Preferred media options.
+        patterns = config['match']['preferred']['media'].as_str_seq()
+        options = [re.compile(r'(\d+x)?(%s)' % pat, re.I) for pat in patterns]
+        if options:
+            dist.add_priority('media', album_info.media, options)
+        # Current media.
+        elif likelies['media']:
+            dist.add_equality('media', album_info.media, likelies['media'])
+
+    # Mediums.
+    if likelies['disctotal'] and album_info.mediums:
+        dist.add_number('mediums', likelies['disctotal'], album_info.mediums)
+
+    # Prefer earliest release.
+    if album_info.year and config['match']['preferred']['original_year']:
+        # Assume 1889 (earliest first gramophone discs) if we don't know the
+        # original year.
+        original = album_info.original_year or 1889
+        diff = abs(album_info.year - original)
+        diff_max = abs(datetime.date.today().year - original)
+        dist.add_ratio('year', diff, diff_max)
+    # Year.
+    elif likelies['year'] and album_info.year:
+        if likelies['year'] in (album_info.year, album_info.original_year):
+            # No penalty for matching release or original year.
+            dist.add('year', 0.0)
+        elif album_info.original_year:
+            # Prefer matchest closest to the release year.
+            diff = abs(likelies['year'] - album_info.year)
+            diff_max = abs(datetime.date.today().year -
+                           album_info.original_year)
+            dist.add_ratio('year', diff, diff_max)
+        else:
+            # Full penalty when there is no original year.
+            dist.add('year', 1.0)
+
+    # Preferred countries.
+    patterns = config['match']['preferred']['countries'].as_str_seq()
+    options = [re.compile(pat, re.I) for pat in patterns]
+    if album_info.country and options:
+        dist.add_priority('country', album_info.country, options)
+    # Country.
+    elif likelies['country'] and album_info.country:
+        dist.add_string('country', likelies['country'], album_info.country)
+
+    # Label.
+    if likelies['label'] and album_info.label:
+        dist.add_string('label', likelies['label'], album_info.label)
+
+    # Catalog number.
+    if likelies['catalognum'] and album_info.catalognum:
+        dist.add_string('catalognum', likelies['catalognum'],
+                        album_info.catalognum)
+
+    # Disambiguation.
+    if likelies['albumdisambig'] and album_info.albumdisambig:
+        dist.add_string('albumdisambig', likelies['albumdisambig'],
+                        album_info.albumdisambig)
+
+    # Album ID.
+    if likelies['mb_albumid']:
+        dist.add_equality('album_id', likelies['mb_albumid'],
+                          album_info.album_id)
+
+    # Tracks.
+    dist.tracks = {}
+    for item, track in mapping.iteritems():
+        dist.tracks[track] = track_distance(item, track, album_info.va)
+        dist.add('tracks', dist.tracks[track].distance)
+
+    # Missing tracks.
+    for i in range(len(album_info.tracks) - len(mapping)):
+        dist.add('missing_tracks', 1.0)
+
+    # Unmatched tracks.
+    for i in range(len(items) - len(mapping)):
+        dist.add('unmatched_tracks', 1.0)
+
+    # Plugins.
+    dist.update(plugins.album_distance(items, album_info, mapping))
+
+    return dist
+
+def match_by_id(items):
+    """If the items are tagged with a MusicBrainz album ID, returns an
+    AlbumInfo object for the corresponding album. Otherwise, returns
+    None.
+    """
+    # Is there a consensus on the MB album ID?
+    albumids = [item.mb_albumid for item in items if item.mb_albumid]
+    if not albumids:
+        log.debug('No album IDs found.')
+        return None
+
+    # If all album IDs are equal, look up the album.
+    if bool(reduce(lambda x,y: x if x==y else (), albumids)):
+        albumid = albumids[0]
+        log.debug('Searching for discovered album ID: ' + albumid)
+        return hooks.album_for_mbid(albumid)
+    else:
+        log.debug('No album ID consensus.')
+
+def _recommendation(results):
+    """Given a sorted list of AlbumMatch or TrackMatch objects, return a
+    recommendation based on the results' distances.
+
+    If the recommendation is higher than the configured maximum for
+    an applied penalty, the recommendation will be downgraded to the
+    configured maximum for that penalty.
+    """
+    if not results:
+        # No candidates: no recommendation.
+        return recommendation.none
+
+    # Basic distance thresholding.
+    min_dist = results[0].distance
+    if min_dist < config['match']['strong_rec_thresh'].as_number():
+        # Strong recommendation level.
+        rec = recommendation.strong
+    elif min_dist <= config['match']['medium_rec_thresh'].as_number():
+        # Medium recommendation level.
+        rec = recommendation.medium
+    elif len(results) == 1:
+        # Only a single candidate.
+        rec = recommendation.low
+    elif results[1].distance - min_dist >= \
+            config['match']['rec_gap_thresh'].as_number():
+        # Gap between first two candidates is large.
+        rec = recommendation.low
+    else:
+        # No conclusion. Return immediately. Can't be downgraded any further.
+        return recommendation.none
+
+    # Downgrade to the max rec if it is lower than the current rec for an
+    # applied penalty.
+    keys = set(min_dist.keys())
+    if isinstance(results[0], hooks.AlbumMatch):
+        for track_dist in min_dist.tracks.values():
+            keys.update(track_dist.keys())
+    max_rec_view = config['match']['max_rec']
+    for key in keys:
+        if key in max_rec_view.keys():
+            max_rec = max_rec_view[key].as_choice({
+                'strong': recommendation.strong,
+                'medium': recommendation.medium,
+                'low': recommendation.low,
+                'none': recommendation.none,
+            })
+            rec = min(rec, max_rec)
+
+    return rec
+
+def _add_candidate(items, results, info):
+    """Given a candidate AlbumInfo object, attempt to add the candidate
+    to the output dictionary of AlbumMatch objects. This involves
+    checking the track count, ordering the items, checking for
+    duplicates, and calculating the distance.
+    """
+    log.debug('Candidate: %s - %s' % (info.artist, info.album))
+
+    # Don't duplicate.
+    if info.album_id in results:
+        log.debug('Duplicate.')
+        return
+
+    # Find mapping between the items and the track info.
+    mapping, extra_items, extra_tracks = assign_items(items, info.tracks)
+
+    # Get the change distance.
+    dist = distance(items, info, mapping)
+
+    # Skip matches with ignored penalties.
+    penalties = [key for _, key in dist]
+    for penalty in config['match']['ignored'].as_str_seq():
+        if penalty in penalties:
+            log.debug('Ignored. Penalty: %s' % penalty)
+            return
+
+    log.debug('Success. Distance: %f' % dist)
+    results[info.album_id] = hooks.AlbumMatch(dist, info, mapping,
+                                              extra_items, extra_tracks)
+
+def tag_album(items, search_artist=None, search_album=None,
+              search_id=None):
+    """Bundles together the functionality used to infer tags for a
+    set of items comprised by an album. Returns everything relevant:
+        - The current artist.
+        - The current album.
+        - A list of AlbumMatch objects. The candidates are sorted by
+          distance (i.e., best match first).
+        - A recommendation.
+    If search_artist and search_album or search_id are provided, then
+    they are used as search terms in place of the current metadata.
+    """
+    # Get current metadata.
+    likelies, consensus = current_metadata(items)
+    cur_artist = likelies['artist']
+    cur_album = likelies['album']
+    log.debug('Tagging %s - %s' % (cur_artist, cur_album))
+
+    # The output result (distance, AlbumInfo) tuples (keyed by MB album
+    # ID).
+    candidates = {}
+
+    # Search by explicit ID.
+    if search_id is not None:
+        log.debug('Searching for album ID: ' + search_id)
+        search_cands = hooks.albums_for_id(search_id)
+
+    # Use existing metadata or text search.
+    else:
+        # Try search based on current ID.
+        id_info = match_by_id(items)
+        if id_info:
+            _add_candidate(items, candidates, id_info)
+            rec = _recommendation(candidates.values())
+            log.debug('Album ID match recommendation is ' + str(rec))
+            if candidates and not config['import']['timid']:
+                # If we have a very good MBID match, return immediately.
+                # Otherwise, this match will compete against metadata-based
+                # matches.
+                if rec == recommendation.strong:
+                    log.debug('ID match.')
+                    return cur_artist, cur_album, candidates.values(), rec
+
+        # Search terms.
+        if not (search_artist and search_album):
+            # No explicit search terms -- use current metadata.
+            search_artist, search_album = cur_artist, cur_album
+        log.debug(u'Search terms: %s - %s' % (search_artist, search_album))
+
+        # Is this album likely to be a "various artist" release?
+        va_likely = ((not consensus['artist']) or
+                    (search_artist.lower() in VA_ARTISTS) or
+                    any(item.comp for item in items))
+        log.debug(u'Album might be VA: %s' % str(va_likely))
+
+        # Get the results from the data sources.
+        search_cands = hooks.album_candidates(items, search_artist,
+                                              search_album, va_likely)
+
+    log.debug(u'Evaluating %i candidates.' % len(search_cands))
+    for info in search_cands:
+        _add_candidate(items, candidates, info)
+
+    # Sort and get the recommendation.
+    candidates = sorted(candidates.itervalues())
+    rec = _recommendation(candidates)
+    return cur_artist, cur_album, candidates, rec
+
+def tag_item(item, search_artist=None, search_title=None,
+             search_id=None):
+    """Attempts to find metadata for a single track. Returns a
+    `(candidates, recommendation)` pair where `candidates` is a list of
+    TrackMatch objects. `search_artist` and `search_title` may be used
+    to override the current metadata for the purposes of the MusicBrainz
+    title; likewise `search_id`.
+    """
+    # Holds candidates found so far: keys are MBIDs; values are
+    # (distance, TrackInfo) pairs.
+    candidates = {}
+
+    # First, try matching by MusicBrainz ID.
+    trackid = search_id or item.mb_trackid
+    if trackid:
+        log.debug('Searching for track ID: ' + trackid)
+        for track_info in hooks.tracks_for_id(trackid):
+            dist = track_distance(item, track_info, incl_artist=True)
+            candidates[track_info.track_id] = \
+                    hooks.TrackMatch(dist, track_info)
+            # If this is a good match, then don't keep searching.
+            rec = _recommendation(candidates.values())
+            if rec == recommendation.strong and not config['import']['timid']:
+                log.debug('Track ID match.')
+                return candidates.values(), rec
+
+    # If we're searching by ID, don't proceed.
+    if search_id is not None:
+        if candidates:
+            return candidates.values(), rec
+        else:
+            return [], recommendation.none
+
+    # Search terms.
+    if not (search_artist and search_title):
+        search_artist, search_title = item.artist, item.title
+    log.debug(u'Item search terms: %s - %s' % (search_artist, search_title))
+
+    # Get and evaluate candidate metadata.
+    for track_info in hooks.item_candidates(item, search_artist, search_title):
+        dist = track_distance(item, track_info, incl_artist=True)
+        candidates[track_info.track_id] = hooks.TrackMatch(dist, track_info)
+
+    # Sort by distance and return with recommendation.
+    log.debug('Found %i candidates.' % len(candidates))
+    candidates = sorted(candidates.itervalues())
+    rec = _recommendation(candidates)
+    return candidates, rec

beets/autotag/mb.py

@@ -1,5 +1,5 @@
 # This file is part of beets.
-# Copyright 2012, Adrian Sampson.
+# Copyright 2013, Adrian Sampson.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -15,17 +15,21 @@
 """Searches for albums in the MusicBrainz database.
 """
 import logging
-import lib.musicbrainzngs as musicbrainzngs
+import musicbrainzngs
+import re
 import traceback
+from urlparse import urljoin
 
-import lib.beets.autotag.hooks
-import lib.beets
-from lib.beets import util
+import beets.autotag.hooks
+import beets
+from beets import util
+from beets import config
 
 SEARCH_LIMIT = 5
 VARIOUS_ARTISTS_ID = '89ad4ac3-39f7-470e-963a-56509c546377'
+BASE_URL = 'http://musicbrainz.org/'
 
-musicbrainzngs.set_useragent('beets', lib.beets.__version__,
+musicbrainzngs.set_useragent('beets', beets.__version__,
                              'http://beets.radbox.org/')
 
 class MusicBrainzAPIError(util.HumanReadableException):
@@ -44,18 +48,45 @@ class MusicBrainzAPIError(util.HumanReadableException):
 log = logging.getLogger('beets')
 
 RELEASE_INCLUDES = ['artists', 'media', 'recordings', 'release-groups',
-                    'labels', 'artist-credits']
-TRACK_INCLUDES = ['artists']
+                    'labels', 'artist-credits', 'aliases']
+TRACK_INCLUDES = ['artists', 'aliases']
 
-# python-musicbrainz-ngs search functions: tolerate different API versions.
-if hasattr(musicbrainzngs, 'release_search'):
-    # Old API names.
-    _mb_release_search = musicbrainzngs.release_search
-    _mb_recording_search = musicbrainzngs.recording_search
-else:
-    # New API names.
-    _mb_release_search = musicbrainzngs.search_releases
-    _mb_recording_search = musicbrainzngs.search_recordings
+def track_url(trackid):
+    return urljoin(BASE_URL, 'recording/' + trackid)
+
+def album_url(albumid):
+    return urljoin(BASE_URL, 'release/' + albumid)
+
+def configure():
+    """Set up the python-musicbrainz-ngs module according to settings
+    from the beets configuration. This should be called at startup.
+    """
+    musicbrainzngs.set_hostname(config['musicbrainz']['host'].get(unicode))
+    musicbrainzngs.set_rate_limit(
+        config['musicbrainz']['ratelimit_interval'].as_number(),
+        config['musicbrainz']['ratelimit'].get(int),
+    )
+
+def _preferred_alias(aliases):
+    """Given an list of alias structures for an artist credit, select
+    and return the user's preferred alias alias or None if no matching
+    alias is found.
+    """
+    if not aliases:
+        return
+
+    # Only consider aliases that have locales set.
+    aliases = [a for a in aliases if 'locale' in a]
+
+    # Search configured locales in order.
+    for locale in config['import']['languages'].as_str_seq():
+        # Find matching primary aliases for this locale.
+        matches = [a for a in aliases if a['locale'] == locale and 'primary' in a]
+        # Skip to the next locale if we have no matches
+        if not matches:
+            continue
+
+        return matches[0]
 
 def _flatten_artist_credit(credit):
     """Given a list representing an ``artist-credit`` block, flatten the
@@ -73,12 +104,19 @@ def _flatten_artist_credit(credit):
             artist_sort_parts.append(el)
 
         else:
+            alias = _preferred_alias(el['artist'].get('alias-list', ()))
+
             # An artist.
-            cur_artist_name = el['artist']['name']
+            if alias:
+                cur_artist_name = alias['alias']
+            else:
+                cur_artist_name = el['artist']['name']
             artist_parts.append(cur_artist_name)
 
             # Artist sort name.
-            if 'sort-name' in el['artist']:
+            if alias:
+                artist_sort_parts.append(alias['sort-name'])
+            elif 'sort-name' in el['artist']:
                 artist_sort_parts.append(el['artist']['sort-name'])
             else:
                 artist_sort_parts.append(cur_artist_name)
@@ -95,19 +133,24 @@ def _flatten_artist_credit(credit):
         ''.join(artist_credit_parts),
     )
 
-def track_info(recording, index=None, medium=None, medium_index=None):
+def track_info(recording, index=None, medium=None, medium_index=None,
+               medium_total=None):
     """Translates a MusicBrainz recording result dictionary into a beets
     ``TrackInfo`` object. Three parameters are optional and are used
     only for tracks that appear on releases (non-singletons): ``index``,
     the overall track number; ``medium``, the disc number;
-    ``medium_index``, the track's index on its medium. Each number is a
-    1-based index.
+    ``medium_index``, the track's index on its medium; ``medium_total``,
+    the number of tracks on the medium. Each number is a 1-based index.
     """
-    info = lib.beets.autotag.hooks.TrackInfo(recording['title'],
-                                         recording['id'],
-                                         index=index,
-                                         medium=medium,
-                                         medium_index=medium_index)
+    info = beets.autotag.hooks.TrackInfo(
+        recording['title'],
+        recording['id'],
+        index=index,
+        medium=medium,
+        medium_index=medium_index,
+        medium_total=medium_total,
+        data_url=track_url(recording['id']),
+    )
 
     if recording.get('artist-credit'):
         # Get the artist names.
@@ -119,19 +162,29 @@ def track_info(recording, index=None, medium=None, medium_index=None):
         info.artist_id = artist['id']
 
     if recording.get('length'):
-        info.length = int(recording['length'])/(1000.0)
+        info.length = int(recording['length']) / (1000.0)
 
+    info.decode()
     return info
 
-def _set_date_str(info, date_str):
+def _set_date_str(info, date_str, original=False):
     """Given a (possibly partial) YYYY-MM-DD string and an AlbumInfo
-    object, set the object's release date fields appropriately.
+    object, set the object's release date fields appropriately. If
+    `original`, then set the original_year, etc., fields.
     """
     if date_str:
         date_parts = date_str.split('-')
         for key in ('year', 'month', 'day'):
             if date_parts:
-                setattr(info, key, int(date_parts.pop(0)))
+                date_part = date_parts.pop(0)
+                try:
+                    date_num = int(date_part)
+                except ValueError:
+                    continue
+
+                if original:
+                    key = 'original_' + key
+                setattr(info, key, date_num)
 
 def album_info(release):
     """Takes a MusicBrainz release result dictionary and returns a beets
@@ -147,18 +200,31 @@ def album_info(release):
     for medium in release['medium-list']:
         disctitle = medium.get('title')
         for track in medium['track-list']:
+            # Basic information from the recording.
             index += 1
-            ti = track_info(track['recording'],
-                            index,
-                            int(medium['position']),
-                            int(track['position']))
-            if track.get('title'):
-                # Track title may be distinct from underling recording
-                # title.
-                ti.title = track['title']
+            ti = track_info(
+                track['recording'],
+                index,
+                int(medium['position']),
+                int(track['position']),
+                len(medium['track-list']),
+            )
             ti.disctitle = disctitle
+
+            # Prefer track data, where present, over recording data.
+            if track.get('title'):
+                ti.title = track['title']
+            if track.get('artist-credit'):
+                # Get the artist names.
+                ti.artist, ti.artist_sort, ti.artist_credit = \
+                    _flatten_artist_credit(track['artist-credit'])
+                ti.artist_id = track['artist-credit'][0]['artist']['id']
+            if track.get('length'):
+                ti.length = int(track['length']) / (1000.0)
+
             track_infos.append(ti)
-    info = lib.beets.autotag.hooks.AlbumInfo(
+
+    info = beets.autotag.hooks.AlbumInfo(
         release['title'],
         release['id'],
         artist_name,
@@ -167,27 +233,37 @@ def album_info(release):
         mediums=len(release['medium-list']),
         artist_sort=artist_sort_name,
         artist_credit=artist_credit_name,
+        data_source='MusicBrainz',
+        data_url=album_url(release['id']),
     )
     info.va = info.artist_id == VARIOUS_ARTISTS_ID
     info.asin = release.get('asin')
     info.releasegroup_id = release['release-group']['id']
-    info.albumdisambig = release['release-group'].get('disambiguation')
     info.country = release.get('country')
     info.albumstatus = release.get('status')
 
+    # Build up the disambiguation string from the release group and release.
+    disambig = []
+    if release['release-group'].get('disambiguation'):
+        disambig.append(release['release-group'].get('disambiguation'))
+    if release.get('disambiguation'):
+        disambig.append(release.get('disambiguation'))
+    info.albumdisambig = u', '.join(disambig)
+
     # Release type not always populated.
     if 'type' in release['release-group']:
         reltype = release['release-group']['type']
         if reltype:
             info.albumtype = reltype.lower()
 
-    # Release date.
-    if 'first-release-date' in release['release-group']:
-        # Try earliest release date for the entire group first.
-        _set_date_str(info, release['release-group']['first-release-date'])
-    elif 'date' in release:
-        # Fall back to release-specific date.
-        _set_date_str(info, release['date'])
+    # Release dates.
+    release_date = release.get('date')
+    release_group_date = release['release-group'].get('first-release-date')
+    if not release_date:
+        # Fall back if release-specific date is not available.
+        release_date = release_group_date
+    _set_date_str(info, release_date, False)
+    _set_date_str(info, release_group_date, True)
 
     # Label name.
     if release.get('label-info-list'):
@@ -209,6 +285,7 @@ def album_info(release):
         first_medium = release['medium-list'][0]
         info.media = first_medium.get('format')
 
+    info.decode()
     return info
 
 def match_album(artist, album, tracks=None, limit=SEARCH_LIMIT):
@@ -234,7 +311,7 @@ def match_album(artist, album, tracks=None, limit=SEARCH_LIMIT):
         return
 
     try:
-        res = _mb_release_search(limit=limit, **criteria)
+        res = musicbrainzngs.search_releases(limit=limit, **criteria)
     except musicbrainzngs.MusicBrainzError as exc:
         raise MusicBrainzAPIError(exc, 'release search', criteria,
                                   traceback.format_exc())
@@ -242,8 +319,8 @@ def match_album(artist, album, tracks=None, limit=SEARCH_LIMIT):
         # The search result is missing some data (namely, the tracks),
         # so we just use the ID and fetch the rest of the information.
         albuminfo = album_for_id(release['id'])
-        assert albuminfo is not None
-        yield albuminfo
+        if albuminfo is not None:
+            yield albuminfo
 
 def match_track(artist, title, limit=SEARCH_LIMIT):
     """Searches for a single track and returns an iterable of TrackInfo
@@ -258,20 +335,34 @@ def match_track(artist, title, limit=SEARCH_LIMIT):
         return
 
     try:
-        res = _mb_recording_search(limit=limit, **criteria)
+        res = musicbrainzngs.search_recordings(limit=limit, **criteria)
     except musicbrainzngs.MusicBrainzError as exc:
         raise MusicBrainzAPIError(exc, 'recording search', criteria,
                                   traceback.format_exc())
     for recording in res['recording-list']:
         yield track_info(recording)
 
+def _parse_id(s):
+    """Search for a MusicBrainz ID in the given string and return it. If
+    no ID can be found, return None.
+    """
+    # Find the first thing that looks like a UUID/MBID.
+    match = re.search('[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}', s)
+    if match:
+        return match.group()
+
 def album_for_id(albumid):
     """Fetches an album by its MusicBrainz ID and returns an AlbumInfo
     object or None if the album is not found. May raise a
     MusicBrainzAPIError.
     """
+    albumid = _parse_id(albumid)
+    if not albumid:
+        log.error('Invalid MBID.')
+        return
     try:
-        res = musicbrainzngs.get_release_by_id(albumid, RELEASE_INCLUDES)
+        res = musicbrainzngs.get_release_by_id(albumid,
+                                               RELEASE_INCLUDES)
     except musicbrainzngs.ResponseError:
         log.debug('Album ID match failed.')
         return None
@@ -284,6 +375,10 @@ def track_for_id(trackid):
     """Fetches a track by its MusicBrainz ID. Returns a TrackInfo object
     or None if no track is found. May raise a MusicBrainzAPIError.
     """
+    trackid = _parse_id(trackid)
+    if not trackid:
+        log.error('Invalid MBID.')
+        return
     try:
         res = musicbrainzngs.get_recording_by_id(trackid, TRACK_INCLUDES)
     except musicbrainzngs.ResponseError:

beets/config_default.yaml

@@ -0,0 +1,102 @@
+library: library.db
+directory: ~/Music
+
+import:
+    write: yes
+    copy: yes
+    move: no
+    delete: no
+    resume: ask
+    incremental: no
+    quiet_fallback: skip
+    none_rec_action: ask
+    timid: no
+    log:
+    autotag: yes
+    quiet: no
+    singletons: no
+    default_action: apply
+    languages: []
+    detail: no
+    flat: no
+    group_albums: no
+
+clutter: ["Thumbs.DB", ".DS_Store"]
+ignore: [".*", "*~", "System Volume Information"]
+replace:
+    '[\\/]': _
+    '^\.': _
+    '[\x00-\x1f]': _
+    '[<>:"\?\*\|]': _
+    '\.$': _
+    '\s+$': ''
+    '^\s+': ''
+path_sep_replace: _
+art_filename: cover
+max_filename_length: 0
+
+plugins: []
+pluginpath: []
+threaded: yes
+color: yes
+timeout: 5.0
+per_disc_numbering: no
+verbose: no
+terminal_encoding: utf8
+original_date: no
+id3v23: no
+
+ui:
+    terminal_width: 80
+    length_diff_thresh: 10.0
+
+list_format_item: $artist - $album - $title
+list_format_album: $albumartist - $album
+time_format: '%Y-%m-%d %H:%M:%S'
+
+paths:
+    default: $albumartist/$album%aunique{}/$track $title
+    singleton: Non-Album/$artist/$title
+    comp: Compilations/$album%aunique{}/$track $title
+
+statefile: state.pickle
+
+musicbrainz:
+    host: musicbrainz.org
+    ratelimit: 1
+    ratelimit_interval: 1.0
+
+match:
+    strong_rec_thresh: 0.04
+    medium_rec_thresh: 0.25
+    rec_gap_thresh: 0.25
+    max_rec:
+        missing_tracks: medium
+        unmatched_tracks: medium
+    distance_weights:
+        source: 2.0
+        artist: 3.0
+        album: 3.0
+        media: 1.0
+        mediums: 1.0
+        year: 1.0
+        country: 0.5
+        label: 0.5
+        catalognum: 0.5
+        albumdisambig: 0.5
+        album_id: 5.0
+        tracks: 2.0
+        missing_tracks: 0.9
+        unmatched_tracks: 0.6
+        track_title: 3.0
+        track_artist: 2.0
+        track_index: 1.0
+        track_length: 2.0
+        track_id: 5.0
+    preferred:
+        countries: []
+        media: []
+        original_year: no
+    ignored: []
+    track_length_grace: 10
+    track_length_max: 30

beets/dbcore/__init__.py

@@ -1,5 +1,5 @@
 # This file is part of beets.
-# Copyright 2012, Adrian Sampson.
+# Copyright 2014, Adrian Sampson.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -12,11 +12,9 @@
 # The above copyright notice and this permission notice shall be
 # included in all copies or substantial portions of the Software.
 
-#
-# MODIFIED TO WORK WITH HEADPHONES!!
-#
-
-__version__ = '1.0b15'
-__author__ = 'Adrian Sampson <adrian@radbox.org>'
-
-from lib.beets.library import Library
+"""DBCore is an abstract database package that forms the basis for beets'
+Library.
+"""
+from .db import Model, Database
+from .query import Query, FieldQuery, MatchQuery, AndQuery, OrQuery
+from .types import Type

beets/dbcore/db.py

@@ -0,0 +1,727 @@
+# This file is part of beets.
+# Copyright 2014, Adrian Sampson.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+
+"""The central Model and Database constructs for DBCore.
+"""
+import time
+import os
+from collections import defaultdict
+import threading
+import sqlite3
+import contextlib
+
+import beets
+from beets.util.functemplate import Template
+from .query import MatchQuery
+
+
+
+# Abstract base for model classes.
+
+
+class Model(object):
+    """An abstract object representing an object in the database. Model
+    objects act like dictionaries (i.e., the allow subscript access like
+    ``obj['field']``). The same field set is available via attribute
+    access as a shortcut (i.e., ``obj.field``). Three kinds of attributes are
+    available:
+
+    * **Fixed attributes** come from a predetermined list of field
+      names. These fields correspond to SQLite table columns and are
+      thus fast to read, write, and query.
+    * **Flexible attributes** are free-form and do not need to be listed
+      ahead of time.
+    * **Computed attributes** are read-only fields computed by a getter
+      function provided by a plugin.
+
+    Access to all three field types is uniform: ``obj.field`` works the
+    same regardless of whether ``field`` is fixed, flexible, or
+    computed.
+
+    Model objects can optionally be associated with a `Library` object,
+    in which case they can be loaded and stored from the database. Dirty
+    flags are used to track which fields need to be stored.
+    """
+
+    # Abstract components (to be provided by subclasses).
+
+    _table = None
+    """The main SQLite table name.
+    """
+
+    _flex_table = None
+    """The flex field SQLite table name.
+    """
+
+    _fields = {}
+    """A mapping indicating available "fixed" fields on this type. The
+    keys are field names and the values are Type objects.
+    """
+
+    _bytes_keys = ()
+    """Keys whose values should be stored as raw bytes blobs rather than
+    strings.
+    """
+
+    _search_fields = ()
+    """The fields that should be queried by default by unqualified query
+    terms.
+    """
+
+    @classmethod
+    def _getters(cls):
+        """Return a mapping from field names to getter functions.
+        """
+        # We could cache this if it becomes a performance problem to
+        # gather the getter mapping every time.
+        raise NotImplementedError()
+
+    def _template_funcs(self):
+        """Return a mapping from function names to text-transformer
+        functions.
+        """
+        # As above: we could consider caching this result.
+        raise NotImplementedError()
+
+
+    # Basic operation.
+
+    def __init__(self, db=None, **values):
+        """Create a new object with an optional Database association and
+        initial field values.
+        """
+        self._db = db
+        self._dirty = set()
+        self._values_fixed = {}
+        self._values_flex = {}
+
+        # Initial contents.
+        self.update(values)
+        self.clear_dirty()
+
+    def __repr__(self):
+        return '{0}({1})'.format(
+            type(self).__name__,
+            ', '.join('{0}={1!r}'.format(k, v) for k, v in dict(self).items()),
+        )
+
+    def clear_dirty(self):
+        """Mark all fields as *clean* (i.e., not needing to be stored to
+        the database).
+        """
+        self._dirty = set()
+
+    def _check_db(self, need_id=True):
+        """Ensure that this object is associated with a database row: it
+        has a reference to a database (`_db`) and an id. A ValueError
+        exception is raised otherwise.
+        """
+        if not self._db:
+            raise ValueError('{0} has no database'.format(type(self).__name__))
+        if need_id and not self.id:
+            raise ValueError('{0} has no id'.format(type(self).__name__))
+
+
+    # Essential field accessors.
+
+    def __getitem__(self, key):
+        """Get the value for a field. Raise a KeyError if the field is
+        not available.
+        """
+        getters = self._getters()
+        if key in getters:  # Computed.
+            return getters[key](self)
+        elif key in self._fields:  # Fixed.
+            return self._values_fixed.get(key)
+        elif key in self._values_flex:  # Flexible.
+            return self._values_flex[key]
+        else:
+            raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        """Assign the value for a field.
+        """
+        source = self._values_fixed if key in self._fields \
+                 else self._values_flex
+        old_value = source.get(key)
+        source[key] = value
+        if old_value != value:
+            self._dirty.add(key)
+
+    def __delitem__(self, key):
+        """Remove a flexible attribute from the model.
+        """
+        if key in self._values_flex:  # Flexible.
+            del self._values_flex[key]
+            self._dirty.add(key)  # Mark for dropping on store.
+        elif key in self._getters():  # Computed.
+            raise KeyError('computed field {0} cannot be deleted'.format(key))
+        elif key in self._fields:  # Fixed.
+            raise KeyError('fixed field {0} cannot be deleted'.format(key))
+        else:
+            raise KeyError('no such field {0}'.format(key))
+
+    def keys(self, computed=False):
+        """Get a list of available field names for this object. The
+        `computed` parameter controls whether computed (plugin-provided)
+        fields are included in the key list.
+        """
+        base_keys = list(self._fields) + self._values_flex.keys()
+        if computed:
+            return base_keys + self._getters().keys()
+        else:
+            return base_keys
+
+
+    # Act like a dictionary.
+
+    def update(self, values):
+        """Assign all values in the given dict.
+        """
+        for key, value in values.items():
+            self[key] = value
+
+    def items(self):
+        """Iterate over (key, value) pairs that this object contains.
+        Computed fields are not included.
+        """
+        for key in self:
+            yield key, self[key]
+
+    def get(self, key, default=None):
+        """Get the value for a given key or `default` if it does not
+        exist.
+        """
+        if key in self:
+            return self[key]
+        else:
+            return default
+
+    def __contains__(self, key):
+        """Determine whether `key` is an attribute on this object.
+        """
+        return key in self.keys(True)
+
+    def __iter__(self):
+        """Iterate over the available field names (excluding computed
+        fields).
+        """
+        return iter(self.keys())
+
+
+    # Convenient attribute access.
+
+    def __getattr__(self, key):
+        if key.startswith('_'):
+            raise AttributeError('model has no attribute {0!r}'.format(key))
+        else:
+            try:
+                return self[key]
+            except KeyError:
+                raise AttributeError('no such field {0!r}'.format(key))
+
+    def __setattr__(self, key, value):
+        if key.startswith('_'):
+            super(Model, self).__setattr__(key, value)
+        else:
+            self[key] = value
+
+    def __delattr__(self, key):
+        if key.startswith('_'):
+            super(Model, self).__delattr__(key)
+        else:
+            del self[key]
+
+
+    # Database interaction (CRUD methods).
+
+    def store(self):
+        """Save the object's metadata into the library database.
+        """
+        self._check_db()
+
+        # Build assignments for query.
+        assignments = ''
+        subvars = []
+        for key in self._fields:
+            if key != 'id' and key in self._dirty:
+                self._dirty.remove(key)
+                assignments += key + '=?,'
+                value = self[key]
+                # Wrap path strings in buffers so they get stored
+                # "in the raw".
+                if key in self._bytes_keys and isinstance(value, str):
+                    value = buffer(value)
+                subvars.append(value)
+        assignments = assignments[:-1]  # Knock off last ,
+
+        with self._db.transaction() as tx:
+            # Main table update.
+            if assignments:
+                query = 'UPDATE {0} SET {1} WHERE id=?'.format(
+                    self._table, assignments
+                )
+                subvars.append(self.id)
+                tx.mutate(query, subvars)
+
+            # Modified/added flexible attributes.
+            for key, value in self._values_flex.items():
+                if key in self._dirty:
+                    self._dirty.remove(key)
+                    tx.mutate(
+                        'INSERT INTO {0} '
+                        '(entity_id, key, value) '
+                        'VALUES (?, ?, ?);'.format(self._flex_table),
+                        (self.id, key, value),
+                    )
+
+            # Deleted flexible attributes.
+            for key in self._dirty:
+                tx.mutate(
+                    'DELETE FROM {0} '
+                    'WHERE entity_id=? AND key=?'.format(self._flex_table),
+                    (self.id, key)
+                )
+
+        self.clear_dirty()
+
+    def load(self):
+        """Refresh the object's metadata from the library database.
+        """
+        self._check_db()
+        stored_obj = self._db._get(type(self), self.id)
+        assert stored_obj is not None, "object {0} not in DB".format(self.id)
+        self.update(dict(stored_obj))
+        self.clear_dirty()
+
+    def remove(self):
+        """Remove the object's associated rows from the database.
+        """
+        self._check_db()
+        with self._db.transaction() as tx:
+            tx.mutate(
+                'DELETE FROM {0} WHERE id=?'.format(self._table),
+                (self.id,)
+            )
+            tx.mutate(
+                'DELETE FROM {0} WHERE entity_id=?'.format(self._flex_table),
+                (self.id,)
+            )
+
+    def add(self, db=None):
+        """Add the object to the library database. This object must be
+        associated with a database; you can provide one via the `db`
+        parameter or use the currently associated database.
+
+        The object's `id` and `added` fields are set along with any
+        current field values.
+        """
+        if db:
+            self._db = db
+        self._check_db(False)
+
+        with self._db.transaction() as tx:
+            new_id = tx.mutate(
+                'INSERT INTO {0} DEFAULT VALUES'.format(self._table)
+            )
+            self.id = new_id
+            self.added = time.time()
+
+            # Mark every non-null field as dirty and store.
+            for key in self:
+                if self[key] is not None:
+                    self._dirty.add(key)
+            self.store()
+
+
+    # Formatting and templating.
+
+    @classmethod
+    def _format(cls, key, value, for_path=False):
+        """Format a value as the given field for this model.
+        """
+        # Format the value as a string according to its type, if any.
+        if key in cls._fields:
+            value = cls._fields[key].format(value)
+            # Formatting must result in a string. To deal with
+            # Python2isms, implicitly convert ASCII strings.
+            assert isinstance(value, basestring), \
+                    u'field formatter must produce strings'
+            if isinstance(value, bytes):
+                value = value.decode('utf8', 'ignore')
+
+        elif not isinstance(value, unicode):
+            # Fallback formatter. Convert to unicode at all cost.
+            if value is None:
+                value = u''
+            elif isinstance(value, basestring):
+                if isinstance(value, bytes):
+                    value = value.decode('utf8', 'ignore')
+            else:
+                value = unicode(value)
+
+        if for_path:
+            sep_repl = beets.config['path_sep_replace'].get(unicode)
+            for sep in (os.path.sep, os.path.altsep):
+                if sep:
+                    value = value.replace(sep, sep_repl)
+
+        return value
+
+    def _get_formatted(self, key, for_path=False):
+        """Get a field value formatted as a string (`unicode` object)
+        for display to the user. If `for_path` is true, then the value
+        will be sanitized for inclusion in a pathname (i.e., path
+        separators will be removed from the value).
+        """
+        return self._format(key, self.get(key), for_path)
+
+    def _formatted_mapping(self, for_path=False):
+        """Get a mapping containing all values on this object formatted
+        as human-readable strings.
+        """
+        # In the future, this could be made "lazy" to avoid computing
+        # fields unnecessarily.
+        out = {}
+        for key in self.keys(True):
+            out[key] = self._get_formatted(key, for_path)
+        return out
+
+    def evaluate_template(self, template, for_path=False):
+        """Evaluate a template (a string or a `Template` object) using
+        the object's fields. If `for_path` is true, then no new path
+        separators will be added to the template.
+        """
+        # Build value mapping.
+        mapping = self._formatted_mapping(for_path)
+
+        # Get template functions.
+        funcs = self._template_funcs()
+
+        # Perform substitution.
+        if isinstance(template, basestring):
+            template = Template(template)
+        return template.substitute(mapping, funcs)
+
+
+    # Parsing.
+
+    @classmethod
+    def _parse(cls, key, string):
+        """Parse a string as a value for the given key.
+        """
+        if not isinstance(string, basestring):
+            raise TypeError("_parse() argument must be a string")
+
+        typ = cls._fields.get(key)
+        if typ:
+            return typ.parse(string)
+        else:
+            # Fall back to unparsed string.
+            return string
+
+
+
+# Database controller and supporting interfaces.
+
+
+class Results(object):
+    """An item query result set. Iterating over the collection lazily
+    constructs LibModel objects that reflect database rows.
+    """
+    def __init__(self, model_class, rows, db, query=None):
+        """Create a result set that will construct objects of type
+        `model_class`, which should be a subclass of `LibModel`, out of
+        the query result mapping in `rows`. The new objects are
+        associated with the database `db`. If `query` is provided, it is
+        used as a predicate to filter the results for a "slow query" that
+        cannot be evaluated by the database directly.
+        """
+        self.model_class = model_class
+        self.rows = rows
+        self.db = db
+        self.query = query
+
+    def __iter__(self):
+        """Construct Python objects for all rows that pass the query
+        predicate.
+        """
+        for row in self.rows:
+            # Get the flexible attributes for the object.
+            with self.db.transaction() as tx:
+                flex_rows = tx.query(
+                    'SELECT * FROM {0} WHERE entity_id=?'.format(
+                        self.model_class._flex_table
+                    ),
+                    (row['id'],)
+                )
+            values = dict(row)
+            values.update(
+                dict((row['key'], row['value']) for row in flex_rows)
+            )
+
+            # Construct the Python object and yield it if it passes the
+            # predicate.
+            obj = self.model_class(self.db, **values)
+            if not self.query or self.query.match(obj):
+                yield obj
+
+    def __len__(self):
+        """Get the number of matching objects.
+        """
+        if self.query:
+            # A slow query. Fall back to testing every object.
+            count = 0
+            for obj in self:
+                count += 1
+            return count
+
+        else:
+            # A fast query. Just count the rows.
+            return len(self.rows)
+
+    def __nonzero__(self):
+        """Does this result contain any objects?
+        """
+        return bool(len(self))
+
+    def __getitem__(self, n):
+        """Get the nth item in this result set. This is inefficient: all
+        items up to n are materialized and thrown away.
+        """
+        it = iter(self)
+        try:
+            for i in range(n):
+                it.next()
+            return it.next()
+        except StopIteration:
+            raise IndexError('result index {0} out of range'.format(n))
+
+    def get(self):
+        """Return the first matching object, or None if no objects
+        match.
+        """
+        it = iter(self)
+        try:
+            return it.next()
+        except StopIteration:
+            return None
+
+
+class Transaction(object):
+    """A context manager for safe, concurrent access to the database.
+    All SQL commands should be executed through a transaction.
+    """
+    def __init__(self, db):
+        self.db = db
+
+    def __enter__(self):
+        """Begin a transaction. This transaction may be created while
+        another is active in a different thread.
+        """
+        with self.db._tx_stack() as stack:
+            first = not stack
+            stack.append(self)
+        if first:
+            # Beginning a "root" transaction, which corresponds to an
+            # SQLite transaction.
+            self.db._db_lock.acquire()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        """Complete a transaction. This must be the most recently
+        entered but not yet exited transaction. If it is the last active
+        transaction, the database updates are committed.
+        """
+        with self.db._tx_stack() as stack:
+            assert stack.pop() is self
+            empty = not stack
+        if empty:
+            # Ending a "root" transaction. End the SQLite transaction.
+            self.db._connection().commit()
+            self.db._db_lock.release()
+
+    def query(self, statement, subvals=()):
+        """Execute an SQL statement with substitution values and return
+        a list of rows from the database.
+        """
+        cursor = self.db._connection().execute(statement, subvals)
+        return cursor.fetchall()
+
+    def mutate(self, statement, subvals=()):
+        """Execute an SQL statement with substitution values and return
+        the row ID of the last affected row.
+        """
+        cursor = self.db._connection().execute(statement, subvals)
+        return cursor.lastrowid
+
+    def script(self, statements):
+        """Execute a string containing multiple SQL statements."""
+        self.db._connection().executescript(statements)
+
+
+class Database(object):
+    """A container for Model objects that wraps an SQLite database as
+    the backend.
+    """
+    _models = ()
+    """The Model subclasses representing tables in this database.
+    """
+
+    def __init__(self, path):
+        self.path = path
+
+        self._connections = {}
+        self._tx_stacks = defaultdict(list)
+
+        # A lock to protect the _connections and _tx_stacks maps, which
+        # both map thread IDs to private resources.
+        self._shared_map_lock = threading.Lock()
+
+        # A lock to protect access to the database itself. SQLite does
+        # allow multiple threads to access the database at the same
+        # time, but many users were experiencing crashes related to this
+        # capability: where SQLite was compiled without HAVE_USLEEP, its
+        # backoff algorithm in the case of contention was causing
+        # whole-second sleeps (!) that would trigger its internal
+        # timeout. Using this lock ensures only one SQLite transaction
+        # is active at a time.
+        self._db_lock = threading.Lock()
+
+        # Set up database schema.
+        for model_cls in self._models:
+            self._make_table(model_cls._table, model_cls._fields)
+            self._make_attribute_table(model_cls._flex_table)
+
+
+    # Primitive access control: connections and transactions.
+
+    def _connection(self):
+        """Get a SQLite connection object to the underlying database.
+        One connection object is created per thread.
+        """
+        thread_id = threading.current_thread().ident
+        with self._shared_map_lock:
+            if thread_id in self._connections:
+                return self._connections[thread_id]
+            else:
+                # Make a new connection.
+                conn = sqlite3.connect(
+                    self.path,
+                    timeout=beets.config['timeout'].as_number(),
+                )
+
+                # Access SELECT results like dictionaries.
+                conn.row_factory = sqlite3.Row
+
+                self._connections[thread_id] = conn
+                return conn
+
+    @contextlib.contextmanager
+    def _tx_stack(self):
+        """A context manager providing access to the current thread's
+        transaction stack. The context manager synchronizes access to
+        the stack map. Transactions should never migrate across threads.
+        """
+        thread_id = threading.current_thread().ident
+        with self._shared_map_lock:
+            yield self._tx_stacks[thread_id]
+
+    def transaction(self):
+        """Get a :class:`Transaction` object for interacting directly
+        with the underlying SQLite database.
+        """
+        return Transaction(self)
+
+
+    # Schema setup and migration.
+
+    def _make_table(self, table, fields):
+        """Set up the schema of the database. `fields` is a mapping
+        from field names to `Type`s. Columns are added if necessary.
+        """
+        # Get current schema.
+        with self.transaction() as tx:
+            rows = tx.query('PRAGMA table_info(%s)' % table)
+        current_fields = set([row[1] for row in rows])
+
+        field_names = set(fields.keys())
+        if current_fields.issuperset(field_names):
+            # Table exists and has all the required columns.
+            return
+
+        if not current_fields:
+            # No table exists.
+            columns = []
+            for name, typ in fields.items():
+                columns.append('{0} {1}'.format(name, typ.sql))
+            setup_sql = 'CREATE TABLE {0} ({1});\n'.format(table,
+                                                           ', '.join(columns))
+
+        else:
+            # Table exists does not match the field set.
+            setup_sql = ''
+            for name, typ in fields.items():
+                if name in current_fields:
+                    continue
+                setup_sql += 'ALTER TABLE {0} ADD COLUMN {1} {2};\n'.format(
+                    table, name, typ.sql
+                )
+
+        with self.transaction() as tx:
+            tx.script(setup_sql)
+
+    def _make_attribute_table(self, flex_table):
+        """Create a table and associated index for flexible attributes
+        for the given entity (if they don't exist).
+        """
+        with self.transaction() as tx:
+            tx.script("""
+                CREATE TABLE IF NOT EXISTS {0} (
+                    id INTEGER PRIMARY KEY,
+                    entity_id INTEGER,
+                    key TEXT,
+                    value TEXT,
+                    UNIQUE(entity_id, key) ON CONFLICT REPLACE);
+                CREATE INDEX IF NOT EXISTS {0}_by_entity
+                    ON {0} (entity_id);
+                """.format(flex_table))
+
+
+    # Querying.
+
+    def _fetch(self, model_cls, query, order_by=None):
+        """Fetch the objects of type `model_cls` matching the given
+        query. The query may be given as a string, string sequence, a
+        Query object, or None (to fetch everything). If provided,
+        `order_by` is a SQLite ORDER BY clause for sorting.
+        """
+        where, subvals = query.clause()
+
+        sql = "SELECT * FROM {0} WHERE {1}".format(
+            model_cls._table,
+            where or '1',
+        )
+        if order_by:
+            sql += " ORDER BY {0}".format(order_by)
+        with self.transaction() as tx:
+            rows = tx.query(sql, subvals)
+
+        return Results(model_cls, rows, self, None if where else query)
+
+    def _get(self, model_cls, id):
+        """Get a Model object by its id or None if the id does not
+        exist.
+        """
+        return self._fetch(model_cls, MatchQuery('id', id)).get()

beets/dbcore/query.py

@@ -0,0 +1,494 @@
+# This file is part of beets.
+# Copyright 2014, Adrian Sampson.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+
+"""The Query type hierarchy for DBCore.
+"""
+import re
+from beets import util
+from datetime import datetime, timedelta
+
+
+class Query(object):
+    """An abstract class representing a query into the item database.
+    """
+    def clause(self):
+        """Generate an SQLite expression implementing the query.
+        Return a clause string, a sequence of substitution values for
+        the clause, and a Query object representing the "remainder"
+        Returns (clause, subvals) where clause is a valid sqlite
+        WHERE clause implementing the query and subvals is a list of
+        items to be substituted for ?s in the clause.
+        """
+        return None, ()
+
+    def match(self, item):
+        """Check whether this query matches a given Item. Can be used to
+        perform queries on arbitrary sets of Items.
+        """
+        raise NotImplementedError
+
+
+class FieldQuery(Query):
+    """An abstract query that searches in a specific field for a
+    pattern. Subclasses must provide a `value_match` class method, which
+    determines whether a certain pattern string matches a certain value
+    string. Subclasses may also provide `col_clause` to implement the
+    same matching functionality in SQLite.
+    """
+    def __init__(self, field, pattern, fast=True):
+        self.field = field
+        self.pattern = pattern
+        self.fast = fast
+
+    def col_clause(self):
+        return None, ()
+
+    def clause(self):
+        if self.fast:
+            return self.col_clause()
+        else:
+            # Matching a flexattr. This is a slow query.
+            return None, ()
+
+    @classmethod
+    def value_match(cls, pattern, value):
+        """Determine whether the value matches the pattern. Both
+        arguments are strings.
+        """
+        raise NotImplementedError()
+
+    def match(self, item):
+        return self.value_match(self.pattern, item.get(self.field))
+
+
+class MatchQuery(FieldQuery):
+    """A query that looks for exact matches in an item field."""
+    def col_clause(self):
+        return self.field + " = ?", [self.pattern]
+
+    @classmethod
+    def value_match(cls, pattern, value):
+        return pattern == value
+
+
+class StringFieldQuery(FieldQuery):
+    """A FieldQuery that converts values to strings before matching
+    them.
+    """
+    @classmethod
+    def value_match(cls, pattern, value):
+        """Determine whether the value matches the pattern. The value
+        may have any type.
+        """
+        return cls.string_match(pattern, util.as_string(value))
+
+    @classmethod
+    def string_match(cls, pattern, value):
+        """Determine whether the value matches the pattern. Both
+        arguments are strings. Subclasses implement this method.
+        """
+        raise NotImplementedError()
+
+
+class SubstringQuery(StringFieldQuery):
+    """A query that matches a substring in a specific item field."""
+    def col_clause(self):
+        search = '%' + (self.pattern.replace('\\','\\\\').replace('%','\\%')
+                            .replace('_','\\_')) + '%'
+        clause = self.field + " like ? escape '\\'"
+        subvals = [search]
+        return clause, subvals
+
+    @classmethod
+    def string_match(cls, pattern, value):
+        return pattern.lower() in value.lower()
+
+
+class RegexpQuery(StringFieldQuery):
+    """A query that matches a regular expression in a specific item
+    field.
+    """
+    @classmethod
+    def string_match(cls, pattern, value):
+        try:
+            res = re.search(pattern, value)
+        except re.error:
+            # Invalid regular expression.
+            return False
+        return res is not None
+
+
+class BooleanQuery(MatchQuery):
+    """Matches a boolean field. Pattern should either be a boolean or a
+    string reflecting a boolean.
+    """
+    def __init__(self, field, pattern, fast=True):
+        super(BooleanQuery, self).__init__(field, pattern, fast)
+        if isinstance(pattern, basestring):
+            self.pattern = util.str2bool(pattern)
+        self.pattern = int(self.pattern)
+
+
+class BytesQuery(MatchQuery):
+    """Match a raw bytes field (i.e., a path). This is a necessary hack
+    to work around the `sqlite3` module's desire to treat `str` and
+    `unicode` equivalently in Python 2. Always use this query instead of
+    `MatchQuery` when matching on BLOB values.
+    """
+    def __init__(self, field, pattern):
+        super(BytesQuery, self).__init__(field, pattern)
+
+        # Use a buffer representation of the pattern for SQLite
+        # matching. This instructs SQLite to treat the blob as binary
+        # rather than encoded Unicode.
+        if isinstance(self.pattern, basestring):
+            # Implicitly coerce Unicode strings to their bytes
+            # equivalents.
+            if isinstance(self.pattern, unicode):
+                self.pattern = self.pattern.encode('utf8')
+            self.buf_pattern = buffer(self.pattern)
+        elif isinstance(self.pattern, buffer):
+            self.buf_pattern = self.pattern
+            self.pattern = bytes(self.pattern)
+
+    def col_clause(self):
+        return self.field + " = ?", [self.buf_pattern]
+
+
+class NumericQuery(FieldQuery):
+    """Matches numeric fields. A syntax using Ruby-style range ellipses
+    (``..``) lets users specify one- or two-sided ranges. For example,
+    ``year:2001..`` finds music released since the turn of the century.
+    """
+    def _convert(self, s):
+        """Convert a string to a numeric type (float or int). If the
+        string cannot be converted, return None.
+        """
+        # This is really just a bit of fun premature optimization.
+        try:
+            return int(s)
+        except ValueError:
+            try:
+                return float(s)
+            except ValueError:
+                return None
+
+    def __init__(self, field, pattern, fast=True):
+        super(NumericQuery, self).__init__(field, pattern, fast)
+
+        parts = pattern.split('..', 1)
+        if len(parts) == 1:
+            # No range.
+            self.point = self._convert(parts[0])
+            self.rangemin = None
+            self.rangemax = None
+        else:
+            # One- or two-sided range.
+            self.point = None
+            self.rangemin = self._convert(parts[0])
+            self.rangemax = self._convert(parts[1])
+
+    def match(self, item):
+        value = getattr(item, self.field)
+        if isinstance(value, basestring):
+            value = self._convert(value)
+
+        if self.point is not None:
+            return value == self.point
+        else:
+            if self.rangemin is not None and value < self.rangemin:
+                return False
+            if self.rangemax is not None and value > self.rangemax:
+                return False
+            return True
+
+    def col_clause(self):
+        if self.point is not None:
+            return self.field + '=?', (self.point,)
+        else:
+            if self.rangemin is not None and self.rangemax is not None:
+                return (u'{0} >= ? AND {0} <= ?'.format(self.field),
+                        (self.rangemin, self.rangemax))
+            elif self.rangemin is not None:
+                return u'{0} >= ?'.format(self.field), (self.rangemin,)
+            elif self.rangemax is not None:
+                return u'{0} <= ?'.format(self.field), (self.rangemax,)
+            else:
+                return '1', ()
+
+
+class CollectionQuery(Query):
+    """An abstract query class that aggregates other queries. Can be
+    indexed like a list to access the sub-queries.
+    """
+    def __init__(self, subqueries=()):
+        self.subqueries = subqueries
+
+    # Act like a sequence.
+    def __len__(self):
+        return len(self.subqueries)
+    def __getitem__(self, key):
+        return self.subqueries[key]
+    def __iter__(self):
+        return iter(self.subqueries)
+    def __contains__(self, item):
+        return item in self.subqueries
+
+    def clause_with_joiner(self, joiner):
+        """Returns a clause created by joining together the clauses of
+        all subqueries with the string joiner (padded by spaces).
+        """
+        clause_parts = []
+        subvals = []
+        for subq in self.subqueries:
+            subq_clause, subq_subvals = subq.clause()
+            if not subq_clause:
+                # Fall back to slow query.
+                return None, ()
+            clause_parts.append('(' + subq_clause + ')')
+            subvals += subq_subvals
+        clause = (' ' + joiner + ' ').join(clause_parts)
+        return clause, subvals
+
+
+class AnyFieldQuery(CollectionQuery):
+    """A query that matches if a given FieldQuery subclass matches in
+    any field. The individual field query class is provided to the
+    constructor.
+    """
+    def __init__(self, pattern, fields, cls):
+        self.pattern = pattern
+        self.fields = fields
+        self.query_class = cls
+
+        subqueries = []
+        for field in self.fields:
+            subqueries.append(cls(field, pattern, True))
+        super(AnyFieldQuery, self).__init__(subqueries)
+
+    def clause(self):
+        return self.clause_with_joiner('or')
+
+    def match(self, item):
+        for subq in self.subqueries:
+            if subq.match(item):
+                return True
+        return False
+
+
+class MutableCollectionQuery(CollectionQuery):
+    """A collection query whose subqueries may be modified after the
+    query is initialized.
+    """
+    def __setitem__(self, key, value):
+        self.subqueries[key] = value
+
+    def __delitem__(self, key):
+        del self.subqueries[key]
+
+
+class AndQuery(MutableCollectionQuery):
+    """A conjunction of a list of other queries."""
+    def clause(self):
+        return self.clause_with_joiner('and')
+
+    def match(self, item):
+        return all([q.match(item) for q in self.subqueries])
+
+
+class OrQuery(MutableCollectionQuery):
+    """A conjunction of a list of other queries."""
+    def clause(self):
+        return self.clause_with_joiner('or')
+
+    def match(self, item):
+        return any([q.match(item) for q in self.subqueries])
+
+
+class TrueQuery(Query):
+    """A query that always matches."""
+    def clause(self):
+        return '1', ()
+
+    def match(self, item):
+        return True
+
+
+class FalseQuery(Query):
+    """A query that never matches."""
+    def clause(self):
+        return '0', ()
+
+    def match(self, item):
+        return False
+
+
+
+# Time/date queries.
+
+
+def _to_epoch_time(date):
+    """Convert a `datetime` object to an integer number of seconds since
+    the (local) Unix epoch.
+    """
+    epoch = datetime.fromtimestamp(0)
+    delta = date - epoch
+    try:
+        return int(delta.total_seconds())
+    except AttributeError:
+        # datetime.timedelta.total_seconds() is not available on Python 2.6
+        return delta.seconds + delta.days * 24 * 3600
+
+
+def _parse_periods(pattern):
+    """Parse a string containing two dates separated by two dots (..).
+    Return a pair of `Period` objects.
+    """
+    parts = pattern.split('..', 1)
+    if len(parts) == 1:
+        instant = Period.parse(parts[0])
+        return (instant, instant)
+    else:
+        start = Period.parse(parts[0])
+        end = Period.parse(parts[1])
+        return (start, end)
+
+
+class Period(object):
+    """A period of time given by a date, time and precision.
+
+    Example: 2014-01-01 10:50:30 with precision 'month' represents all
+    instants of time during January 2014.
+    """
+
+    precisions = ('year', 'month', 'day')
+    date_formats = ('%Y', '%Y-%m', '%Y-%m-%d')
+
+    def __init__(self, date, precision):
+        """Create a period with the given date (a `datetime` object) and
+        precision (a string, one of "year", "month", or "day").
+        """
+        if precision not in Period.precisions:
+            raise ValueError('Invalid precision ' + str(precision))
+        self.date = date
+        self.precision = precision
+
+    @classmethod
+    def parse(cls, string):
+        """Parse a date and return a `Period` object or `None` if the
+        string is empty.
+        """
+        if not string:
+            return None
+        ordinal = string.count('-')
+        if ordinal >= len(cls.date_formats):
+            raise ValueError('date is not in one of the formats '
+                             + ', '.join(cls.date_formats))
+        date_format = cls.date_formats[ordinal]
+        date = datetime.strptime(string, date_format)
+        precision = cls.precisions[ordinal]
+        return cls(date, precision)
+
+    def open_right_endpoint(self):
+        """Based on the precision, convert the period to a precise
+        `datetime` for use as a right endpoint in a right-open interval.
+        """
+        precision = self.precision
+        date = self.date
+        if 'year' == self.precision:
+            return date.replace(year=date.year + 1, month=1)
+        elif 'month' == precision:
+            if (date.month < 12):
+                return date.replace(month=date.month + 1)
+            else:
+                return date.replace(year=date.year + 1, month=1)
+        elif 'day' == precision:
+            return date + timedelta(days=1)
+        else:
+            raise ValueError('unhandled precision ' + str(precision))
+
+
+class DateInterval(object):
+    """A closed-open interval of dates.
+
+    A left endpoint of None means since the beginning of time.
+    A right endpoint of None means towards infinity.
+    """
+
+    def __init__(self, start, end):
+        if start is not None and end is not None and not start < end:
+            raise ValueError("start date {0} is not before end date {1}"
+                             .format(start, end))
+        self.start = start
+        self.end = end
+
+    @classmethod
+    def from_periods(cls, start, end):
+        """Create an interval with two Periods as the endpoints.
+        """
+        end_date = end.open_right_endpoint() if end is not None else None
+        start_date = start.date if start is not None else None
+        return cls(start_date, end_date)
+
+    def contains(self, date):
+        if self.start is not None and date < self.start:
+            return False
+        if self.end is not None and date >= self.end:
+            return False
+        return True
+
+    def __str__(self):
+        return'[{0}, {1})'.format(self.start, self.end)
+
+
+class DateQuery(FieldQuery):
+    """Matches date fields stored as seconds since Unix epoch time.
+
+    Dates can be specified as ``year-month-day`` strings where only year
+    is mandatory.
+
+    The value of a date field can be matched against a date interval by
+    using an ellipsis interval syntax similar to that of NumericQuery.
+    """
+    def __init__(self, field, pattern, fast=True):
+        super(DateQuery, self).__init__(field, pattern, fast)
+        start, end = _parse_periods(pattern)
+        self.interval = DateInterval.from_periods(start, end)
+
+    def match(self, item):
+        timestamp = float(item[self.field])
+        date = datetime.utcfromtimestamp(timestamp)
+        return self.interval.contains(date)
+
+    _clause_tmpl = "{0} {1} ?"
+
+    def col_clause(self):
+        clause_parts = []
+        subvals = []
+
+        if self.interval.start:
+            clause_parts.append(self._clause_tmpl.format(self.field, ">="))
+            subvals.append(_to_epoch_time(self.interval.start))
+
+        if self.interval.end:
+            clause_parts.append(self._clause_tmpl.format(self.field, "<"))
+            subvals.append(_to_epoch_time(self.interval.end))
+
+        if clause_parts:
+            # One- or two-sided interval.
+            clause = ' AND '.join(clause_parts)
+        else:
+            # Match any date.
+            clause = '1'
+        return clause, subvals

beets/dbcore/types.py

@@ -0,0 +1,140 @@
+# This file is part of beets.
+# Copyright 2014, Adrian Sampson.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+
+"""Representation of type information for DBCore model fields.
+"""
+from . import query
+from beets.util import str2bool
+
+
+
+# Abstract base.
+
+
+class Type(object):
+    """An object encapsulating the type of a model field. Includes
+    information about how to store the value in the database, query,
+    format, and parse a given field.
+    """
+
+    sql = None
+    """The SQLite column type for the value.
+    """
+
+    query = None
+    """The `Query` subclass to be used when querying the field.
+    """
+
+    def format(self, value):
+        """Given a value of this type, produce a Unicode string
+        representing the value. This is used in template evaluation.
+        """
+        raise NotImplementedError()
+
+    def parse(self, string):
+        """Parse a (possibly human-written) string and return the
+        indicated value of this type.
+        """
+        raise NotImplementedError()
+
+
+
+# Reusable types.
+
+
+class Integer(Type):
+    """A basic integer type.
+    """
+    sql = u'INTEGER'
+    query = query.NumericQuery
+
+    def format(self, value):
+        return unicode(value or 0)
+
+    def parse(self, string):
+        try:
+            return int(string)
+        except ValueError:
+            return 0
+
+
+class PaddedInt(Integer):
+    """An integer field that is formatted with a given number of digits,
+    padded with zeroes.
+    """
+    def __init__(self, digits):
+        self.digits = digits
+
+    def format(self, value):
+        return u'{0:0{1}d}'.format(value or 0, self.digits)
+
+
+class ScaledInt(Integer):
+    """An integer whose formatting operation scales the number by a
+    constant and adds a suffix. Good for units with large magnitudes.
+    """
+    def __init__(self, unit, suffix=u''):
+        self.unit = unit
+        self.suffix = suffix
+
+    def format(self, value):
+        return u'{0}{1}'.format((value or 0) // self.unit, self.suffix)
+
+
+class Id(Integer):
+    """An integer used as the row key for a SQLite table.
+    """
+    sql = u'INTEGER PRIMARY KEY'
+
+
+class Float(Type):
+    """A basic floating-point type.
+    """
+    sql = u'REAL'
+    query = query.NumericQuery
+
+    def format(self, value):
+        return u'{0:.1f}'.format(value or 0.0)
+
+    def parse(self, string):
+        try:
+            return float(string)
+        except ValueError:
+            return 0.0
+
+
+class String(Type):
+    """A Unicode string type.
+    """
+    sql = u'TEXT'
+    query = query.SubstringQuery
+
+    def format(self, value):
+        return unicode(value) if value else u''
+
+    def parse(self, string):
+        return string
+
+
+class Boolean(Type):
+    """A boolean type.
+    """
+    sql = u'INTEGER'
+    query = query.BooleanQuery
+
+    def format(self, value):
+        return unicode(bool(value))
+
+    def parse(self, string):
+        return str2bool(string)

beets/plugins.py

@@ -1,5 +1,5 @@
 # This file is part of beets.
-# Copyright 2011, Adrian Sampson.
+# Copyright 2013, Adrian Sampson.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -15,14 +15,13 @@
 """Support for beets plugins."""
 
 import logging
-import itertools
 import traceback
 from collections import defaultdict
 
-from lib.beets import mediafile
+import beets
+from beets import mediafile
 
 PLUGIN_NAMESPACE = 'beetsplug'
-DEFAULT_PLUGINS = []
 
 # Plugins using the Last.fm API can share the same API key.
 LASTFM_KEY = '2dc3914abf35f0d9c92d97d8f8e42b43'
@@ -38,12 +37,19 @@ class BeetsPlugin(object):
     functionality by defining a subclass of BeetsPlugin and overriding
     the abstract methods defined here.
     """
-    def __init__(self):
-        """Perform one-time plugin setup. There is probably no reason to
-        override this method.
+    def __init__(self, name=None):
+        """Perform one-time plugin setup.
         """
         _add_media_fields(self.item_fields())
         self.import_stages = []
+        self.name = name or self.__module__.split('.')[-1]
+        self.config = beets.config[self.name]
+        if not self.template_funcs:
+            self.template_funcs = {}
+        if not self.template_fields:
+            self.template_fields = {}
+        if not self.album_template_fields:
+            self.album_template_fields = {}
 
     def commands(self):
         """Should return a list of beets.ui.Subcommand objects for
@@ -51,36 +57,35 @@ class BeetsPlugin(object):
         """
         return ()
 
+    def queries(self):
+        """Should return a dict mapping prefixes to Query subclasses.
+        """
+        return {}
+
     def track_distance(self, item, info):
-        """Should return a (distance, distance_max) pair to be added
-        to the distance value for every track comparison.
+        """Should return a Distance object to be added to the
+        distance for every track comparison.
         """
-        return 0.0, 0.0
+        return beets.autotag.hooks.Distance()
 
-    def album_distance(self, items, info):
-        """Should return a (distance, distance_max) pair to be added
-        to the distance value for every album-level comparison.
+    def album_distance(self, items, album_info, mapping):
+        """Should return a Distance object to be added to the
+        distance for every album-level comparison.
         """
-        return 0.0, 0.0
+        return beets.autotag.hooks.Distance()
 
-    def candidates(self, items):
+    def candidates(self, items, artist, album, va_likely):
         """Should return a sequence of AlbumInfo objects that match the
         album whose items are provided.
         """
         return ()
 
-    def item_candidates(self, item):
+    def item_candidates(self, item, artist, title):
         """Should return a sequence of TrackInfo objects that match the
         item provided.
         """
         return ()
 
-    def configure(self, config):
-        """This method is called with the ConfigParser object after
-        the CLI starts up.
-        """
-        pass
-
     def item_fields(self):
         """Returns field descriptors to be added to the MediaFile class,
         in the form of a dictionary whose keys are field names and whose
@@ -89,6 +94,19 @@ class BeetsPlugin(object):
         """
         return {}
 
+    def album_for_id(self, album_id):
+        """Return an AlbumInfo object or None if no matching release was
+        found.
+        """
+        return None
+
+    def track_for_id(self, track_id):
+        """Return a TrackInfo object or None if no matching release was
+        found.
+        """
+        return None
+
+
     listeners = None
 
     @classmethod
@@ -123,6 +141,7 @@ class BeetsPlugin(object):
 
     template_funcs = None
     template_fields = None
+    album_template_fields = None
 
     @classmethod
     def template_func(cls, name):
@@ -151,24 +170,30 @@ class BeetsPlugin(object):
             return func
         return helper
 
+_classes = set()
 def load_plugins(names=()):
     """Imports the modules for a sequence of plugin names. Each name
     must be the name of a Python module under the "beetsplug" namespace
     package in sys.path; the module indicated should contain the
-    BeetsPlugin subclasses desired. A default set of plugins is also
-    loaded.
+    BeetsPlugin subclasses desired.
     """
-    for name in itertools.chain(names, DEFAULT_PLUGINS):
+    for name in names:
         modname = '%s.%s' % (PLUGIN_NAMESPACE, name)
         try:
             try:
-                __import__(modname, None, None)
+                namespace = __import__(modname, None, None)
             except ImportError as exc:
                 # Again, this is hacky:
                 if exc.args[0].endswith(' ' + name):
                     log.warn('** plugin %s not found' % name)
                 else:
                     raise
+            else:
+                for obj in getattr(namespace, name).__dict__.values():
+                    if isinstance(obj, type) and issubclass(obj, BeetsPlugin) \
+                            and obj != BeetsPlugin and obj not in _classes:
+                        _classes.add(obj)
+
         except:
             log.warn('** error loading plugin %s' % name)
             log.warn(traceback.format_exc())
@@ -181,7 +206,7 @@ def find_plugins():
     """
     load_plugins()
     plugins = []
-    for cls in BeetsPlugin.__subclasses__():
+    for cls in _classes:
         # Only instantiate each plugin class once.
         if cls not in _instances:
             _instances[cls] = cls()
@@ -199,48 +224,68 @@ def commands():
         out += plugin.commands()
     return out
 
+def queries():
+    """Returns a dict mapping prefix strings to Query subclasses all loaded
+    plugins.
+    """
+    out = {}
+    for plugin in find_plugins():
+        out.update(plugin.queries())
+    return out
+
 def track_distance(item, info):
     """Gets the track distance calculated by all loaded plugins.
-    Returns a (distance, distance_max) pair.
+    Returns a Distance object.
     """
-    dist = 0.0
-    dist_max = 0.0
+    from beets.autotag.hooks import Distance
+    dist = Distance()
     for plugin in find_plugins():
-        d, dm = plugin.track_distance(item, info)
-        dist += d
-        dist_max += dm
-    return dist, dist_max
+        dist.update(plugin.track_distance(item, info))
+    return dist
 
-def album_distance(items, info):
+def album_distance(items, album_info, mapping):
     """Returns the album distance calculated by plugins."""
-    dist = 0.0
-    dist_max = 0.0
+    from beets.autotag.hooks import Distance
+    dist = Distance()
     for plugin in find_plugins():
-        d, dm = plugin.album_distance(items, info)
-        dist += d
-        dist_max += dm
-    return dist, dist_max
+        dist.update(plugin.album_distance(items, album_info, mapping))
+    return dist
 
-def candidates(items):
+def candidates(items, artist, album, va_likely):
     """Gets MusicBrainz candidates for an album from each plugin.
     """
     out = []
     for plugin in find_plugins():
-        out.extend(plugin.candidates(items))
+        out.extend(plugin.candidates(items, artist, album, va_likely))
     return out
 
-def item_candidates(item):
+def item_candidates(item, artist, title):
     """Gets MusicBrainz candidates for an item from the plugins.
     """
     out = []
     for plugin in find_plugins():
-        out.extend(plugin.item_candidates(item))
+        out.extend(plugin.item_candidates(item, artist, title))
     return out
 
-def configure(config):
-    """Sends the configuration object to each plugin."""
+def album_for_id(album_id):
+    """Get AlbumInfo objects for a given ID string.
+    """
+    out = []
     for plugin in find_plugins():
-        plugin.configure(config)
+        res = plugin.album_for_id(album_id)
+        if res:
+            out.append(res)
+    return out
+
+def track_for_id(track_id):
+    """Get TrackInfo objects for a given ID string.
+    """
+    out = []
+    for plugin in find_plugins():
+        res = plugin.track_for_id(track_id)
+        if res:
+            out.append(res)
+    return out
 
 def template_funcs():
     """Get all the template functions declared by plugins as a
@@ -252,17 +297,6 @@ def template_funcs():
             funcs.update(plugin.template_funcs)
     return funcs
 
-def template_values(item):
-    """Get all the template values computed for a given Item by
-    registered field computations.
-    """
-    values = {}
-    for plugin in find_plugins():
-        if plugin.template_fields:
-            for name, func in plugin.template_fields.iteritems():
-                values[name] = unicode(func(item))
-    return values
-
 def _add_media_fields(fields):
     """Adds a {name: descriptor} dictionary of fields to the MediaFile
     class. Called during the plugin initialization.
@@ -279,6 +313,28 @@ def import_stages():
     return stages
 
 
+# New-style (lazy) plugin-provided fields.
+
+def item_field_getters():
+    """Get a dictionary mapping field names to unary functions that
+    compute the field's value.
+    """
+    funcs = {}
+    for plugin in find_plugins():
+        if plugin.template_fields:
+            funcs.update(plugin.template_fields)
+    return funcs
+
+def album_field_getters():
+    """As above, for album fields.
+    """
+    funcs = {}
+    for plugin in find_plugins():
+        if plugin.album_template_fields:
+            funcs.update(plugin.album_template_fields)
+    return funcs
+
+
 # Event dispatch.
 
 def event_handlers():
@@ -297,10 +353,7 @@ def send(event, **arguments):
     name of  the event to send, all other named arguments go to the
     event handler(s).
 
-    Returns the number of handlers called.
+    Returns a list of return values from the handlers.
     """
     log.debug('Sending event: %s' % event)
-    handlers = event_handlers()[event]
-    for handler in handlers:
-        handler(**arguments)
-    return len(handlers)
+    return [handler(**arguments) for handler in event_handlers()[event]]

beets/ui/__init__.py

@@ -1,5 +1,5 @@
 # This file is part of beets.
-# Copyright 2012, Adrian Sampson.
+# Copyright 2014, Adrian Sampson.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -18,25 +18,29 @@ CLI commands are implemented in the ui.commands module.
 """
 from __future__ import print_function
 
-import os
 import locale
 import optparse
 import textwrap
-import ConfigParser
 import sys
 from difflib import SequenceMatcher
 import logging
 import sqlite3
 import errno
 import re
-import codecs
+import struct
+import traceback
+
+from beets import library
+from beets import plugins
+from beets import util
+from beets.util.functemplate import Template
+from beets import config
+from beets.util import confit
+from beets.autotag import mb
 
-from lib.beets import library
-from lib.beets import plugins
-from lib.beets import util
-from lib.beets.util.functemplate import Template
 
 # On Windows platforms, use colorama to support "ANSI" terminal colors.
+
 if sys.platform == 'win32':
     try:
         import colorama
@@ -46,40 +50,36 @@ if sys.platform == 'win32':
         colorama.init()
 
 
+
 # Constants.
-CONFIG_PATH_VAR = 'BEETSCONFIG'
-DEFAULT_CONFIG_FILENAME_UNIX = '.beetsconfig'
-DEFAULT_CONFIG_FILENAME_WINDOWS = 'beetsconfig.ini'
-DEFAULT_LIBRARY_FILENAME_UNIX = '.beetsmusic.blb'
-DEFAULT_LIBRARY_FILENAME_WINDOWS = 'beetsmusic.blb'
-DEFAULT_DIRECTORY_NAME = 'Music'
-WINDOWS_BASEDIR = os.environ.get('APPDATA') or '~'
+
+
 PF_KEY_QUERIES = {
     'comp': 'comp:true',
     'singleton': 'singleton:true',
 }
-DEFAULT_PATH_FORMATS = [
-  (library.PF_KEY_DEFAULT,
-   Template('$albumartist/$album%aunique{}/$track $title')),
-  (PF_KEY_QUERIES['singleton'],
-   Template('Non-Album/$artist/$title')),
-  (PF_KEY_QUERIES['comp'],
-   Template('Compilations/$album%aunique{}/$track $title')),
-]
-DEFAULT_ART_FILENAME = 'cover'
-DEFAULT_TIMEOUT = 5.0
-NULL_REPLACE = '<strip>'
 
 # UI exception. Commands should throw this in order to display
 # nonrecoverable errors to the user.
 class UserError(Exception):
     pass
 
+# Main logger.
+log = logging.getLogger('beets')
+
+
 
 # Utilities.
 
+
 def _encoding():
-    """Tries to guess the encoding uses by the terminal."""
+    """Tries to guess the encoding used by the terminal."""
+    # Configured override?
+    encoding = config['terminal_encoding'].get()
+    if encoding:
+        return encoding
+
+    # Determine from locale settings.
     try:
         return locale.getdefaultlocale()[1] or 'utf8'
     except ValueError:
@@ -87,12 +87,14 @@ def _encoding():
         # failing entirely for no good reason, assume UTF-8.
         return 'utf8'
 
+
 def decargs(arglist):
     """Given a list of command-line argument bytestrings, attempts to
     decode them to Unicode strings.
     """
     return [s.decode(_encoding()) for s in arglist]
 
+
 def print_(*strings):
     """Like print, but rather than raising an error when a character
     is not in the terminal's encoding's character set, just silently
@@ -109,6 +111,7 @@ def print_(*strings):
         txt = txt.encode(_encoding(), 'replace')
     print(txt)
 
+
 def input_(prompt=None):
     """Like `raw_input`, but decodes the result to a Unicode string.
     Raises a UserError if stdin is not available. The prompt is sent to
@@ -128,10 +131,11 @@ def input_(prompt=None):
     except EOFError:
         raise UserError('stdin stream ended while input required')
 
-    return resp.decode(sys.stdin.encoding, 'ignore')
+    return resp.decode(sys.stdin.encoding or 'utf8', 'ignore')
+
 
 def input_options(options, require=False, prompt=None, fallback_prompt=None,
-                  numrange=None, default=None, color=False, max_width=72):
+                  numrange=None, default=None, max_width=72):
     """Prompts a user for input. The sequence of `options` defines the
     choices the user has. A single-letter shortcut is inferred for each
     option; the user's choice is returned as that single, lower-case
@@ -139,10 +143,9 @@ def input_options(options, require=False, prompt=None, fallback_prompt=None,
     a particular shortcut is desired; in that case, only that letter
     should be capitalized.
 
-    By default, the first option is the default. If `require` is
-    provided, then there is no default. `default` can be provided to
-    override this. The prompt and fallback prompt are also inferred but
-    can be overridden.
+    By default, the first option is the default. `default` can be provided to
+    override this. If `require` is provided, then there is no default. The
+    prompt and fallback prompt are also inferred but can be overridden.
 
     If numrange is provided, it is a pair of `(high, low)` (both ints)
     indicating that, in addition to `options`, the user may enter an
@@ -178,9 +181,9 @@ def input_options(options, require=False, prompt=None, fallback_prompt=None,
         index = option.index(found_letter)
 
         # Mark the option's shortcut letter for display.
-        if (default is None and not numrange and first) \
-           or (isinstance(default, basestring) and
-               found_letter.lower() == default.lower()):
+        if not require and ((default is None and not numrange and first) or
+                (isinstance(default, basestring) and
+                 found_letter.lower() == default.lower())):
             # The first option is the default; mark it.
             show_letter = '[%s]' % found_letter.upper()
             is_default = True
@@ -188,24 +191,23 @@ def input_options(options, require=False, prompt=None, fallback_prompt=None,
             show_letter = found_letter.upper()
             is_default = False
 
-        # Possibly colorize the letter shortcut.
-        if color:
-            color = 'turquoise' if is_default else 'blue'
-            show_letter = colorize(color, show_letter)
+        # Colorize the letter shortcut.
+        show_letter = colorize('turquoise' if is_default else 'blue',
+                               show_letter)
 
         # Insert the highlighted letter back into the word.
         capitalized.append(
-            option[:index] + show_letter + option[index+1:]
+            option[:index] + show_letter + option[index + 1:]
         )
         display_letters.append(found_letter.upper())
 
         first = False
 
     # The default is just the first option if unspecified.
-    if default is None:
-        if require:
-            default = None
-        elif numrange:
+    if require:
+        default = None
+    elif default is None:
+        if numrange:
             default = numrange[0]
         else:
             default = display_letters[0].lower()
@@ -217,8 +219,7 @@ def input_options(options, require=False, prompt=None, fallback_prompt=None,
         if numrange:
             if isinstance(default, int):
                 default_name = str(default)
-                if color:
-                    default_name = colorize('turquoise', default_name)
+                default_name = colorize('turquoise', default_name)
                 tmpl = '# selection (default %s)'
                 prompt_parts.append(tmpl % default_name)
                 prompt_part_lengths.append(len(tmpl % str(default)))
@@ -291,35 +292,16 @@ def input_options(options, require=False, prompt=None, fallback_prompt=None,
         # Prompt for new input.
         resp = input_(fallback_prompt)
 
-def input_yn(prompt, require=False, color=False):
+
+def input_yn(prompt, require=False):
     """Prompts the user for a "yes" or "no" response. The default is
     "yes" unless `require` is `True`, in which case there is no default.
     """
     sel = input_options(
-        ('y', 'n'), require, prompt, 'Enter Y or N:', color=color
+        ('y', 'n'), require, prompt, 'Enter Y or N:'
     )
     return sel == 'y'
 
-def config_val(config, section, name, default, vtype=None):
-    """Queries the configuration file for a value (given by the section
-    and name). If no value is present, returns default.  vtype
-    optionally specifies the return type (although only ``bool`` and
-    ``list`` are supported for now).
-    """
-    if not config.has_section(section):
-        config.add_section(section)
-
-    try:
-        if vtype is bool:
-            return config.getboolean(section, name)
-        elif vtype is list:
-            # Whitespace-separated strings.
-            strval = config.get(section, name, True)
-            return strval.split()
-        else:
-            return config.get(section, name, True)
-    except ConfigParser.NoOptionError:
-        return default
 
 def human_bytes(size):
     """Formats size, a number of bytes, in a human-readable way."""
@@ -330,6 +312,7 @@ def human_bytes(size):
         size /= 1024.0
     return "big"
 
+
 def human_seconds(interval):
     """Formats interval, a number of seconds, as a human-readable time
     interval using English words.
@@ -343,9 +326,9 @@ def human_seconds(interval):
         (52, 'year'),
         (10, 'decade'),
     ]
-    for i in range(len(units)-1):
+    for i in range(len(units) - 1):
         increment, suffix = units[i]
-        next_increment, _ = units[i+1]
+        next_increment, _ = units[i + 1]
         interval /= float(increment)
         if interval < next_increment:
             break
@@ -356,6 +339,7 @@ def human_seconds(interval):
 
     return "%3.1f %ss" % (interval, suffix)
 
+
 def human_seconds_short(interval):
     """Formats a number of seconds as a short human-readable M:SS
     string.
@@ -363,6 +347,7 @@ def human_seconds_short(interval):
     interval = int(interval)
     return u'%i:%02i' % (interval // 60, interval % 60)
 
+
 # ANSI terminal colorization code heavily inspired by pygments:
 # http://dev.pocoo.org/hg/pygments-main/file/b2deea5b5030/pygments/console.py
 # (pygments is by Tim Hatch, Armin Ronacher, et al.)
@@ -372,7 +357,7 @@ DARK_COLORS  = ["black", "darkred", "darkgreen", "brown", "darkblue",
 LIGHT_COLORS = ["darkgray", "red", "green", "yellow", "blue",
                 "fuchsia", "turquoise", "white"]
 RESET_COLOR = COLOR_ESCAPE + "39;49;00m"
-def colorize(color, text):
+def _colorize(color, text):
     """Returns a string that prints the given text in the given color
     in a terminal that is ANSI color-aware. The color must be something
     in DARK_COLORS or LIGHT_COLORS.
@@ -385,7 +370,18 @@ def colorize(color, text):
         raise ValueError('no such color %s', color)
     return escape + text + RESET_COLOR
 
-def colordiff(a, b, highlight='red'):
+
+def colorize(color, text):
+    """Colorize text if colored output is enabled. (Like _colorize but
+    conditional.)
+    """
+    if config['color']:
+        return _colorize(color, text)
+    else:
+        return text
+
+
+def _colordiff(a, b, highlight='red', minor_highlight='lightgray'):
     """Given two values, return the same pair of strings except with
     their differences highlighted in the specified color. Strings are
     highlighted intelligently to show differences; other values are
@@ -400,6 +396,11 @@ def colordiff(a, b, highlight='red'):
         else:
             return colorize(highlight, a), colorize(highlight, b)
 
+    if isinstance(a, bytes) or isinstance(b, bytes):
+        # A path field.
+        a = util.displayable_path(a)
+        b = util.displayable_path(b)
+
     a_out = []
     b_out = []
 
@@ -416,100 +417,220 @@ def colordiff(a, b, highlight='red'):
             # Left only.
             a_out.append(colorize(highlight, a[a_start:a_end]))
         elif op == 'replace':
-            # Right and left differ.
-            a_out.append(colorize(highlight, a[a_start:a_end]))
-            b_out.append(colorize(highlight, b[b_start:b_end]))
+            # Right and left differ. Colorise with second highlight if
+            # it's just a case change.
+            if a[a_start:a_end].lower() != b[b_start:b_end].lower():
+                color = highlight
+            else:
+                color = minor_highlight
+            a_out.append(colorize(color, a[a_start:a_end]))
+            b_out.append(colorize(color, b[b_start:b_end]))
         else:
             assert(False)
 
     return u''.join(a_out), u''.join(b_out)
 
-def default_paths(pathmod=None):
-    """Produces the appropriate default config, library, and directory
-    paths for the current system. On Unix, this is always in ~. On
-    Windows, tries ~ first and then $APPDATA for the config and library
-    files (for backwards compatibility).
+
+def colordiff(a, b, highlight='red'):
+    """Colorize differences between two values if color is enabled.
+    (Like _colordiff but conditional.)
     """
-    pathmod = pathmod or os.path
-    windows = pathmod.__name__ == 'ntpath'
-    if windows:
-        windata = os.environ.get('APPDATA') or '~'
-
-    # Shorthand for joining paths.
-    def exp(*vals):
-        return pathmod.expanduser(pathmod.join(*vals))
-
-    config = exp('~', DEFAULT_CONFIG_FILENAME_UNIX)
-    if windows and not pathmod.exists(config):
-        config = exp(windata, DEFAULT_CONFIG_FILENAME_WINDOWS)
-
-    libpath = exp('~', DEFAULT_LIBRARY_FILENAME_UNIX)
-    if windows and not pathmod.exists(libpath):
-        libpath = exp(windata, DEFAULT_LIBRARY_FILENAME_WINDOWS)
-
-    libdir = exp('~', DEFAULT_DIRECTORY_NAME)
-
-    return config, libpath, libdir
-
-def _get_replacements(config):
-    """Given a ConfigParser, get the list of replacement pairs. If no
-    replacements are specified, returns None. Otherwise, returns a list
-    of (compiled regex, replacement string) pairs.
-    """
-    repl_string = config_val(config, 'beets', 'replace', None)
-    if not repl_string:
-        return
-    if not isinstance(repl_string, unicode):
-        repl_string = repl_string.decode('utf8')
-
-    parts = repl_string.strip().split()
-    if not parts:
-        return
-    if len(parts) % 2 != 0:
-        # Must have an even number of parts.
-        raise UserError(u'"replace" config value must consist of'
-                        u' pattern/replacement pairs')
-
-    out = []
-    for index in xrange(0, len(parts), 2):
-        pattern = parts[index]
-        replacement = parts[index+1]
-        if replacement.lower() == NULL_REPLACE:
-            replacement = ''
-        out.append((re.compile(pattern), replacement))
-    return out
-
-def _get_path_formats(config):
-    """Returns a list of path formats (query/template pairs); reflecting
-    the config's specified path formats.
-    """
-    legacy_path_format = config_val(config, 'beets', 'path_format', None)
-    if legacy_path_format:
-        # Old path formats override the default values.
-        path_formats = [(library.PF_KEY_DEFAULT,
-                         Template(legacy_path_format))]
+    if config['color']:
+        return _colordiff(a, b, highlight)
     else:
-        # If no legacy path format, use the defaults instead.
-        path_formats = DEFAULT_PATH_FORMATS
+        return unicode(a), unicode(b)
 
-    if config.has_section('paths'):
-        custom_path_formats = []
-        for key, value in config.items('paths', True):
-            if key in PF_KEY_QUERIES:
-                # Special values that indicate simple queries.
-                key = PF_KEY_QUERIES[key]
-            elif key != library.PF_KEY_DEFAULT:
-                # For non-special keys (literal queries), the _
-                # character denotes a :.
-                key = key.replace('_', ':')
-            custom_path_formats.append((key, Template(value)))
-        path_formats = custom_path_formats + path_formats
 
+def color_diff_suffix(a, b, highlight='red'):
+    """Colorize the differing suffix between two strings."""
+    a, b = unicode(a), unicode(b)
+    if not config['color']:
+        return a, b
+
+    # Fast path.
+    if a == b:
+        return a, b
+
+    # Find the longest common prefix.
+    first_diff = None
+    for i in range(min(len(a), len(b))):
+        if a[i] != b[i]:
+            first_diff = i
+            break
+    else:
+        first_diff = min(len(a), len(b))
+
+    # Colorize from the first difference on.
+    return a[:first_diff] + colorize(highlight, a[first_diff:]), \
+           b[:first_diff] + colorize(highlight, b[first_diff:])
+
+
+def get_path_formats(subview=None):
+    """Get the configuration's path formats as a list of query/template
+    pairs.
+    """
+    path_formats = []
+    subview = subview or config['paths']
+    for query, view in subview.items():
+        query = PF_KEY_QUERIES.get(query, query)  # Expand common queries.
+        path_formats.append((query, Template(view.get(unicode))))
     return path_formats
 
 
+def get_replacements():
+    """Confit validation function that reads regex/string pairs.
+    """
+    replacements = []
+    for pattern, repl in config['replace'].get(dict).items():
+        repl = repl or ''
+        try:
+            replacements.append((re.compile(pattern), repl))
+        except re.error:
+            raise UserError(
+                u'malformed regular expression in replace: {0}'.format(
+                    pattern
+                )
+            )
+    return replacements
+
+
+def get_plugin_paths():
+    """Get the list of search paths for plugins from the config file.
+    The value for "pluginpath" may be a single string or a list of
+    strings.
+    """
+    pluginpaths = config['pluginpath'].get()
+    if isinstance(pluginpaths, basestring):
+        pluginpaths = [pluginpaths]
+    if not isinstance(pluginpaths, list):
+        raise confit.ConfigTypeError(
+            u'pluginpath must be string or a list of strings'
+        )
+    return map(util.normpath, pluginpaths)
+
+
+def _pick_format(album, fmt=None):
+    """Pick a format string for printing Album or Item objects,
+    falling back to config options and defaults.
+    """
+    if fmt:
+        return fmt
+    if album:
+        return config['list_format_album'].get(unicode)
+    else:
+        return config['list_format_item'].get(unicode)
+
+
+def print_obj(obj, lib, fmt=None):
+    """Print an Album or Item object. If `fmt` is specified, use that
+    format string. Otherwise, use the configured template.
+    """
+    album = isinstance(obj, library.Album)
+    fmt = _pick_format(album, fmt)
+    if isinstance(fmt, Template):
+        template = fmt
+    else:
+        template = Template(fmt)
+    print_(obj.evaluate_template(template))
+
+
+def term_width():
+    """Get the width (columns) of the terminal."""
+    fallback = config['ui']['terminal_width'].get(int)
+
+    # The fcntl and termios modules are not available on non-Unix
+    # platforms, so we fall back to a constant.
+    try:
+        import fcntl
+        import termios
+    except ImportError:
+        return fallback
+
+    try:
+        buf = fcntl.ioctl(0, termios.TIOCGWINSZ, ' ' * 4)
+    except IOError:
+        return fallback
+    try:
+        height, width = struct.unpack('hh', buf)
+    except struct.error:
+        return fallback
+    return width
+
+
+FLOAT_EPSILON = 0.01
+def _field_diff(field, old, new):
+    """Given two Model objects, format their values for `field` and
+    highlight changes among them. Return a human-readable string. If the
+    value has not changed, return None instead.
+    """
+    oldval = old.get(field)
+    newval = new.get(field)
+
+    # If no change, abort.
+    if isinstance(oldval, float) and isinstance(newval, float) and \
+            abs(oldval - newval) < FLOAT_EPSILON:
+        return None
+    elif oldval == newval:
+        return None
+
+    # Get formatted values for output.
+    oldstr = old._get_formatted(field)
+    newstr = new._get_formatted(field)
+
+    # For strings, highlight changes. For others, colorize the whole
+    # thing.
+    if isinstance(oldval, basestring):
+        oldstr, newstr = colordiff(oldval, newval)
+    else:
+        oldstr, newstr = colorize('red', oldstr), colorize('red', newstr)
+
+    return u'{0} -> {1}'.format(oldstr, newstr)
+
+
+def show_model_changes(new, old=None, fields=None, always=False):
+    """Given a Model object, print a list of changes from its pristine
+    version stored in the database. Return a boolean indicating whether
+    any changes were found.
+
+    `old` may be the "original" object to avoid using the pristine
+    version from the database. `fields` may be a list of fields to
+    restrict the detection to. `always` indicates whether the object is
+    always identified, regardless of whether any changes are present.
+    """
+    old = old or new._db._get(type(new), new.id)
+
+    # Build up lines showing changed fields.
+    changes = []
+    for field in old:
+        # Subset of the fields. Never show mtime.
+        if field == 'mtime' or (fields and field not in fields):
+            continue
+
+        # Detect and show difference for this field.
+        line = _field_diff(field, old, new)
+        if line:
+            changes.append(u'  {0}: {1}'.format(field, line))
+
+    # New fields.
+    for field in set(new) - set(old):
+        changes.append(u'  {0}: {1}'.format(
+            field,
+            colorize('red', new._get_formatted(field))
+        ))
+
+    # Print changes.
+    if changes or always:
+        print_obj(old, old._db)
+    if changes:
+        print_(u'\n'.join(changes))
+
+    return bool(changes)
+
+
+
 # Subcommand parsing infrastructure.
 
+
 # This is a fairly generic subcommand parser for optparse. It is
 # maintained externally here:
 # http://gist.github.com/462717
@@ -520,7 +641,7 @@ class Subcommand(object):
     """A subcommand of a root command-line application that may be
     invoked by a SubcommandOptionParser.
     """
-    def __init__(self, name, parser=None, help='', aliases=()):
+    def __init__(self, name, parser=None, help='', aliases=(), hide=False):
         """Creates a new subcommand. name is the primary way to invoke
         the subcommand; aliases are alternate names. parser is an
         OptionParser responsible for parsing the subcommand's options.
@@ -531,6 +652,7 @@ class Subcommand(object):
         self.parser = parser or optparse.OptionParser()
         self.aliases = aliases
         self.help = help
+        self.hide = hide
 
 class SubcommandsOptionParser(optparse.OptionParser):
     """A variant of OptionParser that parses subcommands and their
@@ -588,7 +710,8 @@ class SubcommandsOptionParser(optparse.OptionParser):
         # Also determine the help position.
         disp_names = []
         help_position = 0
-        for subcommand in self.subcommands:
+        subcommands = [c for c in self.subcommands if not c.hide]
+        for subcommand in subcommands:
             name = subcommand.name
             if subcommand.aliases:
                 name += ' (%s)' % ', '.join(subcommand.aliases)
@@ -600,7 +723,7 @@ class SubcommandsOptionParser(optparse.OptionParser):
                 help_position = max(help_position, proposed_help_position)
 
         # Add each subcommand to the output.
-        for subcommand, name in zip(self.subcommands, disp_names):
+        for subcommand, name in zip(subcommands, disp_names):
             # Lifted directly from optparse.py.
             name_width = help_position - formatter.current_indent - 2
             if len(name) > name_width:
@@ -641,7 +764,14 @@ class SubcommandsOptionParser(optparse.OptionParser):
         - subargs: the positional arguments passed to the subcommand
         """
         options, args = optparse.OptionParser.parse_args(self, a, v)
+        subcommand, suboptions, subargs = self._parse_sub(args)
+        return options, subcommand, suboptions, subargs
 
+    def _parse_sub(self, args):
+        """Given the `args` left unused by a typical OptionParser
+        `parse_args`, return the invoked subcommand, the subcommand
+        options, and the subcommand arguments.
+        """
         if not args:
             # No command given.
             self.print_help()
@@ -668,108 +798,183 @@ class SubcommandsOptionParser(optparse.OptionParser):
                 self.print_help()
                 self.exit()
 
-        return options, subcommand, suboptions, subargs
+        return subcommand, suboptions, subargs
 
 
-# The root parser and its main function.
+optparse.Option.ALWAYS_TYPED_ACTIONS += ('callback',)
+def vararg_callback(option, opt_str, value, parser):
+    """Callback for an option with variable arguments.
+    Manually collect arguments right of a callback-action
+    option (ie. with action="callback"), and add the resulting
+    list to the destination var.
 
-def main(args=None, configfh=None):
-    """Run the main command-line interface for beets."""
-    # Get the default subcommands.
-    from lib.beets.ui.commands import default_commands
+    Usage:
+    parser.add_option("-c", "--callback", dest="vararg_attr",
+                      action="callback", callback=vararg_callback)
 
-    # Get default file paths.
-    default_config, default_libpath, default_dir = default_paths()
+    Details:
+    http://docs.python.org/2/library/optparse.html#callback-example-6-variable
+    -arguments
+    """
+    value = [value]
 
-    # Read defaults from config file.
-    config = ConfigParser.SafeConfigParser()
-    if configfh:
-        configpath = None
-    elif CONFIG_PATH_VAR in os.environ:
-        configpath = os.path.expanduser(os.environ[CONFIG_PATH_VAR])
-    else:
-        configpath = default_config
-    if configpath:
-        configpath = util.syspath(configpath)
-        if os.path.exists(util.syspath(configpath)):
-            configfh = codecs.open(configpath, 'r', encoding='utf-8')
-        else:
-            configfh = None
-    if configfh:
-        config.readfp(configfh)
+    def floatable(str):
+        try:
+            float(str)
+            return True
+        except ValueError:
+            return False
 
+    for arg in parser.rargs:
+        # stop on --foo like options
+        if arg[:2] == "--" and len(arg) > 2:
+            break
+        # stop on -a, but not on -3 or -3.0
+        if arg[:1] == "-" and len(arg) > 1 and not floatable(arg):
+            break
+        value.append(arg)
+
+    del parser.rargs[:len(value) - 1]
+    setattr(parser.values, option.dest, value)
+
+
+
+# The main entry point and bootstrapping.
+
+
+def _load_plugins():
+    """Load the plugins specified in the configuration.
+    """
     # Add plugin paths.
-    plugpaths = config_val(config, 'beets', 'pluginpath', '')
-    for plugpath in plugpaths.split(':'):
-        sys.path.append(os.path.expanduser(plugpath))
+    import beetsplug
+    beetsplug.__path__ = get_plugin_paths() + beetsplug.__path__
+
+    # For backwards compatibility.
+    sys.path += get_plugin_paths()
+
     # Load requested plugins.
-    plugnames = config_val(config, 'beets', 'plugins', '')
-    plugins.load_plugins(plugnames.split())
+    plugins.load_plugins(config['plugins'].as_str_seq())
     plugins.send("pluginload")
-    plugins.configure(config)
+
+
+def _configure(args):
+    """Parse the command line, load configuration files (including
+    loading any indicated plugins), and return the invoked subcomand,
+    the subcommand options, and the subcommand arguments.
+    """
+    # Temporary: Migrate from 1.0-style configuration.
+    from beets.ui import migrate
+    migrate.automigrate()
+
+    # Get the default subcommands.
+    from beets.ui.commands import default_commands
 
     # Construct the root parser.
     commands = list(default_commands)
-    commands += plugins.commands()
+    commands.append(migrate.migrate_cmd)  # Temporary.
     parser = SubcommandsOptionParser(subcommands=commands)
-    parser.add_option('-l', '--library', dest='libpath',
+    parser.add_option('-l', '--library', dest='library',
                       help='library database file to use')
     parser.add_option('-d', '--directory', dest='directory',
                       help="destination music directory")
     parser.add_option('-v', '--verbose', dest='verbose', action='store_true',
                       help='print debugging information')
+    parser.add_option('-c', '--config', dest='config',
+                      help='path to configuration file')
 
     # Parse the command-line!
-    options, subcommand, suboptions, subargs = parser.parse_args(args)
+    options, args = optparse.OptionParser.parse_args(parser, args)
+
+    # Add any additional config files specified with --config. This
+    # special handling lets specified plugins get loaded before we
+    # finish parsing the command line.
+    if getattr(options, 'config', None) is not None:
+        config_path = options.config
+        del options.config
+        config.set_file(config_path)
+    config.set_args(options)
+
+    # Now add the plugin commands to the parser.
+    _load_plugins()
+    for cmd in plugins.commands():
+        parser.add_subcommand(cmd)
+
+    # Parse the remainder of the command line with loaded plugins.
+    return parser._parse_sub(args)
+
+
+def _raw_main(args):
+    """A helper function for `main` without top-level exception
+    handling.
+    """
+    subcommand, suboptions, subargs = _configure(args)
 
     # Open library file.
-    libpath = options.libpath or \
-        config_val(config, 'beets', 'library', default_libpath)
-    directory = options.directory or \
-        config_val(config, 'beets', 'directory', default_dir)
-    path_formats = _get_path_formats(config)
-    art_filename = \
-        config_val(config, 'beets', 'art_filename', DEFAULT_ART_FILENAME)
-    lib_timeout = config_val(config, 'beets', 'timeout', DEFAULT_TIMEOUT)
-    replacements = _get_replacements(config)
+    dbpath = config['library'].as_filename()
     try:
-        lib_timeout = float(lib_timeout)
-    except ValueError:
-        lib_timeout = DEFAULT_TIMEOUT
-    db_path = os.path.expanduser(libpath)
-    try:
-        lib = library.Library(db_path,
-                              directory,
-                              path_formats,
-                              art_filename,
-                              lib_timeout,
-                              replacements)
+        lib = library.Library(
+            dbpath,
+            config['directory'].as_filename(),
+            get_path_formats(),
+            get_replacements(),
+        )
     except sqlite3.OperationalError:
-        raise UserError("database file %s could not be opened" % db_path)
+        raise UserError(u"database file {0} could not be opened".format(
+            util.displayable_path(dbpath)
+        ))
     plugins.send("library_opened", lib=lib)
 
     # Configure the logger.
-    log = logging.getLogger('beets')
-    if options.verbose:
+    if config['verbose'].get(bool):
         log.setLevel(logging.DEBUG)
     else:
         log.setLevel(logging.INFO)
-    log.debug(u'config file: %s' % util.displayable_path(configpath))
-    log.debug(u'library database: %s' % util.displayable_path(lib.path))
-    log.debug(u'library directory: %s' % util.displayable_path(lib.directory))
+    log.debug(u'data directory: {0}\n'
+              u'library database: {1}\n'
+              u'library directory: {2}'
+        .format(
+            util.displayable_path(config.config_dir()),
+            util.displayable_path(lib.path),
+            util.displayable_path(lib.directory),
+        )
+    )
+
+    # Configure the MusicBrainz API.
+    mb.configure()
 
     # Invoke the subcommand.
+    subcommand.func(lib, suboptions, subargs)
+    plugins.send('cli_exit', lib=lib)
+
+
+def main(args=None):
+    """Run the main command-line interface for beets. Includes top-level
+    exception handlers that print friendly error messages.
+    """
     try:
-        subcommand.func(lib, config, suboptions, subargs)
+        _raw_main(args)
     except UserError as exc:
         message = exc.args[0] if exc.args else None
-        subcommand.parser.error(message)
+        log.error(u'error: {0}'.format(message))
+        sys.exit(1)
     except util.HumanReadableException as exc:
         exc.log(log)
         sys.exit(1)
+    except library.FileOperationError as exc:
+        # These errors have reasonable human-readable descriptions, but
+        # we still want to log their tracebacks for debugging.
+        log.debug(traceback.format_exc())
+        log.error(exc)
+        sys.exit(1)
+    except confit.ConfigError as exc:
+        log.error(u'configuration error: {0}'.format(exc))
+        sys.exit(1)
     except IOError as exc:
         if exc.errno == errno.EPIPE:
             # "Broken pipe". End silently.
             pass
         else:
             raise
+    except KeyboardInterrupt:
+        # Silently ignore ^C except in verbose mode.
+        log.debug(traceback.format_exc())

beets/ui/completion_base.sh

@@ -0,0 +1,162 @@
+# This file is part of beets.
+# Copyright (c) 2014, Thomas Scholtes.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+
+
+
+# Completion for the `beet` command
+# =================================
+#
+# Load this script to complete beets subcommands, options, and
+# queries.
+#
+# If a beets command is found on the command line it completes filenames and
+# the subcommand's options. Otherwise it will complete global options and
+# subcommands. If the previous option on the command line expects an argument,
+# it also completes filenames or directories.  Options are only
+# completed if '-' has already been typed on the command line.
+#
+# Note that completion of plugin commands only works for those plugins
+# that were enabled when running `beet completion`. It does not check
+# plugins dynamically
+#
+# Currently, only Bash 3.2 and newer is supported and the
+# `bash-completion` package is requied.
+#
+# TODO
+# ----
+#
+# * There are some issues with arguments that are quoted on the command line.
+#
+# * Complete arguments for the `--format` option by expanding field variables.
+#
+#     beet ls -f "$tit[TAB]
+#     beet ls -f "$title
+#
+# * Support long options with `=`, e.g. `--config=file`. Debian's bash
+#   completion package can handle this.
+#
+
+
+# Determines the beets subcommand and dispatches the completion
+# accordingly.
+_beet_dispatch() {
+  local cur prev cmd=
+
+  COMPREPLY=()
+  _get_comp_words_by_ref -n : cur prev
+
+  # Look for the beets subcommand
+  local arg
+  for (( i=1; i < COMP_CWORD; i++ )); do
+      arg="${COMP_WORDS[i]}"
+      if _list_include_item "${opts___global}" $arg; then
+        ((i++))
+      elif [[ "$arg" != -* ]]; then
+        cmd="$arg"
+        break
+      fi
+  done
+
+  # Replace command shortcuts
+  if [[ -n $cmd ]] && _list_include_item "$aliases" "$cmd"; then
+    eval "cmd=\$alias__$cmd"
+  fi
+
+  case $cmd in
+    help)
+      COMPREPLY+=( $(compgen -W "$commands" -- $cur) )
+      ;;
+    list|remove|move|update|write|stats)
+      _beet_complete_query
+      ;;
+    "")
+      _beet_complete_global
+      ;;
+    *)
+      _beet_complete
+      ;;
+  esac
+}
+
+
+# Adds option and file completion to COMPREPLY for the subcommand $cmd
+_beet_complete() {
+  if [[ $cur == -* ]]; then
+    local opts flags completions
+    eval "opts=\$opts__$cmd"
+    eval "flags=\$flags__$cmd"
+    completions="${flags___common} ${opts} ${flags}"
+    COMPREPLY+=( $(compgen -W "$completions"  -- $cur) )
+  else
+    _filedir
+  fi
+}
+
+
+# Add global options and subcommands to the completion
+_beet_complete_global() {
+  case $prev in
+    -h|--help)
+      # Complete commands
+      COMPREPLY+=( $(compgen -W "$commands" -- $cur) )
+      return
+      ;;
+    -l|--library|-c|--config)
+      # Filename completion
+      _filedir
+      return
+      ;;
+    -d|--directory)
+      # Directory completion
+      _filedir -d
+      return
+      ;;
+  esac
+
+  if [[ $cur == -* ]]; then
+    local completions="$opts___global $flags___global"
+    COMPREPLY+=( $(compgen -W "$completions" -- $cur) )
+  elif [[ -n $cur ]] && _list_include_item "$aliases" "$cur"; then
+    local cmd
+    eval "cmd=\$alias__$cur"
+    COMPREPLY+=( "$cmd" )
+  else
+    COMPREPLY+=( $(compgen -W "$commands" -- $cur) )
+  fi
+}
+
+_beet_complete_query() {
+  local opts
+  eval "opts=\$opts__$cmd"
+
+  if [[ $cur == -* ]] || _list_include_item "$opts" "$prev"; then
+    _beet_complete
+  elif [[ $cur != \'* && $cur != \"* &&
+          $cur != *:* ]]; then
+    # Do not complete quoted queries or those who already have a field
+    # set.
+    compopt -o nospace
+    COMPREPLY+=( $(compgen -S : -W "$fields" -- $cur) )
+    return 0
+  fi
+}
+
+# Returns true if the space separated list $1 includes $2
+_list_include_item() {
+  [[ " $1 " == *[[:space:]]$2[[:space:]]* ]]
+}
+
+# This is where beets dynamically adds the _beet function. This
+# function sets the variables $flags, $opts, $commands, and $aliases.
+complete -o filenames -F _beet beet

beets/ui/migrate.py

@@ -0,0 +1,401 @@
+# This file is part of beets.
+# Copyright 2013, Adrian Sampson.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+
+"""Conversion from legacy (pre-1.1) configuration to Confit/YAML
+configuration.
+"""
+import os
+import ConfigParser
+import codecs
+import yaml
+import logging
+import time
+import itertools
+import re
+
+import beets
+from beets import util
+from beets import ui
+from beets.util import confit
+
+CONFIG_PATH_VAR = 'BEETSCONFIG'
+DEFAULT_CONFIG_FILENAME_UNIX = '.beetsconfig'
+DEFAULT_CONFIG_FILENAME_WINDOWS = 'beetsconfig.ini'
+DEFAULT_LIBRARY_FILENAME_UNIX = '.beetsmusic.blb'
+DEFAULT_LIBRARY_FILENAME_WINDOWS = 'beetsmusic.blb'
+WINDOWS_BASEDIR = os.environ.get('APPDATA') or '~'
+
+OLD_CONFIG_SUFFIX = '.old'
+PLUGIN_NAMES = {
+    'rdm': 'random',
+    'fuzzy_search': 'fuzzy',
+}
+AUTO_KEYS = ('automatic', 'autofetch', 'autoembed', 'autoscrub')
+IMPORTFEEDS_PREFIX = 'feeds_'
+CONFIG_MIGRATED_MESSAGE = u"""
+You appear to be upgrading from beets 1.0 (or earlier) to 1.1. Your
+configuration file has been migrated automatically to:
+{newconfig}
+Edit this file to configure beets. You might want to remove your
+old-style ".beetsconfig" file now. See the documentation for more
+details on the new configuration system:
+http://beets.readthedocs.org/page/reference/config.html
+""".strip()
+DB_MIGRATED_MESSAGE = u'Your database file has also been copied to:\n{newdb}'
+YAML_COMMENT = '# Automatically migrated from legacy .beetsconfig.\n\n'
+
+log = logging.getLogger('beets')
+
+# An itertools recipe.
+def grouper(n, iterable):
+    args = [iter(iterable)] * n
+    return itertools.izip_longest(*args)
+
+def _displace(fn):
+    """Move a file aside using a timestamp suffix so a new file can be
+    put in its place.
+    """
+    util.move(
+        fn,
+        u'{0}.old.{1}'.format(fn, int(time.time())),
+        True
+    )
+
+def default_paths():
+    """Produces the appropriate default config and library database
+    paths for the current system. On Unix, this is always in ~. On
+    Windows, tries ~ first and then $APPDATA for the config and library
+    files (for backwards compatibility).
+    """
+    windows = os.path.__name__ == 'ntpath'
+    if windows:
+        windata = os.environ.get('APPDATA') or '~'
+
+    # Shorthand for joining paths.
+    def exp(*vals):
+        return os.path.expanduser(os.path.join(*vals))
+
+    config = exp('~', DEFAULT_CONFIG_FILENAME_UNIX)
+    if windows and not os.path.exists(config):
+        config = exp(windata, DEFAULT_CONFIG_FILENAME_WINDOWS)
+
+    libpath = exp('~', DEFAULT_LIBRARY_FILENAME_UNIX)
+    if windows and not os.path.exists(libpath):
+        libpath = exp(windata, DEFAULT_LIBRARY_FILENAME_WINDOWS)
+
+    return config, libpath
+
+def get_config():
+    """Using the same logic as beets 1.0, locate and read the
+    .beetsconfig file. Return a ConfigParser instance or None if no
+    config is found.
+    """
+    default_config, default_libpath = default_paths()
+    if CONFIG_PATH_VAR in os.environ:
+        configpath = os.path.expanduser(os.environ[CONFIG_PATH_VAR])
+    else:
+        configpath = default_config
+
+    config = ConfigParser.SafeConfigParser()
+    if os.path.exists(util.syspath(configpath)):
+        with codecs.open(configpath, 'r', encoding='utf-8') as f:
+            config.readfp(f)
+        return config, configpath
+    else:
+        return None, configpath
+
+def flatten_config(config):
+    """Given a ConfigParser, flatten the values into a dict-of-dicts
+    representation where each section gets its own dictionary of values.
+    """
+    out = confit.OrderedDict()
+    for section in config.sections():
+        sec_dict = out[section] = confit.OrderedDict()
+        for option in config.options(section):
+            sec_dict[option] = config.get(section, option, True)
+    return out
+
+def transform_value(value):
+    """Given a string read as the value of a config option, return a
+    massaged version of that value (possibly with a different type).
+    """
+    # Booleans.
+    if value.lower() in ('false', 'no', 'off'):
+        return False
+    elif value.lower() in ('true', 'yes', 'on'):
+        return True
+
+    # Integers.
+    try:
+        return int(value)
+    except ValueError:
+        pass
+
+    # Floats.
+    try:
+        return float(value)
+    except ValueError:
+        pass
+    
+    return value
+
+def transform_data(data):
+    """Given a dict-of-dicts representation of legacy config data, tweak
+    the data into a new form. This new form is suitable for dumping as
+    YAML.
+    """
+    out = confit.OrderedDict()
+
+    for section, pairs in data.items():
+        if section == 'beets':
+            # The "main" section. In the new config system, these values
+            # are in the "root": no section at all.
+            for key, value in pairs.items():
+                value = transform_value(value)
+
+                if key.startswith('import_'):
+                    # Importer config is now under an "import:" key.
+                    if 'import' not in out:
+                        out['import'] = confit.OrderedDict()
+                    out['import'][key[7:]] = value
+
+                elif key == 'plugins':
+                    # Renamed plugins.
+                    plugins = value.split()
+                    new_plugins = [PLUGIN_NAMES.get(p, p) for p in plugins]
+                    out['plugins'] = ' '.join(new_plugins)
+
+                elif key == 'replace':
+                    # YAMLy representation for character replacements.
+                    replacements = confit.OrderedDict()
+                    for pat, repl in grouper(2, value.split()):
+                        if repl == '<strip>':
+                            repl = ''
+                        replacements[pat] = repl
+                    out['replace'] = replacements
+
+                elif key == 'pluginpath':
+                    # Used to be a colon-separated string. Now a list.
+                    out['pluginpath'] = value.split(':')
+
+                else:
+                    out[key] = value
+
+        elif pairs:
+            # Other sections (plugins, etc).
+            sec_out = out[section] = confit.OrderedDict()
+            for key, value in pairs.items():
+
+                # Standardized "auto" option.
+                if key in AUTO_KEYS:
+                    key = 'auto'
+
+                # Unnecessary : hack in queries.
+                if section == 'paths':
+                    key = key.replace('_', ':')
+
+                # Changed option names for importfeeds plugin.
+                if section == 'importfeeds':
+                    if key.startswith(IMPORTFEEDS_PREFIX):
+                        key = key[len(IMPORTFEEDS_PREFIX):]
+                
+                sec_out[key] = transform_value(value)
+
+    return out
+
+class Dumper(yaml.SafeDumper):
+    """A PyYAML Dumper that represents OrderedDicts as ordinary mappings
+    (in order, of course).
+    """
+    # From http://pyyaml.org/attachment/ticket/161/use_ordered_dict.py
+    def represent_mapping(self, tag, mapping, flow_style=None):
+        value = []
+        node = yaml.MappingNode(tag, value, flow_style=flow_style)
+        if self.alias_key is not None:
+            self.represented_objects[self.alias_key] = node
+        best_style = True
+        if hasattr(mapping, 'items'):
+            mapping = list(mapping.items())
+        for item_key, item_value in mapping:
+            node_key = self.represent_data(item_key)
+            node_value = self.represent_data(item_value)
+            if not (isinstance(node_key, yaml.ScalarNode) and \
+                    not node_key.style):
+                best_style = False
+            if not (isinstance(node_value, yaml.ScalarNode) and \
+                    not node_value.style):
+                best_style = False
+            value.append((node_key, node_value))
+        if flow_style is None:
+            if self.default_flow_style is not None:
+                node.flow_style = self.default_flow_style
+            else:
+                node.flow_style = best_style
+        return node
+Dumper.add_representer(confit.OrderedDict, Dumper.represent_dict)
+
+def migrate_config(replace=False):
+    """Migrate a legacy beetsconfig file to a new-style config.yaml file
+    in an appropriate place. If `replace` is enabled, then any existing
+    config.yaml will be moved aside. Otherwise, the process is aborted
+    when the file exists.
+    """
+
+    # Load legacy configuration data, if any.
+    config, configpath = get_config()
+    if not config:
+        log.debug(u'no config file found at {0}'.format(
+            util.displayable_path(configpath)
+            ))
+        return
+
+    # Get the new configuration file path and possibly move it out of
+    # the way.
+    destfn = os.path.join(beets.config.config_dir(), confit.CONFIG_FILENAME)
+    if os.path.exists(destfn):
+        if replace:
+            log.debug(u'moving old config aside: {0}'.format(
+                util.displayable_path(destfn)
+            ))
+            _displace(destfn)
+        else:
+            # File exists and we won't replace it. We're done.
+            return
+
+    log.debug(u'migrating config file {0}'.format(
+        util.displayable_path(configpath)
+    ))
+
+    # Convert the configuration to a data structure ready to be dumped
+    # as the new Confit file.
+    data = transform_data(flatten_config(config))
+
+    # Encode result as YAML.
+    yaml_out = yaml.dump(
+        data,
+        Dumper=Dumper,
+        default_flow_style=False,
+        indent=4,
+        width=1000,
+    )
+    # A ridiculous little hack to add some whitespace between "sections"
+    # in the YAML output. I hope this doesn't break any YAML syntax.
+    yaml_out = re.sub(r'(\n\w+:\n    [^-\s])', '\n\\1', yaml_out)
+    yaml_out = YAML_COMMENT + yaml_out
+
+    # Write the data to the new config destination.
+    log.debug(u'writing migrated config to {0}'.format(
+        util.displayable_path(destfn)
+    ))
+    with open(destfn, 'w') as f:
+        f.write(yaml_out)
+    return destfn
+
+def migrate_db(replace=False):
+    """Copy the beets library database file to the new location (e.g.,
+    from ~/.beetsmusic.blb to ~/.config/beets/library.db).
+    """
+    _, srcfn = default_paths()
+    destfn = beets.config['library'].as_filename()
+
+    if not os.path.exists(srcfn) or srcfn == destfn:
+        # Old DB does not exist or we're configured to point to the same
+        # database. Do nothing.
+        return
+    
+    if os.path.exists(destfn):
+        if replace:
+            log.debug(u'moving old database aside: {0}'.format(
+                util.displayable_path(destfn)
+            ))
+            _displace(destfn)
+        else:
+            return
+
+    log.debug(u'copying database from {0} to {1}'.format(
+        util.displayable_path(srcfn), util.displayable_path(destfn)
+    ))
+    util.copy(srcfn, destfn)
+    return destfn
+
+def migrate_state(replace=False):
+    """Copy the beets runtime state file from the old path (i.e.,
+    ~/.beetsstate) to the new path (i.e., ~/.config/beets/state.pickle).
+    """
+    srcfn = os.path.expanduser(os.path.join('~', '.beetsstate'))
+    if not os.path.exists(srcfn):
+        return
+
+    destfn = beets.config['statefile'].as_filename()
+    if os.path.exists(destfn):
+        if replace:
+            _displace(destfn)
+        else:
+            return
+
+    log.debug(u'copying state file from {0} to {1}'.format(
+        util.displayable_path(srcfn), util.displayable_path(destfn)
+    ))
+    util.copy(srcfn, destfn)
+    return destfn
+
+
+# Automatic migration when beets starts.
+
+def automigrate():
+    """Migrate the configuration, database, and state files. If any
+    migration occurs, print out a notice with some helpful next steps.
+    """
+    config_fn = migrate_config()
+    db_fn = migrate_db()
+    migrate_state()
+
+    if config_fn:
+        ui.print_(ui.colorize('fuchsia', u'MIGRATED CONFIGURATION'))
+
+        ui.print_(CONFIG_MIGRATED_MESSAGE.format(
+            newconfig=util.displayable_path(config_fn))
+        )
+        if db_fn:
+            ui.print_(DB_MIGRATED_MESSAGE.format(
+                newdb=util.displayable_path(db_fn)
+            ))
+
+        ui.input_(ui.colorize('fuchsia', u'Press ENTER to continue:'))
+        ui.print_()
+
+
+# CLI command for explicit migration.
+
+migrate_cmd = ui.Subcommand('migrate', help='convert legacy config')
+def migrate_func(lib, opts, args):
+    """Explicit command for migrating files. Existing files in each
+    destination are moved aside.
+    """
+    config_fn = migrate_config(replace=True)
+    if config_fn:
+        log.info(u'Migrated configuration to: {0}'.format(
+            util.displayable_path(config_fn)
+        ))
+    db_fn = migrate_db(replace=True)
+    if db_fn:
+        log.info(u'Migrated library database to: {0}'.format(
+            util.displayable_path(db_fn)
+        ))
+    state_fn = migrate_state(replace=True)
+    if state_fn:
+        log.info(u'Migrated state file to: {0}'.format(
+            util.displayable_path(state_fn)
+        ))
+migrate_cmd.func = migrate_func

beets/util/__init__.py

@@ -1,5 +1,5 @@
 # This file is part of beets.
-# Copyright 2012, Adrian Sampson.
+# Copyright 2013, Adrian Sampson.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -22,8 +22,10 @@ import shutil
 import fnmatch
 from collections import defaultdict
 import traceback
+import subprocess
 
 MAX_FILENAME_LENGTH = 200
+WINDOWS_MAGIC_PREFIX = u'\\\\?\\'
 
 class HumanReadableException(Exception):
     """An Exception that can include a human-readable error message to
@@ -57,12 +59,14 @@ class HumanReadableException(Exception):
     
     def _reasonstr(self):
         """Get the reason as a string."""
-        if isinstance(self.reason, basestring):
+        if isinstance(self.reason, unicode):
             return self.reason
+        elif isinstance(self.reason, basestring):  # Byte string.
+            return self.reason.decode('utf8', 'ignore')
         elif hasattr(self.reason, 'strerror'):  # i.e., EnvironmentError
             return self.reason.strerror
         else:
-            return u'"{0}"'.format(self.reason)
+            return u'"{0}"'.format(unicode(self.reason))
 
     def get_message(self):
         """Create the human-readable description of the error, sans
@@ -90,16 +94,19 @@ class FilesystemError(HumanReadableException):
     def get_message(self):
         # Use a nicer English phrasing for some specific verbs.
         if self.verb in ('move', 'copy', 'rename'):
-            clause = 'while {0} {1} to {2}'.format(
-                self._gerund(), repr(self.paths[0]), repr(self.paths[1])
+            clause = u'while {0} {1} to {2}'.format(
+                self._gerund(),
+                displayable_path(self.paths[0]),
+                displayable_path(self.paths[1])
             )
-        elif self.verb in ('delete',):
-            clause = 'while {0} {1}'.format(
-                self._gerund(), repr(self.paths[0])
+        elif self.verb in ('delete', 'write', 'create', 'read'):
+            clause = u'while {0} {1}'.format(
+                self._gerund(),
+                displayable_path(self.paths[0])
             )
         else:
-            clause = 'during {0} of paths {1}'.format(
-                self.verb, u', '.join(repr(p) for p in self.paths)
+            clause = u'during {0} of paths {1}'.format(
+                self.verb, u', '.join(displayable_path(p) for p in self.paths)
             )
 
         return u'{0} {1}'.format(self._reasonstr(), clause)
@@ -108,19 +115,23 @@ def normpath(path):
     """Provide the canonical form of the path suitable for storing in
     the database.
     """
-    return os.path.normpath(os.path.abspath(os.path.expanduser(path)))
+    path = syspath(path, prefix=False)
+    path = os.path.normpath(os.path.abspath(os.path.expanduser(path)))
+    return bytestring_path(path)
 
-def ancestry(path, pathmod=None):
+def ancestry(path):
     """Return a list consisting of path's parent directory, its
     grandparent, and so on. For instance:
+
        >>> ancestry('/a/b/c')
        ['/', '/a', '/a/b']
+
+    The argument should *not* be the result of a call to `syspath`.
     """
-    pathmod = pathmod or os.path
     out = []
     last_path = None
     while path:
-        path = pathmod.dirname(path)
+        path = os.path.dirname(path)
 
         if path == last_path:
             break
@@ -130,18 +141,27 @@ def ancestry(path, pathmod=None):
             out.insert(0, path)
     return out
 
-def sorted_walk(path, ignore=()):
-    """Like ``os.walk``, but yields things in sorted, breadth-first
-    order.  Directory and file names matching any glob pattern in
-    ``ignore`` are skipped.
+def sorted_walk(path, ignore=(), logger=None):
+    """Like `os.walk`, but yields things in case-insensitive sorted,
+    breadth-first order.  Directory and file names matching any glob
+    pattern in `ignore` are skipped. If `logger` is provided, then
+    warning messages are logged there when a directory cannot be listed.
     """
     # Make sure the path isn't a Unicode string.
     path = bytestring_path(path)
 
     # Get all the directories and files at this level.
+    try:
+        contents = os.listdir(syspath(path))
+    except OSError as exc:
+        if logger:
+            logger.warn(u'could not list directory {0}: {1}'.format(
+                displayable_path(path), exc.strerror
+            ))
+        return
     dirs = []
     files = []
-    for base in os.listdir(syspath(path)):
+    for base in contents:
         base = bytestring_path(base)
 
         # Skip ignored filenames.
@@ -160,16 +180,16 @@ def sorted_walk(path, ignore=()):
         else:
             files.append(base)
 
-    # Sort lists and yield the current level.
-    dirs.sort()
-    files.sort()
+    # Sort lists (case-insensitive) and yield the current level.
+    dirs.sort(key=bytes.lower)
+    files.sort(key=bytes.lower)
     yield (path, dirs, files)
 
     # Recurse into directories.
     for base in dirs:
         cur = os.path.join(path, base)
         # yield from sorted_walk(...)
-        for res in sorted_walk(cur, ignore):
+        for res in sorted_walk(cur, ignore, logger):
             yield res
 
 def mkdirall(path):
@@ -178,13 +198,31 @@ def mkdirall(path):
     """
     for ancestor in ancestry(path):
         if not os.path.isdir(syspath(ancestor)):
-            os.mkdir(syspath(ancestor))
+            try:
+                os.mkdir(syspath(ancestor))
+            except (OSError, IOError) as exc:
+                raise FilesystemError(exc, 'create', (ancestor,),
+                                      traceback.format_exc())
+
+def fnmatch_all(names, patterns):
+    """Determine whether all strings in `names` match at least one of
+    the `patterns`, which should be shell glob expressions.
+    """
+    for name in names:
+        matches = False
+        for pattern in patterns:
+            matches = fnmatch.fnmatch(name, pattern)
+            if matches:
+                break
+        if not matches:
+            return False
+    return True
 
 def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')):
     """If path is an empty directory, then remove it. Recursively remove
     path's ancestry up to root (which is never removed) where there are
     empty directories. If path is not contained in root, then nothing is
-    removed. Filenames in clutter are ignored when determining
+    removed. Glob patterns in clutter are ignored when determining
     emptiness. If root is not provided, then only path may be removed
     (i.e., no recursive removal).
     """
@@ -211,8 +249,7 @@ def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')):
         if not os.path.exists(directory):
             # Directory gone already.
             continue
-
-        if all(fn in clutter for fn in os.listdir(directory)):
+        if fnmatch_all(os.listdir(directory), clutter):
             # Directory contains only clutter (or nothing).
             try:
                 shutil.rmtree(directory)
@@ -221,27 +258,43 @@ def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')):
         else:
             break
 
-def components(path, pathmod=None):
+def components(path):
     """Return a list of the path components in path. For instance:
+
        >>> components('/a/b/c')
        ['a', 'b', 'c']
+
+    The argument should *not* be the result of a call to `syspath`.
     """
-    pathmod = pathmod or os.path
     comps = []
-    ances = ancestry(path, pathmod)
+    ances = ancestry(path)
     for anc in ances:
-        comp = pathmod.basename(anc)
+        comp = os.path.basename(anc)
         if comp:
             comps.append(comp)
         else:  # root
             comps.append(anc)
 
-    last = pathmod.basename(path)
+    last = os.path.basename(path)
     if last:
         comps.append(last)
 
     return comps
 
+def _fsencoding():
+    """Get the system's filesystem encoding. On Windows, this is always
+    UTF-8 (not MBCS).
+    """
+    encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
+    if encoding == 'mbcs':
+        # On Windows, a broken encoding known to Python as "MBCS" is
+        # used for the filesystem. However, we only use the Unicode API
+        # for Windows paths, so the encoding is actually immaterial so
+        # we can avoid dealing with this nastiness. We arbitrarily
+        # choose UTF-8.
+        encoding = 'utf8'
+    return encoding
+
 def bytestring_path(path):
     """Given a path, which is either a str or a unicode, returns a str
     path (ensuring that we never deal with Unicode pathnames).
@@ -250,46 +303,45 @@ def bytestring_path(path):
     if isinstance(path, str):
         return path
 
+    # On Windows, remove the magic prefix added by `syspath`. This makes
+    # ``bytestring_path(syspath(X)) == X``, i.e., we can safely
+    # round-trip through `syspath`.
+    if os.path.__name__ == 'ntpath' and path.startswith(WINDOWS_MAGIC_PREFIX):
+        path = path[len(WINDOWS_MAGIC_PREFIX):]
+
     # Try to encode with default encodings, but fall back to UTF8.
-    encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
-    if encoding == 'mbcs':
-        # On Windows, a broken encoding known to Python as "MBCS" is
-        # used for the filesystem. However, we only use the Unicode API
-        # for Windows paths, so the encoding is actually immaterial so
-        # we can avoid dealing with this nastiness. We arbitrarily
-        # choose UTF-8.
-        encoding = 'utf8'
     try:
-        return path.encode(encoding)
+        return path.encode(_fsencoding())
     except (UnicodeError, LookupError):
         return path.encode('utf8')
 
-def displayable_path(path):
+def displayable_path(path, separator=u'; '):
     """Attempts to decode a bytestring path to a unicode object for the
-    purpose of displaying it to the user.
+    purpose of displaying it to the user. If the `path` argument is a
+    list or a tuple, the elements are joined with `separator`.
     """
-    if isinstance(path, unicode):
+    if isinstance(path, (list, tuple)):
+        return separator.join(displayable_path(p) for p in path)
+    elif isinstance(path, unicode):
         return path
     elif not isinstance(path, str):
         # A non-string object: just get its unicode representation.
         return unicode(path)
 
-    encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
     try:
-        return path.decode(encoding, 'ignore')
+        return path.decode(_fsencoding(), 'ignore')
     except (UnicodeError, LookupError):
         return path.decode('utf8', 'ignore')
 
-def syspath(path, pathmod=None):
+def syspath(path, prefix=True):
     """Convert a path for use by the operating system. In particular,
     paths on Windows must receive a magic prefix and must be converted
-    to unicode before they are sent to the OS.
+    to Unicode before they are sent to the OS. To disable the magic
+    prefix on Windows, set `prefix` to False---but only do this if you
+    *really* know what you're doing.
     """
-    pathmod = pathmod or os.path
-    windows = pathmod.__name__ == 'ntpath'
-
     # Don't do anything if we're not on windows
-    if not windows:
+    if os.path.__name__ != 'ntpath':
         return path
 
     if not isinstance(path, unicode):
@@ -305,8 +357,8 @@ def syspath(path, pathmod=None):
             path = path.decode(encoding, 'replace')
 
     # Add the magic prefix if it isn't already there
-    if not path.startswith(u'\\\\?\\'):
-        path = u'\\\\?\\' + path
+    if prefix and not path.startswith(WINDOWS_MAGIC_PREFIX):
+        path = WINDOWS_MAGIC_PREFIX + path
 
     return path
 
@@ -326,7 +378,7 @@ def remove(path, soft=True):
     except (OSError, IOError) as exc:
         raise FilesystemError(exc, 'delete', (path,), traceback.format_exc())
 
-def copy(path, dest, replace=False, pathmod=os.path):
+def copy(path, dest, replace=False):
     """Copy a plain file. Permissions are not copied. If `dest` already
     exists, raises a FilesystemError unless `replace` is True. Has no
     effect if `path` is the same as `dest`. Paths are translated to
@@ -336,7 +388,7 @@ def copy(path, dest, replace=False, pathmod=os.path):
         return
     path = syspath(path)
     dest = syspath(dest)
-    if not replace and pathmod.exists(dest):
+    if not replace and os.path.exists(dest):
         raise FilesystemError('file exists', 'copy', (path, dest))
     try:
         shutil.copyfile(path, dest)
@@ -344,7 +396,7 @@ def copy(path, dest, replace=False, pathmod=os.path):
         raise FilesystemError(exc, 'copy', (path, dest),
                               traceback.format_exc())
 
-def move(path, dest, replace=False, pathmod=os.path):
+def move(path, dest, replace=False):
     """Rename a file. `dest` may not be a directory. If `dest` already
     exists, raises an OSError unless `replace` is True. Has no effect if
     `path` is the same as `dest`. If the paths are on different
@@ -356,7 +408,7 @@ def move(path, dest, replace=False, pathmod=os.path):
         return
     path = syspath(path)
     dest = syspath(dest)
-    if pathmod.exists(dest):
+    if os.path.exists(dest) and not replace:
         raise FilesystemError('file exists', 'rename', (path, dest),
                               traceback.format_exc())
 
@@ -405,60 +457,41 @@ CHAR_REPLACE = [
     (re.compile(ur'\.$'), u'_'),  # Trailing dots.
     (re.compile(ur'\s+$'), u''),  # Trailing whitespace.
 ]
-def sanitize_path(path, pathmod=None, replacements=None):
+def sanitize_path(path, replacements=None):
     """Takes a path (as a Unicode string) and makes sure that it is
     legal. Returns a new path. Only works with fragments; won't work
     reliably on Windows when a path begins with a drive letter. Path
     separators (including altsep!) should already be cleaned from the
     path components. If replacements is specified, it is used *instead*
-    of the default set of replacements for the platform; it must be a
-    list of (compiled regex, replacement string) pairs.
+    of the default set of replacements; it must be a list of (compiled
+    regex, replacement string) pairs.
     """
-    pathmod = pathmod or os.path
+    replacements = replacements or CHAR_REPLACE
 
-    # Choose the appropriate replacements.
-    if not replacements:
-        replacements = list(CHAR_REPLACE)
-
-    comps = components(path, pathmod)
+    comps = components(path)
     if not comps:
         return ''
     for i, comp in enumerate(comps):
-        # Replace special characters.
         for regex, repl in replacements:
             comp = regex.sub(repl, comp)
-
-        # Truncate each component.
-        comp = comp[:MAX_FILENAME_LENGTH]
-
         comps[i] = comp
-    return pathmod.join(*comps)
+    return os.path.join(*comps)
 
-def sanitize_for_path(value, pathmod, key=None):
-    """Sanitize the value for inclusion in a path: replace separators
-    with _, etc. Doesn't guarantee that the whole path will be valid;
-    you should still call sanitize_path on the complete path.
+def truncate_path(path, length=MAX_FILENAME_LENGTH):
+    """Given a bytestring path or a Unicode path fragment, truncate the
+    components to a legal length. In the last component, the extension
+    is preserved.
     """
-    if isinstance(value, basestring):
-        for sep in (pathmod.sep, pathmod.altsep):
-            if sep:
-                value = value.replace(sep, u'_')
-    elif key in ('track', 'tracktotal', 'disc', 'disctotal'):
-        # Pad indices with zeros.
-        value = u'%02i' % (value or 0)
-    elif key == 'year':
-        value = u'%04i' % (value or 0)
-    elif key in ('month', 'day'):
-        value = u'%02i' % (value or 0)
-    elif key == 'bitrate':
-        # Bitrate gets formatted as kbps.
-        value = u'%ikbps' % ((value or 0) // 1000)
-    elif key == 'samplerate':
-        # Sample rate formatted as kHz.
-        value = u'%ikHz' % ((value or 0) // 1000)
-    else:
-        value = unicode(value)
-    return value
+    comps = components(path)
+
+    out = [c[:length] for c in comps]
+    base, ext = os.path.splitext(comps[-1])
+    if ext:
+        # Last component has an extension.
+        base = base[:length - len(ext)]
+        out[-1] = base + ext
+
+    return os.path.join(*out)
 
 def str2bool(value):
     """Returns a boolean reflecting a human-entered string."""
@@ -467,6 +500,19 @@ def str2bool(value):
     else:
         return False
 
+def as_string(value):
+    """Convert a value to a Unicode object for matching with a query.
+    None becomes the empty string. Bytestrings are silently decoded.
+    """
+    if value is None:
+        return u''
+    elif isinstance(value, buffer):
+        return str(value).decode('utf8', 'ignore')
+    elif isinstance(value, str):
+        return value.decode('utf8', 'ignore')
+    else:
+        return unicode(value)
+
 def levenshtein(s1, s2):
     """A nice DP edit distance implementation from Wikibooks:
     http://en.wikibooks.org/wiki/Algorithm_implementation/Strings/
@@ -511,3 +557,62 @@ def plurality(objs):
             res = obj
 
     return res, max_freq
+
+def cpu_count():
+    """Return the number of hardware thread contexts (cores or SMT
+    threads) in the system.
+    """
+    # Adapted from the soundconverter project:
+    # https://github.com/kassoulet/soundconverter
+    if sys.platform == 'win32':
+        try:
+            num = int(os.environ['NUMBER_OF_PROCESSORS'])
+        except (ValueError, KeyError):
+            num = 0
+    elif sys.platform == 'darwin':
+        try:
+            num = int(os.popen('sysctl -n hw.ncpu').read())
+        except ValueError:
+            num = 0
+    else:
+        try:
+            num = os.sysconf('SC_NPROCESSORS_ONLN')
+        except (ValueError, OSError, AttributeError):
+            num = 0
+    if num >= 1:
+        return num
+    else:
+        return 1
+
+def command_output(cmd):
+    """Wraps the `subprocess` module to invoke a command (given as a
+    list of arguments starting with the command name) and collect
+    stdout. The stderr stream is ignored. May raise
+    `subprocess.CalledProcessError` or an `OSError`.
+
+    This replaces `subprocess.check_output`, which isn't available in
+    Python 2.6 and which can have problems if lots of output is sent to
+    stderr.
+    """
+    with open(os.devnull, 'w') as devnull:
+        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=devnull)
+        stdout, _ = proc.communicate()
+    if proc.returncode:
+        raise subprocess.CalledProcessError(proc.returncode, cmd)
+    return stdout
+
+def max_filename_length(path, limit=MAX_FILENAME_LENGTH):
+    """Attempt to determine the maximum filename length for the
+    filesystem containing `path`. If the value is greater than `limit`,
+    then `limit` is used instead (to prevent errors when a filesystem
+    misreports its capacity). If it cannot be determined (e.g., on
+    Windows), return `limit`.
+    """
+    if hasattr(os, 'statvfs'):
+        try:
+            res = os.statvfs(path)
+        except OSError:
+            return limit
+        return min(res[9], limit)
+    else:
+        return limit

beets/util/artresizer.py

@@ -0,0 +1,188 @@
+# This file is part of beets.
+# Copyright 2013, Fabrice Laporte
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+
+"""Abstraction layer to resize images using PIL, ImageMagick, or a
+public resizing proxy if neither is available.
+"""
+import urllib
+import subprocess
+import os
+from tempfile import NamedTemporaryFile
+import logging
+from beets import util
+
+# Resizing methods
+PIL = 1
+IMAGEMAGICK = 2
+WEBPROXY = 3
+
+PROXY_URL = 'http://images.weserv.nl/'
+
+log = logging.getLogger('beets')
+
+
+def resize_url(url, maxwidth):
+    """Return a proxied image URL that resizes the original image to
+    maxwidth (preserving aspect ratio).
+    """
+    return '{0}?{1}'.format(PROXY_URL, urllib.urlencode({
+        'url': url.replace('http://',''),
+        'w': str(maxwidth),
+    }))
+
+
+def temp_file_for(path):
+    """Return an unused filename with the same extension as the
+    specified path.
+    """
+    ext = os.path.splitext(path)[1]
+    with NamedTemporaryFile(suffix=ext, delete=False) as f:
+        return f.name
+
+
+def pil_resize(maxwidth, path_in, path_out=None):
+    """Resize using Python Imaging Library (PIL).  Return the output path
+    of resized image.
+    """
+    path_out = path_out or temp_file_for(path_in)
+    from PIL import Image
+    log.debug(u'artresizer: PIL resizing {0} to {1}'.format(
+        util.displayable_path(path_in), util.displayable_path(path_out)
+    ))
+
+    try:
+        im = Image.open(util.syspath(path_in))
+        size = maxwidth, maxwidth
+        im.thumbnail(size, Image.ANTIALIAS)
+        im.save(path_out)
+        return path_out
+    except IOError:
+        log.error(u"PIL cannot create thumbnail for '{0}'".format(
+            util.displayable_path(path_in)
+        ))
+        return path_in
+
+
+def im_resize(maxwidth, path_in, path_out=None):
+    """Resize using ImageMagick's ``convert`` tool.
+    tool. Return the output path of resized image.
+    """
+    path_out = path_out or temp_file_for(path_in)
+    log.debug(u'artresizer: ImageMagick resizing {0} to {1}'.format(
+        util.displayable_path(path_in), util.displayable_path(path_out)
+    ))
+
+    # "-resize widthxheight>" shrinks images with dimension(s) larger
+    # than the corresponding width and/or height dimension(s). The >
+    # "only shrink" flag is prefixed by ^ escape char for Windows
+    # compatibility.
+    try:
+        util.command_output([
+            'convert', util.syspath(path_in),
+            '-resize', '{0}x^>'.format(maxwidth), path_out
+        ])
+    except subprocess.CalledProcessError:
+        log.warn(u'artresizer: IM convert failed for {0}'.format(
+            util.displayable_path(path_in)
+        ))
+        return path_in
+    return path_out
+
+
+BACKEND_FUNCS = {
+    PIL: pil_resize,
+    IMAGEMAGICK: im_resize,
+}
+
+
+class Shareable(type):
+    """A pseudo-singleton metaclass that allows both shared and
+    non-shared instances. The ``MyClass.shared`` property holds a
+    lazily-created shared instance of ``MyClass`` while calling
+    ``MyClass()`` to construct a new object works as usual.
+    """
+    def __init__(cls, name, bases, dict):
+        super(Shareable, cls).__init__(name, bases, dict)
+        cls._instance = None
+
+    @property
+    def shared(cls):
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+
+class ArtResizer(object):
+    """A singleton class that performs image resizes.
+    """
+    __metaclass__ = Shareable
+
+    def __init__(self, method=None):
+        """Create a resizer object for the given method or, if none is
+        specified, with an inferred method.
+        """
+        self.method = method or self._guess_method()
+        log.debug(u"artresizer: method is {0}".format(self.method))
+
+    def resize(self, maxwidth, path_in, path_out=None):
+        """Manipulate an image file according to the method, returning a
+        new path. For PIL or IMAGEMAGIC methods, resizes the image to a
+        temporary file. For WEBPROXY, returns `path_in` unmodified.
+        """
+        if self.local:
+            func = BACKEND_FUNCS[self.method]
+            return func(maxwidth, path_in, path_out)
+        else:
+            return path_in
+
+    def proxy_url(self, maxwidth, url):
+        """Modifies an image URL according the method, returning a new
+        URL. For WEBPROXY, a URL on the proxy server is returned.
+        Otherwise, the URL is returned unmodified.
+        """
+        if self.local:
+            return url
+        else:
+            return resize_url(url, maxwidth)
+
+    @property
+    def local(self):
+        """A boolean indicating whether the resizing method is performed
+        locally (i.e., PIL or IMAGEMAGICK).
+        """
+        return self.method in BACKEND_FUNCS
+
+    @staticmethod
+    def _guess_method():
+        """Determine which resizing method to use. Returns PIL,
+        IMAGEMAGICK, or WEBPROXY depending on available dependencies.
+        """
+        # Try importing PIL.
+        try:
+            __import__('PIL', fromlist=['Image'])
+            return PIL
+        except ImportError:
+            pass
+
+        # Try invoking ImageMagick's "convert".
+        try:
+            out = util.command_output(['convert', '--version'])
+            if 'imagemagick' in out.lower():
+                # system32/convert.exe may be interfering
+                return IMAGEMAGICK
+        except (subprocess.CalledProcessError, OSError):
+            pass
+
+        # Fall back to Web proxy method.
+        return WEBPROXY

beets/util/bluelet.py

@@ -25,7 +25,8 @@ if PY3:
 else:
     exec("""
 def _reraise(typ, exc, tb):
-    raise typ, exc, tb""")
+    raise typ, exc, tb
+""")
 
 
 # Basic events used for thread scheduling.
@@ -51,7 +52,7 @@ class WaitableEvent(Event):
         return (), (), ()
 
     def fire(self):
-        """Called when an assoicated file descriptor becomes ready
+        """Called when an associated file descriptor becomes ready
         (i.e., is returned from a select() call).
         """
         pass
@@ -78,6 +79,11 @@ class JoinEvent(Event):
     def __init__(self, child):
         self.child = child
 
+class KillEvent(Event):
+    """Unschedule a child thread."""
+    def __init__(self, child):
+        self.child = child
+
 class DelegationEvent(Event):
     """Suspend execution of the current thread, start a new thread and,
     once the child thread finished, return control to the parent
@@ -194,7 +200,14 @@ class ThreadException(Exception):
         _reraise(self.exc_info[0], self.exc_info[1], self.exc_info[2])
 
 SUSPENDED = Event()  # Special sentinel placeholder for suspended threads.
-        
+
+class Delegated(Event):
+    """Placeholder indicating that a thread has delegated execution to a
+    different thread.
+    """
+    def __init__(self, child):
+        self.child = child
+
 def run(root_coro):
     """Schedules a coroutine, running it to completion. This
     encapsulates the Bluelet scheduler, which the root coroutine can
@@ -260,6 +273,20 @@ def run(root_coro):
                 next_event = DelegationEvent(next_event)
             threads[coro] = next_event
 
+    def kill_thread(coro):
+        """Unschedule this thread and its (recursive) delegates.
+        """
+        # Collect all coroutines in the delegation stack.
+        coros = [coro]
+        while isinstance(threads[coro], Delegated):
+            coro = threads[coro].child
+            coros.append(coro)
+
+        # Complete each coroutine from the top to the bottom of the
+        # stack.
+        for coro in reversed(coros):
+            complete_thread(coro, None)
+
     # Continue advancing threads until root thread exits.
     exit_te = None
     while threads:
@@ -280,7 +307,7 @@ def run(root_coro):
                         advance_thread(coro, event.exc_info, True)
                         have_ready = True
                     elif isinstance(event, DelegationEvent):
-                        threads[coro] = SUSPENDED  # Suspend.
+                        threads[coro] = Delegated(event.spawned)  # Suspend.
                         threads[event.spawned] = ValueEvent(None)  # Spawn.
                         delegators[event.spawned] = coro
                         have_ready = True
@@ -292,6 +319,10 @@ def run(root_coro):
                         threads[coro] = SUSPENDED  # Suspend.
                         joiners[event.child].append(coro)
                         have_ready = True
+                    elif isinstance(event, KillEvent):
+                        threads[coro] = ValueEvent(None)
+                        kill_thread(event.child)
+                        have_ready = True
 
                 # Only start the select when nothing else is ready.
                 if not have_ready:
@@ -314,7 +345,7 @@ def run(root_coro):
                     threads[event2coro[event]] = ReturnEvent(None)
                 else:
                     advance_thread(event2coro[event], value)
-    
+
         except ThreadException as te:
             # Exception raised from inside a thread.
             event = ExceptionEvent(te.exc_info)
@@ -327,7 +358,7 @@ def run(root_coro):
                 # The thread is root-level. Raise in client code.
                 exit_te = te
                 break
-        
+
         except:
             # For instance, KeyboardInterrupt during select(). Raise
             # into root thread and terminate others.
@@ -344,12 +375,16 @@ def run(root_coro):
 
 # Sockets and their associated events.
 
+class SocketClosedError(Exception):
+    pass
+
 class Listener(object):
     """A socket wrapper object for listening sockets.
     """
     def __init__(self, host, port):
         """Create a listening socket on the given hostname and port.
         """
+        self._closed = False
         self.host = host
         self.port = port
         self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -362,11 +397,14 @@ class Listener(object):
         When a connection is made, the event returns a Connection
         object.
         """
+        if self._closed:
+            raise SocketClosedError()
         return AcceptEvent(self)
 
     def close(self):
         """Immediately close the listening socket. (Not an event.)
         """
+        self._closed = True
         self.sock.close()
 
 class Connection(object):
@@ -376,13 +414,18 @@ class Connection(object):
         self.sock = sock
         self.addr = addr
         self._buf = b''
+        self._closed = False
 
     def close(self):
         """Close the connection."""
+        self._closed = True
         self.sock.close()
 
     def recv(self, size):
         """Read at most size bytes of data from the socket."""
+        if self._closed:
+            raise SocketClosedError()
+
         if self._buf:
             # We already have data read previously.
             out = self._buf[:size]
@@ -395,14 +438,21 @@ class Connection(object):
         """Sends data on the socket, returning the number of bytes
         successfully sent.
         """
+        if self._closed:
+            raise SocketClosedError()
         return SendEvent(self, data)
 
     def sendall(self, data):
         """Send all of data on the socket."""
+        if self._closed:
+            raise SocketClosedError()
         return SendEvent(self, data, True)
 
     def readline(self, terminator=b"\n", bufsize=1024):
         """Reads a line (delimited by terminator) from the socket."""
+        if self._closed:
+            raise SocketClosedError()
+
         while True:
             if terminator in self._buf:
                 line, self._buf = self._buf.split(terminator, 1)
@@ -490,13 +540,13 @@ def call(coro):
         raise ValueError('%s is not a coroutine' % str(coro))
     return DelegationEvent(coro)
 
-def end(value = None):
+def end(value=None):
     """Event: ends the coroutine and returns a value to its
     delegator.
     """
     return ReturnEvent(value)
 
-def read(fd, bufsize = None):
+def read(fd, bufsize=None):
     """Event: read from a file descriptor asynchronously."""
     if bufsize is None:
         # Read all.
@@ -536,6 +586,11 @@ def join(coro):
     """
     return JoinEvent(coro)
 
+def kill(coro):
+    """Halt the execution of a different `spawn`ed thread.
+    """
+    return KillEvent(coro)
+
 
 # Convenience function for running socket servers.
 
@@ -550,7 +605,7 @@ def server(host, port, func):
             yield func(conn)
         finally:
             conn.close()
-            
+
     listener = Listener(host, port)
     try:
         while True:

beets/util/confit.py

@@ -0,0 +1,900 @@
+# This file is part of Confit.
+# Copyright 2014, Adrian Sampson.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+
+"""Worry-free YAML configuration files.
+"""
+from __future__ import unicode_literals
+import platform
+import os
+import pkgutil
+import sys
+import yaml
+import types
+try:
+    from collections import OrderedDict
+except ImportError:
+    from ordereddict import OrderedDict
+
+UNIX_DIR_VAR = 'XDG_CONFIG_HOME'
+UNIX_DIR_FALLBACK = '~/.config'
+WINDOWS_DIR_VAR = 'APPDATA'
+WINDOWS_DIR_FALLBACK = '~\\AppData\\Roaming'
+MAC_DIR = '~/Library/Application Support'
+
+CONFIG_FILENAME = 'config.yaml'
+DEFAULT_FILENAME = 'config_default.yaml'
+ROOT_NAME = 'root'
+
+YAML_TAB_PROBLEM = "found character '\\t' that cannot start any token"
+
+
+# Utilities.
+
+PY3 = sys.version_info[0] == 3
+STRING = str if PY3 else unicode
+BASESTRING = str if PY3 else basestring
+NUMERIC_TYPES = (int, float) if PY3 else (int, float, long)
+TYPE_TYPES = (type,) if PY3 else (type, types.ClassType)
+
+def iter_first(sequence):
+    """Get the first element from an iterable or raise a ValueError if
+    the iterator generates no values.
+    """
+    it = iter(sequence)
+    try:
+        if PY3:
+            return next(it)
+        else:
+            return it.next()
+    except StopIteration:
+        raise ValueError()
+
+
+# Exceptions.
+
+class ConfigError(Exception):
+    """Base class for exceptions raised when querying a configuration.
+    """
+
+class NotFoundError(ConfigError):
+    """A requested value could not be found in the configuration trees.
+    """
+
+class ConfigTypeError(ConfigError, TypeError):
+    """The value in the configuration did not match the expected type.
+    """
+
+class ConfigValueError(ConfigError, ValueError):
+    """The value in the configuration is illegal."""
+
+class ConfigReadError(ConfigError):
+    """A configuration file could not be read."""
+    def __init__(self, filename, reason=None):
+        self.filename = filename
+        self.reason = reason
+
+        message = 'file {0} could not be read'.format(filename)
+        if isinstance(reason, yaml.scanner.ScannerError) and \
+                reason.problem == YAML_TAB_PROBLEM:
+            # Special-case error message for tab indentation in YAML markup.
+            message += ': found tab character at line {0}, column {1}'.format(
+                reason.problem_mark.line + 1,
+                reason.problem_mark.column + 1,
+            )
+        elif reason:
+            # Generic error message uses exception's message.
+            message += ': {0}'.format(reason)
+
+        super(ConfigReadError, self).__init__(message)
+
+
+# Views and sources.
+
+class ConfigSource(dict):
+    """A dictionary augmented with metadata about the source of the
+    configuration.
+    """
+    def __init__(self, value, filename=None, default=False):
+        super(ConfigSource, self).__init__(value)
+        if filename is not None and not isinstance(filename, BASESTRING):
+            raise TypeError('filename must be a string or None')
+        self.filename = filename
+        self.default = default
+
+    def __repr__(self):
+        return 'ConfigSource({0}, {1}, {2})'.format(
+            super(ConfigSource, self).__repr__(),
+            repr(self.filename),
+            repr(self.default)
+        )
+
+    @classmethod
+    def of(self, value):
+        """Given either a dictionary or a `ConfigSource` object, return
+        a `ConfigSource` object. This lets a function accept either type
+        of object as an argument.
+        """
+        if isinstance(value, ConfigSource):
+            return value
+        elif isinstance(value, dict):
+            return ConfigSource(value)
+        else:
+            raise TypeError('source value must be a dict')
+
+class ConfigView(object):
+    """A configuration "view" is a query into a program's configuration
+    data. A view represents a hypothetical location in the configuration
+    tree; to extract the data from the location, a client typically
+    calls the ``view.get()`` method. The client can access children in
+    the tree (subviews) by subscripting the parent view (i.e.,
+    ``view[key]``).
+    """
+
+    name = None
+    """The name of the view, depicting the path taken through the
+    configuration in Python-like syntax (e.g., ``foo['bar'][42]``).
+    """
+
+    def resolve(self):
+        """The core (internal) data retrieval method. Generates (value,
+        source) pairs for each source that contains a value for this
+        view. May raise ConfigTypeError if a type error occurs while
+        traversing a source.
+        """
+        raise NotImplementedError
+
+    def first(self):
+        """Return a (value, source) pair for the first object found for
+        this view. This amounts to the first element returned by
+        `resolve`. If no values are available, a NotFoundError is
+        raised.
+        """
+        pairs = self.resolve()
+        try:
+            return iter_first(pairs)
+        except ValueError:
+            raise NotFoundError("{0} not found".format(self.name))
+
+    def exists(self):
+        """Determine whether the view has a setting in any source.
+        """
+        try:
+            self.first()
+        except NotFoundError:
+            return False
+        return True
+
+    def add(self, value):
+        """Set the *default* value for this configuration view. The
+        specified value is added as the lowest-priority configuration
+        data source.
+        """
+        raise NotImplementedError
+
+    def set(self, value):
+        """*Override* the value for this configuration view. The
+        specified value is added as the highest-priority configuration
+        data source.
+        """
+        raise NotImplementedError
+
+    def root(self):
+        """The RootView object from which this view is descended.
+        """
+        raise NotImplementedError
+
+    def __repr__(self):
+        return '<ConfigView: %s>' % self.name
+
+    def __getitem__(self, key):
+        """Get a subview of this view."""
+        return Subview(self, key)
+
+    def __setitem__(self, key, value):
+        """Create an overlay source to assign a given key under this
+        view.
+        """
+        self.set({key: value})
+
+    def set_args(self, namespace):
+        """Overlay parsed command-line arguments, generated by a library
+        like argparse or optparse, onto this view's value.
+        """
+        args = {}
+        for key, value in namespace.__dict__.items():
+            if value is not None:  # Avoid unset options.
+                args[key] = value
+        self.set(args)
+
+    # Magical conversions. These special methods make it possible to use
+    # View objects somewhat transparently in certain circumstances. For
+    # example, rather than using ``view.get(bool)``, it's possible to
+    # just say ``bool(view)`` or use ``view`` in a conditional.
+
+    def __str__(self):
+        """Gets the value for this view as a byte string."""
+        return str(self.get())
+
+    def __unicode__(self):
+        """Gets the value for this view as a unicode string. (Python 2
+        only.)
+        """
+        return unicode(self.get())
+
+    def __nonzero__(self):
+        """Gets the value for this view as a boolean. (Python 2 only.)
+        """
+        return self.__bool__()
+
+    def __bool__(self):
+        """Gets the value for this view as a boolean. (Python 3 only.)
+        """
+        return bool(self.get())
+
+    # Dictionary emulation methods.
+
+    def keys(self):
+        """Returns a list containing all the keys available as subviews
+        of the current views. This enumerates all the keys in *all*
+        dictionaries matching the current view, in contrast to
+        ``view.get(dict).keys()``, which gets all the keys for the
+        *first* dict matching the view. If the object for this view in
+        any source is not a dict, then a ConfigTypeError is raised. The
+        keys are ordered according to how they appear in each source.
+        """
+        keys = []
+
+        for dic, _ in self.resolve():
+            try:
+                cur_keys = dic.keys()
+            except AttributeError:
+                raise ConfigTypeError(
+                    '{0} must be a dict, not {1}'.format(
+                        self.name, type(dic).__name__
+                    )
+                )
+
+            for key in cur_keys:
+                if key not in keys:
+                    keys.append(key)
+
+        return keys
+
+    def items(self):
+        """Iterates over (key, subview) pairs contained in dictionaries
+        from *all* sources at this view. If the object for this view in
+        any source is not a dict, then a ConfigTypeError is raised.
+        """
+        for key in self.keys():
+            yield key, self[key]
+
+    def values(self):
+        """Iterates over all the subviews contained in dictionaries from
+        *all* sources at this view. If the object for this view in any
+        source is not a dict, then a ConfigTypeError is raised.
+        """
+        for key in self.keys():
+            yield self[key]
+
+    # List/sequence emulation.
+
+    def all_contents(self):
+        """Iterates over all subviews from collections at this view from
+        *all* sources. If the object for this view in any source is not
+        iterable, then a ConfigTypeError is raised. This method is
+        intended to be used when the view indicates a list; this method
+        will concatenate the contents of the list from all sources.
+        """
+        for collection, _ in self.resolve():
+            try:
+                it = iter(collection)
+            except TypeError:
+                raise ConfigTypeError(
+                    '{0} must be an iterable, not {1}'.format(
+                        self.name, type(collection).__name__
+                    )
+                )
+            for value in it:
+                yield value
+
+    # Validation and conversion.
+
+    def get(self, typ=None):
+        """Returns the canonical value for the view, checked against the
+        passed-in type. If the value is not an instance of the given
+        type, a ConfigTypeError is raised. May also raise a
+        NotFoundError.
+        """
+        value, _ = self.first()
+
+        if typ is not None:
+            if not isinstance(typ, TYPE_TYPES):
+                raise TypeError('argument to get() must be a type')
+
+            if not isinstance(value, typ):
+                raise ConfigTypeError(
+                    "{0} must be of type {1}, not {2}".format(
+                        self.name, typ.__name__, type(value).__name__
+                    )
+                )
+
+        return value
+
+    def as_filename(self):
+        """Get a string as a normalized as an absolute, tilde-free path.
+
+        Relative paths are relative to the configuration directory (see
+        the `config_dir` method) if they come from a file. Otherwise,
+        they are relative to the current working directory. This helps
+        attain the expected behavior when using command-line options.
+        """
+        path, source = self.first()
+        if not isinstance(path, BASESTRING):
+            raise ConfigTypeError('{0} must be a filename, not {1}'.format(
+                self.name, type(path).__name__
+            ))
+        path = os.path.expanduser(STRING(path))
+
+        if not os.path.isabs(path) and source.filename:
+            # From defaults: relative to the app's directory.
+            path = os.path.join(self.root().config_dir(), path)
+
+        return os.path.abspath(path)
+
+    def as_choice(self, choices):
+        """Ensure that the value is among a collection of choices and
+        return it. If `choices` is a dictionary, then return the
+        corresponding value rather than the value itself (the key).
+        """
+        value = self.get()
+
+        if value not in choices:
+            raise ConfigValueError(
+                '{0} must be one of {1}, not {2}'.format(
+                    self.name, repr(list(choices)), repr(value)
+                )
+            )
+
+        if isinstance(choices, dict):
+            return choices[value]
+        else:
+            return value
+
+    def as_number(self):
+        """Ensure that a value is of numeric type."""
+        value = self.get()
+        if isinstance(value, NUMERIC_TYPES):
+            return value
+        raise ConfigTypeError(
+            '{0} must be numeric, not {1}'.format(
+                self.name, type(value).__name__
+            )
+        )
+
+    def as_str_seq(self):
+        """Get the value as a list of strings. The underlying configured
+        value can be a sequence or a single string. In the latter case,
+        the string is treated as a white-space separated list of words.
+        """
+        value = self.get()
+        if isinstance(value, bytes):
+            value = value.decode('utf8', 'ignore')
+
+        if isinstance(value, STRING):
+            return value.split()
+        else:
+            try:
+                return list(value)
+            except TypeError:
+                raise ConfigTypeError(
+                    '{0} must be a whitespace-separated string or '
+                    'a list'.format(self.name)
+                )
+
+    def flatten(self):
+        """Create a hierarchy of OrderedDicts containing the data from
+        this view, recursively reifying all views to get their
+        represented values.
+        """
+        od = OrderedDict()
+        for key, view in self.items():
+            try:
+                od[key] = view.flatten()
+            except ConfigTypeError:
+                od[key] = view.get()
+        return od
+
+
+class RootView(ConfigView):
+    """The base of a view hierarchy. This view keeps track of the
+    sources that may be accessed by subviews.
+    """
+    def __init__(self, sources):
+        """Create a configuration hierarchy for a list of sources. At
+        least one source must be provided. The first source in the list
+        has the highest priority.
+        """
+        self.sources = list(sources)
+        self.name = ROOT_NAME
+
+    def add(self, obj):
+        self.sources.append(ConfigSource.of(obj))
+
+    def set(self, value):
+        self.sources.insert(0, ConfigSource.of(value))
+
+    def resolve(self):
+        return ((dict(s), s) for s in self.sources)
+
+    def clear(self):
+        """Remove all sources from this configuration."""
+        del self.sources[:]
+
+    def root(self):
+        return self
+
+
+class Subview(ConfigView):
+    """A subview accessed via a subscript of a parent view."""
+    def __init__(self, parent, key):
+        """Make a subview of a parent view for a given subscript key.
+        """
+        self.parent = parent
+        self.key = key
+
+        # Choose a human-readable name for this view.
+        if isinstance(self.parent, RootView):
+            self.name = ''
+        else:
+            self.name = self.parent.name
+            if not isinstance(self.key, int):
+                self.name += '.'
+        if isinstance(self.key, int):
+            self.name += '#{0}'.format(self.key)
+        elif isinstance(self.key, BASESTRING):
+            self.name += '{0}'.format(self.key)
+        else:
+            self.name += '{0}'.format(repr(self.key))
+
+    def resolve(self):
+        for collection, source in self.parent.resolve():
+            try:
+                value = collection[self.key]
+            except IndexError:
+                # List index out of bounds.
+                continue
+            except KeyError:
+                # Dict key does not exist.
+                continue
+            except TypeError:
+                # Not subscriptable.
+                raise ConfigTypeError(
+                    "{0} must be a collection, not {1}".format(
+                        self.parent.name, type(collection).__name__
+                    )
+                )
+            yield value, source
+
+    def set(self, value):
+        self.parent.set({self.key: value})
+
+    def add(self, value):
+        self.parent.add({self.key: value})
+
+    def root(self):
+        return self.parent.root()
+
+
+# Config file paths, including platform-specific paths and in-package
+# defaults.
+
+# Based on get_root_path from Flask by Armin Ronacher.
+def _package_path(name):
+    """Returns the path to the package containing the named module or
+    None if the path could not be identified (e.g., if
+    ``name == "__main__"``).
+    """
+    loader = pkgutil.get_loader(name)
+    if loader is None or name == '__main__':
+        return None
+
+    if hasattr(loader, 'get_filename'):
+        filepath = loader.get_filename(name)
+    else:
+        # Fall back to importing the specified module.
+        __import__(name)
+        filepath = sys.modules[name].__file__
+
+    return os.path.dirname(os.path.abspath(filepath))
+
+def config_dirs():
+    """Return a platform-specific list of candidates for user
+    configuration directories on the system.
+
+    The candidates are in order of priority, from highest to lowest. The
+    last element is the "fallback" location to be used when no
+    higher-priority config file exists.
+    """
+    paths = []
+
+    if platform.system() == 'Darwin':
+        paths.append(MAC_DIR)
+        paths.append(UNIX_DIR_FALLBACK)
+        if UNIX_DIR_VAR in os.environ:
+            paths.append(os.environ[UNIX_DIR_VAR])
+
+    elif platform.system() == 'Windows':
+        paths.append(WINDOWS_DIR_FALLBACK)
+        if WINDOWS_DIR_VAR in os.environ:
+            paths.append(os.environ[WINDOWS_DIR_VAR])
+
+    else:
+        # Assume Unix.
+        paths.append(UNIX_DIR_FALLBACK)
+        if UNIX_DIR_VAR in os.environ:
+            paths.append(os.environ[UNIX_DIR_VAR])
+
+    # Expand and deduplicate paths.
+    out = []
+    for path in paths:
+        path = os.path.abspath(os.path.expanduser(path))
+        if path not in out:
+            out.append(path)
+    return out
+
+
+# YAML loading.
+
+class Loader(yaml.SafeLoader):
+    """A customized YAML loader. This loader deviates from the official
+    YAML spec in a few convenient ways:
+
+    - All strings as are Unicode objects.
+    - All maps are OrderedDicts.
+    - Strings can begin with % without quotation.
+    """
+    # All strings should be Unicode objects, regardless of contents.
+    def _construct_unicode(self, node):
+        return self.construct_scalar(node)
+
+    # Use ordered dictionaries for every YAML map.
+    # From https://gist.github.com/844388
+    def construct_yaml_map(self, node):
+        data = OrderedDict()
+        yield data
+        value = self.construct_mapping(node)
+        data.update(value)
+
+    def construct_mapping(self, node, deep=False):
+        if isinstance(node, yaml.MappingNode):
+            self.flatten_mapping(node)
+        else:
+            raise yaml.constructor.ConstructorError(
+                None, None,
+                'expected a mapping node, but found %s' % node.id,
+                node.start_mark
+            )
+
+        mapping = OrderedDict()
+        for key_node, value_node in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            try:
+                hash(key)
+            except TypeError as exc:
+                raise yaml.constructor.ConstructorError(
+                    'while constructing a mapping',
+                    node.start_mark, 'found unacceptable key (%s)' % exc,
+                    key_node.start_mark
+                )
+            value = self.construct_object(value_node, deep=deep)
+            mapping[key] = value
+        return mapping
+
+    # Allow bare strings to begin with %. Directives are still detected.
+    def check_plain(self):
+        plain = super(Loader, self).check_plain()
+        return plain or self.peek() == '%'
+
+Loader.add_constructor('tag:yaml.org,2002:str', Loader._construct_unicode)
+Loader.add_constructor('tag:yaml.org,2002:map', Loader.construct_yaml_map)
+Loader.add_constructor('tag:yaml.org,2002:omap', Loader.construct_yaml_map)
+
+def load_yaml(filename):
+    """Read a YAML document from a file. If the file cannot be read or
+    parsed, a ConfigReadError is raised.
+    """
+    try:
+        with open(filename, 'r') as f:
+            return yaml.load(f, Loader=Loader)
+    except (IOError, yaml.error.YAMLError) as exc:
+        raise ConfigReadError(filename, exc)
+
+
+# YAML dumping.
+
+class Dumper(yaml.SafeDumper):
+    """A PyYAML Dumper that represents OrderedDicts as ordinary mappings
+    (in order, of course).
+    """
+    # From http://pyyaml.org/attachment/ticket/161/use_ordered_dict.py
+    def represent_mapping(self, tag, mapping, flow_style=None):
+        value = []
+        node = yaml.MappingNode(tag, value, flow_style=flow_style)
+        if self.alias_key is not None:
+            self.represented_objects[self.alias_key] = node
+        best_style = False
+        if hasattr(mapping, 'items'):
+            mapping = list(mapping.items())
+        for item_key, item_value in mapping:
+            node_key = self.represent_data(item_key)
+            node_value = self.represent_data(item_value)
+            if not (isinstance(node_key, yaml.ScalarNode)
+                    and not node_key.style):
+                best_style = False
+            if not (isinstance(node_value, yaml.ScalarNode)
+                    and not node_value.style):
+                best_style = False
+            value.append((node_key, node_value))
+        if flow_style is None:
+            if self.default_flow_style is not None:
+                node.flow_style = self.default_flow_style
+            else:
+                node.flow_style = best_style
+        return node
+
+    def represent_list(self, data):
+        """If a list has less than 4 items, represent it in inline style
+        (i.e. comma separated, within square brackets).
+        """
+        node = super(Dumper, self).represent_list(data)
+        length = len(data)
+        if self.default_flow_style is None and length < 4:
+            node.flow_style = True
+        elif self.default_flow_style is None:
+            node.flow_style = False
+        return node
+
+    def represent_bool(self, data):
+        """Represent bool as 'yes' or 'no' instead of 'true' or 'false'.
+        """
+        if data:
+            value = 'yes'
+        else:
+            value = 'no'
+        return self.represent_scalar('tag:yaml.org,2002:bool', value)
+
+    def represent_none(self, data):
+        """Represent a None value with nothing instead of 'none'.
+        """
+        return self.represent_scalar('tag:yaml.org,2002:null', '')
+
+Dumper.add_representer(OrderedDict, Dumper.represent_dict)
+Dumper.add_representer(bool, Dumper.represent_bool)
+Dumper.add_representer(type(None), Dumper.represent_none)
+Dumper.add_representer(list, Dumper.represent_list)
+
+def restore_yaml_comments(data, default_data):
+    """Scan default_data for comments (we include empty lines in our
+    definition of comments) and place them before the same keys in data.
+    Only works with comments that are on one or more own lines, i.e.
+    not next to a yaml mapping.
+    """
+    comment_map = dict()
+    default_lines = iter(default_data.splitlines())
+    for line in default_lines:
+        if not line:
+            comment = "\n"
+        elif line.startswith("#"):
+            comment = "{0}\n".format(line)
+        else:
+            continue
+        while True:
+            line = next(default_lines)
+            if line and not line.startswith("#"):
+                break
+            comment += "{0}\n".format(line)
+        key = line.split(':')[0].strip()
+        comment_map[key] = comment
+    out_lines = iter(data.splitlines())
+    out_data = ""
+    for line in out_lines:
+        key = line.split(':')[0].strip()
+        if key in comment_map:
+            out_data += comment_map[key]
+        out_data += "{0}\n".format(line)
+    return out_data
+
+
+# Main interface.
+
+class Configuration(RootView):
+    def __init__(self, appname, modname=None, read=True):
+        """Create a configuration object by reading the
+        automatically-discovered config files for the application for a
+        given name. If `modname` is specified, it should be the import
+        name of a module whose package will be searched for a default
+        config file. (Otherwise, no defaults are used.) Pass `False` for
+        `read` to disable automatic reading of all discovered
+        configuration files. Use this when creating a configuration
+        object at module load time and then call the `read` method
+        later.
+        """
+        super(Configuration, self).__init__([])
+        self.appname = appname
+        self.modname = modname
+
+        self._env_var = '{0}DIR'.format(self.appname.upper())
+
+        if read:
+            self.read()
+
+    def user_config_path(self):
+        """Points to the location of the user configuration.
+
+        The file may not exist.
+        """
+        return os.path.join(self.config_dir(), CONFIG_FILENAME)
+
+    def _add_user_source(self):
+        """Add the configuration options from the YAML file in the
+        user's configuration directory (given by `config_dir`) if it
+        exists.
+        """
+        filename = self.user_config_path()
+        if os.path.isfile(filename):
+            self.add(ConfigSource(load_yaml(filename) or {}, filename))
+
+    def _add_default_source(self):
+        """Add the package's default configuration settings. This looks
+        for a YAML file located inside the package for the module
+        `modname` if it was given.
+        """
+        if self.modname:
+            pkg_path = _package_path(self.modname)
+            if pkg_path:
+                filename = os.path.join(pkg_path, DEFAULT_FILENAME)
+                if os.path.isfile(filename):
+                    self.add(ConfigSource(load_yaml(filename), filename, True))
+
+    def read(self, user=True, defaults=True):
+        """Find and read the files for this configuration and set them
+        as the sources for this configuration. To disable either
+        discovered user configuration files or the in-package defaults,
+        set `user` or `defaults` to `False`.
+        """
+        if user:
+            self._add_user_source()
+        if defaults:
+            self._add_default_source()
+
+    def config_dir(self):
+        """Get the path to the user configuration directory. The
+        directory is guaranteed to exist as a postcondition (one may be
+        created if none exist).
+
+        If the application's ``...DIR`` environment variable is set, it
+        is used as the configuration directory. Otherwise,
+        platform-specific standard configuration locations are searched
+        for a ``config.yaml`` file. If no configuration file is found, a
+        fallback path is used.
+        """
+        # If environment variable is set, use it.
+        if self._env_var in os.environ:
+            appdir = os.environ[self._env_var]
+            appdir = os.path.abspath(os.path.expanduser(appdir))
+            if os.path.isfile(appdir):
+                raise ConfigError('{0} must be a directory'.format(
+                    self._env_var
+                ))
+
+        else:
+            # Search platform-specific locations. If no config file is
+            # found, fall back to the final directory in the list.
+            for confdir in config_dirs():
+                appdir = os.path.join(confdir, self.appname)
+                if os.path.isfile(os.path.join(appdir, CONFIG_FILENAME)):
+                    break
+
+        # Ensure that the directory exists.
+        if not os.path.isdir(appdir):
+            os.makedirs(appdir)
+        return appdir
+
+    def set_file(self, filename):
+        """Parses the file as YAML and inserts it into the configuration
+        sources with highest priority.
+        """
+        filename = os.path.abspath(filename)
+        self.set(ConfigSource(load_yaml(filename), filename))
+
+    def dump(self, full=True):
+        """Dump the Configuration object to a YAML file.
+
+        The order of the keys is determined from the default
+        configuration file. All keys not in the default configuration
+        will be appended to the end of the file.
+
+        :param filename:  The file to dump the configuration to, or None
+                          if the YAML string should be returned instead
+        :type filename:   unicode
+        :param full:      Dump settings that don't differ from the defaults
+                          as well
+        """
+        if full:
+            out_dict = self.flatten()
+        else:
+            # Exclude defaults when flattening.
+            sources = [s for s in self.sources if not s.default]
+            out_dict = RootView(sources).flatten()
+
+        yaml_out = yaml.dump(out_dict, Dumper=Dumper,
+                             default_flow_style=None, indent=4,
+                             width=1000)
+
+        # Restore comments to the YAML text.
+        default_source = None
+        for source in self.sources:
+            if source.default:
+                default_source = source
+                break
+        if default_source:
+            with open(default_source.filename, 'r') as fp:
+                default_data = fp.read()
+            yaml_out = restore_yaml_comments(yaml_out, default_data)
+
+        return yaml_out
+
+
+class LazyConfig(Configuration):
+    """A Configuration at reads files on demand when it is first
+    accessed. This is appropriate for using as a global config object at
+    the module level.
+    """
+    def __init__(self, appname, modname=None):
+        super(LazyConfig, self).__init__(appname, modname, False)
+        self._materialized = False  # Have we read the files yet?
+        self._lazy_prefix = []  # Pre-materialization calls to set().
+        self._lazy_suffix = []  # Calls to add().
+
+    def read(self, user=True, defaults=True):
+        self._materialized = True
+        super(LazyConfig, self).read(user, defaults)
+
+    def resolve(self):
+        if not self._materialized:
+            # Read files and unspool buffers.
+            self.read()
+            self.sources += self._lazy_suffix
+            self.sources[:0] = self._lazy_prefix
+        return super(LazyConfig, self).resolve()
+
+    def add(self, value):
+        super(LazyConfig, self).add(value)
+        if not self._materialized:
+            # Buffer additions to end.
+            self._lazy_suffix += self.sources
+            del self.sources[:]
+
+    def set(self, value):
+        super(LazyConfig, self).set(value)
+        if not self._materialized:
+            # Buffer additions to beginning.
+            self._lazy_prefix[:0] = self.sources
+            del self.sources[:]
+
+    def clear(self):
+        """Remove all sources from this configuration."""
+        del self.sources[:]
+        self._lazy_suffix = []
+        self._lazy_prefix = []

beets/util/enumeration.py

@@ -1,5 +1,5 @@
 # This file is part of beets.
-# Copyright 2011, Adrian Sampson.
+# Copyright 2013, Adrian Sampson.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the

beets/util/functemplate.py

@@ -1,5 +1,5 @@
 # This file is part of beets.
-# Copyright 2011, Adrian Sampson.
+# Copyright 2013, Adrian Sampson.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -496,6 +496,9 @@ class Template(object):
         self.original = template
         self.compiled = self.translate()
 
+    def __eq__(self, other):
+        return self.original == other.original
+
     def interpret(self, values={}, functions={}):
         """Like `substitute`, but forces the interpreter (rather than
         the compiled version) to be used. The interpreter includes

beets/util/pipeline.py

@@ -1,5 +1,5 @@
 # This file is part of beets.
-# Copyright 2011, Adrian Sampson.
+# Copyright 2013, Adrian Sampson.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -304,32 +304,18 @@ class Pipeline(object):
             raise ValueError('pipeline must have at least two stages')
         self.stages = []
         for stage in stages:
-            if isinstance(stage, types.GeneratorType):
+            if isinstance(stage, (list, tuple)):
+                self.stages.append(stage)
+            else:
                 # Default to one thread per stage.
                 self.stages.append((stage,))
-            else:
-                self.stages.append(stage)
 
     def run_sequential(self):
         """Run the pipeline sequentially in the current thread. The
         stages are run one after the other. Only the first coroutine
         in each stage is used.
         """
-        coros = [stage[0] for stage in self.stages]
-
-        # "Prime" the coroutines.
-        for coro in coros[1:]:
-            coro.next()
-
-        # Begin the pipeline.
-        for out in coros[0]:
-            msgs = _allmsgs(out)
-            for coro in coros[1:]:
-                next_msgs = []
-                for msg in msgs:
-                    out = coro.send(msg)
-                    next_msgs.extend(_allmsgs(out))
-                msgs = next_msgs
+        list(self.pull())
 
     def run_parallel(self, queue_size=DEFAULT_QUEUE_SIZE):
         """Run the pipeline in parallel using one thread per stage. The
@@ -386,6 +372,31 @@ class Pipeline(object):
                 # Make the exception appear as it was raised originally.
                 raise exc_info[0], exc_info[1], exc_info[2]
 
+    def pull(self):
+        """Yield elements from the end of the pipeline. Runs the stages
+        sequentially until the last yields some messages. Each of the messages
+        is then yielded by ``pulled.next()``. If the pipeline has a consumer,
+        that is the last stage does not yield any messages, then pull will not
+        yield any messages. Only the first coroutine in each stage is used
+        """
+        coros = [stage[0] for stage in self.stages]
+
+        # "Prime" the coroutines.
+        for coro in coros[1:]:
+            coro.next()
+
+        # Begin the pipeline.
+        for out in coros[0]:
+            msgs = _allmsgs(out)
+            for coro in coros[1:]:
+                next_msgs = []
+                for msg in msgs:
+                    out = coro.send(msg)
+                    next_msgs.extend(_allmsgs(out))
+                msgs = next_msgs
+            for msg in msgs:
+                yield msg
+
 # Smoke test.
 if __name__ == '__main__':
     import time
@@ -432,7 +443,7 @@ if __name__ == '__main__':
             print('processing %i' % num)
             time.sleep(3)
             if num == 3:
-               raise Exception()
+                raise Exception()
             num = yield num * 2
     def exc_consume():
         while True:

beets/vfs.py

@@ -1,5 +1,5 @@
 # This file is part of beets.
-# Copyright 2011, Adrian Sampson.
+# Copyright 2013, Adrian Sampson.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -16,7 +16,7 @@
 libraries.
 """
 from collections import namedtuple
-from lib.beets import util
+from beets import util
 
 Node = namedtuple('Node', ['files', 'dirs'])
 
@@ -42,7 +42,7 @@ def libtree(lib):
     """
     root = Node({}, {})
     for item in lib.items():
-        dest = lib.destination(item, fragment=True)
+        dest = item.destination(fragment=True)
         parts = util.components(dest)
         _insert(root, parts, item.id)
     return root

data/interfaces/default/artist.html

@@ -265,6 +265,7 @@
 			"aoColumnDefs": [
 			          { 'bSortable': false, 'aTargets': [ 0,1 ] }
 			],
+			"bStateSave": true,
 			"oLanguage": {
 				"sLengthMenu":"Show _MENU_ albums per page",
 				"sEmptyTable": "No album information available",

data/interfaces/default/config.html

@@ -702,6 +702,9 @@
                                         <label>Priority (-1,0, or 1):</label>
                                         <input type="text" name="pushover_priority" value="${config['pushover_priority']}" size="2">
                                     </div>
+                                    <div class="row">
+                                        <label>API Token (leave blank to use Headphones default):</label><input type="text" name="pushover_apitoken" value="${config['pushover_apitoken']}" size="50">
+                                    </div>
                                 </div>
                             </fieldset>
 
@@ -809,6 +812,18 @@
                                     <option value="xld" ${xldselect}>xld</option>
                                 </select>
                             </div>
+                            <div class="row">
+                                <div class="row">
+                                    <input type="checkbox" name="encoder_multicore" value="1" ${config['encoder_multicore']}/><label>Enable multi-core</label>
+                                </div>
+                            </div>
+                            <div class="row">
+                                <div class="row">
+                                    <label>Multi-core count:</label>
+                                    <input type="text" name="encoder_multicore_count" value="${config['encoder_multicore_count']}" size="7">
+                                    <small>Set equal to the number of cores, or 0 for auto</small>
+                                </div>
+                            </div>
                         </fieldset>
                         <fieldset>
                             <legend>Audio Properties</legend>

headphones/__init__.py

@@ -178,7 +178,7 @@ REQUIRED_WORDS = None
 
 LASTFM_USERNAME = None
 
-LOSSY_MEDIA_FORMATS = ["mp3", "aac", "ogg", "ape", "m4a"]
+LOSSY_MEDIA_FORMATS = ["mp3", "aac", "ogg", "ape", "m4a", "asf", "wma"]
 LOSSLESS_MEDIA_FORMATS = ["flac"]
 MEDIA_FORMATS = LOSSY_MEDIA_FORMATS + LOSSLESS_MEDIA_FORMATS
 
@@ -218,6 +218,8 @@ ENCODEROUTPUTFORMAT = None
 ENCODERQUALITY = None
 ENCODERVBRCBR = None
 ENCODERLOSSLESS = False
+ENCODER_MULTICORE = False
+ENCODER_MULTICORE_COUNT = 0
 DELETE_LOSSLESS_FILES = False
 GROWL_ENABLED = True
 GROWL_HOST = None
@@ -252,6 +254,7 @@ PUSHOVER_ENABLED = True
 PUSHOVER_PRIORITY = 1
 PUSHOVER_KEYS = None
 PUSHOVER_ONSNATCH = True
+PUSHOVER_APITOKEN = None
 PUSHBULLET_ENABLED = True
 PUSHBULLET_APIKEY = None
 PUSHBULLET_DEVICEID = None
@@ -341,8 +344,8 @@ def initialize():
                 NZBSORG, NZBSORG_UID, NZBSORG_HASH, NZBSRUS, NZBSRUS_UID, NZBSRUS_APIKEY, OMGWTFNZBS, OMGWTFNZBS_UID, OMGWTFNZBS_APIKEY, \
                 NZB_DOWNLOADER, TORRENT_DOWNLOADER, PREFERRED_WORDS, REQUIRED_WORDS, IGNORED_WORDS, LASTFM_USERNAME, \
                 INTERFACE, FOLDER_PERMISSIONS, FILE_PERMISSIONS, ENCODERFOLDER, ENCODER_PATH, ENCODER, XLDPROFILE, BITRATE, SAMPLINGFREQUENCY, \
-                MUSIC_ENCODER, ADVANCEDENCODER, ENCODEROUTPUTFORMAT, ENCODERQUALITY, ENCODERVBRCBR, ENCODERLOSSLESS, DELETE_LOSSLESS_FILES, \
-                GROWL_ENABLED, GROWL_HOST, GROWL_PASSWORD, GROWL_ONSNATCH, PROWL_ENABLED, PROWL_PRIORITY, PROWL_KEYS, PROWL_ONSNATCH, PUSHOVER_ENABLED, PUSHOVER_PRIORITY, PUSHOVER_KEYS, PUSHOVER_ONSNATCH, MIRRORLIST, \
+                MUSIC_ENCODER, ADVANCEDENCODER, ENCODEROUTPUTFORMAT, ENCODERQUALITY, ENCODERVBRCBR, ENCODERLOSSLESS, ENCODER_MULTICORE, ENCODER_MULTICORE_COUNT, DELETE_LOSSLESS_FILES, \
+                GROWL_ENABLED, GROWL_HOST, GROWL_PASSWORD, GROWL_ONSNATCH, PROWL_ENABLED, PROWL_PRIORITY, PROWL_KEYS, PROWL_ONSNATCH, PUSHOVER_ENABLED, PUSHOVER_PRIORITY, PUSHOVER_KEYS, PUSHOVER_ONSNATCH, PUSHOVER_APITOKEN, MIRRORLIST, \
                 TWITTER_ENABLED, TWITTER_ONSNATCH, TWITTER_USERNAME, TWITTER_PASSWORD, TWITTER_PREFIX, \
                 PUSHBULLET_ENABLED, PUSHBULLET_APIKEY, PUSHBULLET_DEVICEID, PUSHBULLET_ONSNATCH, \
                 MIRROR, CUSTOMHOST, CUSTOMPORT, CUSTOMSLEEP, HPUSER, HPPASS, XBMC_ENABLED, XBMC_HOST, XBMC_USERNAME, XBMC_PASSWORD, XBMC_UPDATE, \
@@ -539,6 +542,8 @@ def initialize():
         ENCODERQUALITY = check_setting_int(CFG, 'General', 'encoderquality', 2)
         ENCODERVBRCBR = check_setting_str(CFG, 'General', 'encodervbrcbr', 'cbr')
         ENCODERLOSSLESS = bool(check_setting_int(CFG, 'General', 'encoderlossless', 1))
+        ENCODER_MULTICORE = bool(check_setting_int(CFG, 'General', 'encoder_multicore', 0))
+        ENCODER_MULTICORE_COUNT = max(0, check_setting_int(CFG, 'General', 'encoder_multicore_count', 0))
         DELETE_LOSSLESS_FILES = bool(check_setting_int(CFG, 'General', 'delete_lossless_files', 1))
 
         GROWL_ENABLED = bool(check_setting_int(CFG, 'Growl', 'growl_enabled', 0))
@@ -581,6 +586,7 @@ def initialize():
         PUSHOVER_KEYS = check_setting_str(CFG, 'Pushover', 'pushover_keys', '')
         PUSHOVER_ONSNATCH = bool(check_setting_int(CFG, 'Pushover', 'pushover_onsnatch', 0))
         PUSHOVER_PRIORITY = check_setting_int(CFG, 'Pushover', 'pushover_priority', 0)
+        PUSHOVER_APITOKEN = check_setting_str(CFG, 'Pushover', 'pushover_apitoken', '')
 
         PUSHBULLET_ENABLED = bool(check_setting_int(CFG, 'PushBullet', 'pushbullet_enabled', 0))
         PUSHBULLET_APIKEY = check_setting_str(CFG, 'PushBullet', 'pushbullet_apikey', '')
@@ -983,6 +989,7 @@ def config_write():
     new_config['Pushover']['pushover_keys'] = PUSHOVER_KEYS
     new_config['Pushover']['pushover_onsnatch'] = int(PUSHOVER_ONSNATCH)
     new_config['Pushover']['pushover_priority'] = int(PUSHOVER_PRIORITY)
+    new_config['Pushover']['pushover_apitoken'] = PUSHOVER_APITOKEN
 
     new_config['PushBullet'] = {}
     new_config['PushBullet']['pushbullet_enabled'] = int(PUSHBULLET_ENABLED)
@@ -1022,6 +1029,8 @@ def config_write():
     new_config['General']['encoderquality'] = ENCODERQUALITY
     new_config['General']['encodervbrcbr'] = ENCODERVBRCBR
     new_config['General']['encoderlossless'] = int(ENCODERLOSSLESS)
+    new_config['General']['encoder_multicore'] = int(ENCODER_MULTICORE)
+    new_config['General']['encoder_multicore_count'] = int(ENCODER_MULTICORE_COUNT)
     new_config['General']['delete_lossless_files'] = int(DELETE_LOSSLESS_FILES)
 
     new_config['General']['mirror'] = MIRROR

headphones/helpers.py

@@ -18,8 +18,16 @@ from operator import itemgetter
 import datetime
 import re, shutil
 
+from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError
+
 import headphones
 
+# Modified from https://github.com/Verrus/beets-plugin-featInTitle
+RE_FEATURING = re.compile(r"[fF]t\.|[fF]eaturing|[fF]eat\.|\b[wW]ith\b|&|vs\.")
+
+RE_CD_ALBUM = re.compile(r"\(?((CD|disc)\s*[0-9]+)\)", re.I)
+RE_CD = re.compile(r"^(CD|dics)\s*[0-9]+$", re.I)
+
 def multikeysort(items, columns):
 
     comparers = [ ((itemgetter(col[1:].strip()), -1) if col.startswith('-') else (itemgetter(col.strip()), 1)) for col in columns]
@@ -209,7 +217,100 @@ def extract_data(s):
         return (name, album, year)
     else:
         return (None, None, None)
-        
+
+def extract_metadata(f):
+    """
+    Scan all files in the given directory and decide on an artist, album and
+    year based on the metadata. A decision is based on the number of different
+    artists, albums and years found in the media files.
+    """
+
+    from headphones import logger
+
+    # Walk directory and scan all media files
+    results = []
+    count = 0
+
+    for root, dirs, files in os.walk(f):
+        for file in files:
+            # Count the number of potential media files
+            extension = os.path.splitext(file)[1].lower()[1:]
+
+            if extension in headphones.MEDIA_FORMATS:
+                count += 1
+
+            # Try to read the file info
+            try:
+                media_file = MediaFile(os.path.join(root, file))
+            except FileTypeError, UnreadableFileError:
+                # Probably not a media file
+                continue
+
+            # Append metadata to file
+            artist = media_file.albumartist or media_file.artist
+            album = media_file.album
+            year = media_file.year
+
+            if artist and album and year:
+                results.append((artist.lower(), album.lower(), year))
+
+    # Verify results
+    if len(results) == 0:
+        logger.info("No metadata in media files found, ignoring")
+        return (None, None, None)
+
+    # Require that some percentage of files have tags
+    count_ratio = 0.75
+
+    if count < (count_ratio * len(results)):
+        logger.info("Counted %d media files, but only %d have tags, ignoring" % (count, len(results)))
+        return (None, None, None)
+
+    # Count distinct values
+    artists = list(set([ x[0] for x in results ]))
+    albums = list(set([ x[1] for x in results ]))
+    years = list(set([ x[2] for x in results ]))
+
+    # Remove things such as CD2 from album names
+    if len(albums) > 1:
+        new_albums = list(albums)
+
+        # Replace occurences of e.g. CD1
+        for index, album in enumerate(new_albums):
+            if RE_CD_ALBUM.search(album):
+                new_albums[index] = RE_CD_ALBUM.sub("", album).strip()
+
+        # Remove duplicates
+        new_albums = list(set(new_albums))
+
+        # Safety check: if nothing has merged, then ignore the work. This can
+        # happen if only one CD of a multi part CD is processed.
+        if len(new_albums) < len(albums):
+            albums = new_albums
+
+    # All files have the same metadata, so it's trivial
+    if len(artists) == 1 and len(albums) == 1 and len(years) == 1:
+        return (artists[0], albums[0], years[0])
+
+    # (Lots of) different artists. Could be a featuring album, so test for this.
+    if len(artists) > 1 and len(albums) == 1 and len(years) == 1:
+        split_artists = [ RE_FEATURING.split(artist) for artist in artists ]
+        featurings = [ len(split_artist) - 1 for split_artist in split_artists ]
+        logger.info("Album seem to feature %d different artists" % sum(featurings))
+
+        if sum(featurings) > 0:
+            # Find the artist of which the least splits have been generated.
+            # Ideally, this should be 0, which should be the album artist
+            # itself.
+            artist = split_artists[featurings.index(min(featurings))][0]
+
+            # Done
+            return (artist, albums[0], years[0])
+
+    # Not sure what to do here.
+    logger.info("Found %d artists, %d albums and %d years in metadata, ignoring" % (len(artists), len(albums), len(years)))
+    return (None, None, None)
+
 def extract_logline(s):
     # Default log format
     pattern = re.compile(r'(?P<timestamp>.*?)\s\-\s(?P<level>.*?)\s*\:\:\s(?P<thread>.*?)\s\:\s(?P<message>.*)', re.VERBOSE)

headphones/importer.py

@@ -17,7 +17,7 @@ from lib.pyItunes import *
 import time
 import threading
 import os
-from lib.beets.mediafile import MediaFile
+from beets.mediafile import MediaFile
 
 import headphones
 from headphones import logger, helpers, db, mb, albumart, lastfm

headphones/librarysync.py

@@ -16,7 +16,7 @@
 import os
 import glob
 
-from lib.beets.mediafile import MediaFile
+from beets.mediafile import MediaFile
 
 import headphones
 from headphones import db, logger, helpers, importer

headphones/music_encoder.py

@@ -17,10 +17,11 @@ import os
 import headphones
 import shutil
 import time
+import multiprocessing
 
 import subprocess
 from headphones import logger
-from lib.beets.mediafile import MediaFile
+from beets.mediafile import MediaFile
 
 try:
     import argparse
@@ -100,6 +101,7 @@ def encode(albumPath):
 
     i=0
     encoder_failed = False
+    jobs = []
 
     for music in musicFiles:        
         infoMusic=MediaFile(music)
@@ -131,15 +133,45 @@ def encode(albumPath):
                     encode = True
         # encode
         if encode:
-            if not command(encoder,music,musicTempFiles[i],albumPath):
-                encoder_failed = True
-                break
+            job = (encoder, music, musicTempFiles[i], albumPath)
+            jobs.append(job)
         else:
             musicFiles[i] = None
             musicTempFiles[i] = None
 
         i=i+1
 
+    # Encode music files
+    if len(jobs) > 0:
+        if headphones.ENCODER_MULTICORE:
+            if headphones.ENCODER_MULTICORE_COUNT == 0:
+                processes = multiprocessing.cpu_count()
+            else:
+                processes = headphones.ENCODER_MULTICORE_COUNT
+
+            logger.debug("Multi-core encoding enabled, %d processes" % processes)
+        else:
+            processes = 1
+
+        # Use multiprocessing only if it's worth the overhead. and if it is
+        # enabled. If not, then use the old fashioned way.
+        if processes > 1:
+            pool = multiprocessing.Pool(processes=processes)
+            results = pool.map_async(command_map, jobs)
+
+            # No new processes will be created, so close it and wait for all
+            # processes to finish
+            pool.close()
+            pool.join()
+
+            # Retrieve the results
+            results = results.get()
+        else:
+            results = map(command_map, jobs)
+
+        # The results are either True or False, so determine if one is False
+        encoder_failed = not all(results)
+
     musicFiles = filter(None, musicFiles)
     musicTempFiles = filter(None, musicTempFiles)
 
@@ -187,7 +219,10 @@ def encode(albumPath):
         logger.info('Encoding for folder %s is not required' % (albumPath.decode(headphones.SYS_ENCODING, 'replace')))
     
     return musicFinalFiles
-    
+
+def command_map(args):
+    return command(*args)
+
 def command(encoder,musicSource,musicDest,albumPath):
 
     cmd=[]

headphones/notifiers.py

@@ -538,6 +538,8 @@ class PUSHOVER:
         self.enabled = headphones.PUSHOVER_ENABLED
         self.keys = headphones.PUSHOVER_KEYS
         self.priority = headphones.PUSHOVER_PRIORITY   
+        if headphones.PUSHOVER_APITOKEN:
+            self.application_token = headphones.PUSHOVER_APITOKEN
         pass
 
     def conf(self, options):

headphones/postprocessor.py

@@ -22,9 +22,9 @@ import music_encoder
 import urllib, shutil, re
 import uuid
 from headphones import notifiers
-import lib.beets as beets
-from lib.beets import autotag
-from lib.beets.mediafile import MediaFile
+import beets
+from beets import autotag
+from beets.mediafile import MediaFile
 
 import headphones
 from headphones import db, albumart, librarysync, lyrics, logger, helpers
@@ -960,15 +960,17 @@ def forcePostProcess():
     myDB = db.DBConnection()
     
     for folder in folders:
-
         folder_basename = os.path.basename(folder).decode(headphones.SYS_ENCODING, 'replace')
-
         logger.info('Processing: %s' % folder_basename)
-        
-        # First try to see if there's a match in the snatched table, then we'll try to parse the foldername
-        # TODO: Iterate through underscores -> spaces, spaces -> dots, underscores -> dots (this might be hit or miss since it assumes
-        # all spaces/underscores came from sab replacing values
+
+        # Attempt 1: First try to see if there's a match in the snatched table,
+        # then we'll try to parse the foldername.
+        # TODO: Iterate through underscores -> spaces, spaces -> dots,
+        # underscores -> dots (this might be hit or miss since it assumes all
+        # spaces/underscores came from sab replacing values
+        logger.debug('Attempting to find album in the snatched table')
         snatched = myDB.action('SELECT AlbumID, Title, Kind, Status from snatched WHERE FolderName LIKE ?', [folder_basename]).fetchone()
+
         if snatched:
             if headphones.KEEP_TORRENT_FILES and snatched['Kind'] == 'torrent' and snatched['Status'] == 'Processed':
                 logger.info(folder_basename + ' is a torrent folder being preserved for seeding and has already been processed. Skipping.')
@@ -977,16 +979,23 @@ def forcePostProcess():
                 logger.info('Found a match in the database: %s. Verifying to make sure it is the correct album' % snatched['Title'])
                 verify(snatched['AlbumID'], folder, snatched['Kind'])
                 continue
-        
-        # Try to parse the folder name into a valid format
-        # TODO: Add metadata lookup
+
+        # Attempt 2a: parse the folder name into a valid format
         try:
+            logger.debug('Attempting to extract name, album and year from folder name')
             name, album, year = helpers.extract_data(folder_basename)
-        except:
+        except Exception as e:
             name = None
 
+        # Attempt 2b: deduce meta data into a valid format
+        if name is None:
+            try:
+                logger.debug('Attempting to extract name, album and year from metadata')
+                name, album, year = helpers.extract_metadata(folder)
+            except Exception as e:
+                name = None
+
         if name and album and year:
-            
             release = myDB.action('SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?', [name, album]).fetchone()
             if release:
                 logger.info('Found a match in the database: %s - %s. Verifying to make sure it is the correct album' % (release['ArtistName'], release['AlbumTitle']))
@@ -998,30 +1007,28 @@ def forcePostProcess():
                     rgid = mb.findAlbumID(helpers.latinToAscii(name), helpers.latinToAscii(album))
                 except:
                     logger.error('Can not get release information for this album')
-                    continue
                 if rgid:
                     verify(rgid, folder)
+                    continue
                 else:
                     logger.info('No match found on MusicBrainz for: %s - %s' % (name, album))
-                    continue
-                    
-        else:
-            try:
-                possible_rgid = folder_basename[-36:]
-                # re pattern match: [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} 
-                rgid = uuid.UUID(possible_rgid)
-                    
-            except:
-                logger.info("Couldn't parse " + folder_basename + " into any valid format. If adding albums from another source, they must be in an 'Artist - Album [Year]' format, or end with the musicbrainz release group id")
-                continue
-            
-            
-            if rgid:
-                rgid = possible_rgid
-                release = myDB.action('SELECT ArtistName, AlbumTitle, AlbumID from albums WHERE AlbumID=?', [rgid]).fetchone()
-                if release:
-                    logger.info('Found a match in the database: %s - %s. Verifying to make sure it is the correct album' % (release['ArtistName'], release['AlbumTitle']))
-                    verify(release['AlbumID'], folder, forced=True)
-                else:
-                    logger.info('Found a (possibly) valid Musicbrainz identifier in album folder name - continuing post-processing')
-                    verify(rgid, folder, forced=True)
+
+        # Attempt 3: strip release group id from filename
+        try:
+            logger.debug('Attempting to extract release group from folder name')
+            possible_rgid = folder_basename[-36:]
+            # re pattern match: [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
+            rgid = uuid.UUID(possible_rgid)
+        except:
+            logger.info("Couldn't parse " + folder_basename + " into any valid format. If adding albums from another source, they must be in an 'Artist - Album [Year]' format, or end with the musicbrainz release group id")
+            rgid = None
+
+        if rgid:
+            rgid = possible_rgid
+            release = myDB.action('SELECT ArtistName, AlbumTitle, AlbumID from albums WHERE AlbumID=?', [rgid]).fetchone()
+            if release:
+                logger.info('Found a match in the database: %s - %s. Verifying to make sure it is the correct album' % (release['ArtistName'], release['AlbumTitle']))
+                verify(release['AlbumID'], folder, forced=True)
+            else:
+                logger.info('Found a (possibly) valid Musicbrainz identifier in album folder name - continuing post-processing')
+                verify(rgid, folder, forced=True)

headphones/transmission.py

@@ -59,8 +59,8 @@ def addTorrent(link):
             prowl.notify(name,"Download started")
         if headphones.PUSHOVER_ENABLED and headphones.PUSHOVER_ONSNATCH:
             logger.info(u"Sending Pushover notification")
-            prowl = notifiers.PUSHOVER()
-            prowl.notify(name,"Download started")
+            pushover = notifiers.PUSHOVER()
+            pushover.notify(name,"Download started")
         if headphones.TWITTER_ENABLED and headphones.TWITTER_ONSNATCH:
             logger.info(u"Sending Twitter notification")
             twitter = notifiers.TwitterNotifier()

headphones/webserve.py

@@ -930,6 +930,8 @@ class WebInterface(object):
                     "encodervbrcbr": headphones.ENCODERVBRCBR,
                     "encoderquality": headphones.ENCODERQUALITY,
                     "encoderlossless": checked(headphones.ENCODERLOSSLESS),
+                    "encoder_multicore": checked(headphones.ENCODER_MULTICORE),
+                    "encoder_multicore_count": int(headphones.ENCODER_MULTICORE_COUNT),
                     "delete_lossless_files": checked(headphones.DELETE_LOSSLESS_FILES),
                     "growl_enabled": checked(headphones.GROWL_ENABLED),
                     "growl_onsnatch": checked(headphones.GROWL_ONSNATCH),
@@ -963,6 +965,7 @@ class WebInterface(object):
                     "pushover_enabled": checked(headphones.PUSHOVER_ENABLED),
                     "pushover_onsnatch": checked(headphones.PUSHOVER_ONSNATCH),
                     "pushover_keys": headphones.PUSHOVER_KEYS,
+                    "pushover_apitoken": headphones.PUSHOVER_APITOKEN,
                     "pushover_priority": headphones.PUSHOVER_PRIORITY,
                     "pushbullet_enabled": checked(headphones.PUSHBULLET_ENABLED),
                     "pushbullet_onsnatch": checked(headphones.PUSHBULLET_ONSNATCH),
@@ -1016,10 +1019,10 @@ class WebInterface(object):
         bitrate=None, samplingfrequency=None, encoderfolder=None, advancedencoder=None, encoderoutputformat=None, encodervbrcbr=None, encoderquality=None, encoderlossless=0,
         delete_lossless_files=0, growl_enabled=0, growl_onsnatch=0, growl_host=None, growl_password=None, prowl_enabled=0, prowl_onsnatch=0, prowl_keys=None, prowl_priority=0, xbmc_enabled=0, xbmc_host=None, xbmc_username=None, xbmc_password=None,
         xbmc_update=0, xbmc_notify=0, nma_enabled=False, nma_apikey=None, nma_priority=0, nma_onsnatch=0, pushalot_enabled=False, pushalot_apikey=None, pushalot_onsnatch=0, synoindex_enabled=False,
-        pushover_enabled=0, pushover_onsnatch=0, pushover_keys=None, pushover_priority=0, pushbullet_enabled=0, pushbullet_onsnatch=0, pushbullet_apikey=None, pushbullet_deviceid=None, twitter_enabled=0, twitter_onsnatch=0, mirror=None, customhost=None, customport=None,
+        pushover_enabled=0, pushover_onsnatch=0, pushover_keys=None, pushover_priority=0, pushover_apitoken=None, pushbullet_enabled=0, pushbullet_onsnatch=0, pushbullet_apikey=None, pushbullet_deviceid=None, twitter_enabled=0, twitter_onsnatch=0, mirror=None, customhost=None, customport=None,
         customsleep=None, hpuser=None, hppass=None, preferred_bitrate_high_buffer=None, preferred_bitrate_low_buffer=None, preferred_bitrate_allow_lossless=0, cache_sizemb=None, 
         enable_https=0, https_cert=None, https_key=None, file_permissions=None, folder_permissions=None, plex_enabled=0, plex_server_host=None, plex_client_host=None, plex_username=None, 
-        plex_password=None, plex_update=0, plex_notify=0, songkick_enabled=0, songkick_apikey=None, songkick_location=None, songkick_filter_enabled=0, **kwargs):
+        plex_password=None, plex_update=0, plex_notify=0, songkick_enabled=0, songkick_apikey=None, songkick_location=None, songkick_filter_enabled=0, encoder_multicore=False, encoder_multicore_count=0, **kwargs):
 
         headphones.HTTP_HOST = http_host
         headphones.HTTP_PORT = http_port
@@ -1127,6 +1130,8 @@ class WebInterface(object):
         headphones.ENCODERVBRCBR = encodervbrcbr
         headphones.ENCODERQUALITY = int(encoderquality)
         headphones.ENCODERLOSSLESS = int(encoderlossless)
+        headphones.ENCODER_MULTICORE = encoder_multicore
+        headphones.ENCODER_MULTICORE_COUNT = max(0, int(encoder_multicore_count))
         headphones.DELETE_LOSSLESS_FILES = int(delete_lossless_files)
         headphones.GROWL_ENABLED = growl_enabled
         headphones.GROWL_ONSNATCH = growl_onsnatch
@@ -1161,6 +1166,7 @@ class WebInterface(object):
         headphones.PUSHOVER_ONSNATCH = pushover_onsnatch
         headphones.PUSHOVER_KEYS = pushover_keys
         headphones.PUSHOVER_PRIORITY = pushover_priority
+        headphones.PUSHOVER_APITOKEN = pushover_apitoken
         headphones.PUSHBULLET_ENABLED = pushbullet_enabled
         headphones.PUSHBULLET_ONSNATCH = pushbullet_onsnatch
         headphones.PUSHBULLET_APIKEY = pushbullet_apikey

lib/beets/autotag/__init__.py

@@ -1,191 +0,0 @@
-# This file is part of beets.
-# Copyright 2012, Adrian Sampson.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-
-"""Facilities for automatically determining files' correct metadata.
-"""
-import os
-import logging
-import re
-
-from lib.beets import library, mediafile
-from lib.beets.util import sorted_walk, ancestry
-
-# Parts of external interface.
-from .hooks import AlbumInfo, TrackInfo, AlbumMatch, TrackMatch
-from .match import AutotagError
-from .match import tag_item, tag_album
-from .match import RECOMMEND_STRONG, RECOMMEND_MEDIUM, RECOMMEND_NONE
-from .match import STRONG_REC_THRESH, MEDIUM_REC_THRESH, REC_GAP_THRESH
-
-# Global logger.
-log = logging.getLogger('beets')
-
-# Constants for directory walker.
-MULTIDISC_MARKERS = (r'part', r'volume', r'vol\.', r'disc', r'cd')
-MULTIDISC_PAT_FMT = r'%s\s*\d'
-
-
-# Additional utilities for the main interface.
-
-def albums_in_dir(path, ignore=()):
-    """Recursively searches the given directory and returns an iterable
-    of (path, items) where path is a containing directory and items is
-    a list of Items that is probably an album. Specifically, any folder
-    containing any media files is an album. Directories and file names
-    that match the glob patterns in ``ignore`` are skipped.
-    """
-    collapse_root = None
-    collapse_items = None
-
-    for root, dirs, files in sorted_walk(path, ignore):
-        # Get a list of items in the directory.
-        items = []
-        for filename in files:
-            try:
-                i = library.Item.from_path(os.path.join(root, filename))
-            except mediafile.FileTypeError:
-                pass
-            except mediafile.UnreadableFileError:
-                log.warn('unreadable file: ' + filename)
-            else:
-                items.append(i)
-
-        # If we're collapsing, test to see whether we should continue to
-        # collapse. If so, just add to the collapsed item set;
-        # otherwise, end the collapse and continue as normal.
-        if collapse_root is not None:
-            if collapse_root in ancestry(root):
-                # Still collapsing.
-                collapse_items += items
-                continue
-            else:
-                # Collapse finished. Yield the collapsed directory and
-                # proceed to process the current one.
-                if collapse_items:
-                    yield collapse_root, collapse_items
-                collapse_root = collapse_items = None
-
-        # Does the current directory look like a multi-disc album? If
-        # so, begin collapsing here.
-        if dirs and not items: # Must be only directories.
-            multidisc = False
-            for marker in MULTIDISC_MARKERS:
-                pat = MULTIDISC_PAT_FMT % marker
-                if all(re.search(pat, dirname, re.I) for dirname in dirs):
-                    multidisc = True
-                    break
-
-            # This becomes True only when all directories match a
-            # pattern for a single marker.
-            if multidisc:
-                # Start collapsing; continue to the next iteration.
-                collapse_root = root
-                collapse_items = []
-                continue
-
-        # If it's nonempty, yield it.
-        if items:
-            yield root, items
-
-    # Clear out any unfinished collapse.
-    if collapse_root is not None and collapse_items:
-        yield collapse_root, collapse_items
-
-def apply_item_metadata(item, track_info):
-    """Set an item's metadata from its matched TrackInfo object.
-    """
-    item.artist = track_info.artist
-    item.artist_sort = track_info.artist_sort
-    item.artist_credit = track_info.artist_credit
-    item.title = track_info.title
-    item.mb_trackid = track_info.track_id
-    if track_info.artist_id:
-        item.mb_artistid = track_info.artist_id
-    # At the moment, the other metadata is left intact (including album
-    # and track number). Perhaps these should be emptied?
-
-def apply_metadata(album_info, mapping, per_disc_numbering=False):
-    """Set the items' metadata to match an AlbumInfo object using a
-    mapping from Items to TrackInfo objects. If `per_disc_numbering`,
-    then the track numbers are per-disc instead of per-release.
-    """
-    for item, track_info in mapping.iteritems():
-        # Album, artist, track count.
-        if not item:
-            continue
-        if track_info.artist:
-            item.artist = track_info.artist
-        else:
-            item.artist = album_info.artist
-        item.albumartist = album_info.artist
-        item.album = album_info.album
-        item.tracktotal = len(album_info.tracks)
-
-        # Artist sort and credit names.
-        item.artist_sort = track_info.artist_sort or album_info.artist_sort
-        item.artist_credit = track_info.artist_credit or \
-                             album_info.artist_credit
-        item.albumartist_sort = album_info.artist_sort
-        item.albumartist_credit = album_info.artist_credit
-
-        # Release date.
-        if album_info.year:
-            item.year = album_info.year
-        if album_info.month:
-            item.month = album_info.month
-        if album_info.day:
-            item.day = album_info.day
-
-        # Title.
-        item.title = track_info.title
-
-        if per_disc_numbering:
-            item.track = track_info.medium_index
-        else:
-            item.track = track_info.index
-
-        # Disc and disc count.
-        item.disc = track_info.medium
-        item.disctotal = album_info.mediums
-
-        # MusicBrainz IDs.
-        item.mb_trackid = track_info.track_id
-        item.mb_albumid = album_info.album_id
-        if track_info.artist_id:
-            item.mb_artistid = track_info.artist_id
-        else:
-            item.mb_artistid = album_info.artist_id
-        item.mb_albumartistid = album_info.artist_id
-        item.mb_releasegroupid = album_info.releasegroup_id
-
-        # Compilation flag.
-        item.comp = album_info.va
-
-        # Miscellaneous metadata.
-        item.albumtype = album_info.albumtype
-        if album_info.label:
-            item.label = album_info.label
-        
-        item.asin = album_info.asin
-        item.catalognum = album_info.catalognum
-        item.script = album_info.script
-        item.language = album_info.language
-        item.country = album_info.country
-        item.albumstatus = album_info.albumstatus
-        item.media = album_info.media
-        item.albumdisambig = album_info.albumdisambig
-        item.disctitle = track_info.disctitle
-        
-        # Headphones seal of approval
-        item.comments = 'tagged by headphones/beets'

lib/beets/autotag/art.py

@@ -1,152 +0,0 @@
-# This file is part of beets.
-# Copyright 2011, Adrian Sampson.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-# 
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-
-"""Finding album art for tagged albums."""
-
-import urllib
-import sys
-import logging
-import os
-import re
-
-from lib.beets.autotag.mb import album_for_id
-
-IMAGE_EXTENSIONS = ['png', 'jpg', 'jpeg']
-COVER_NAMES = ['cover', 'front', 'art', 'album', 'folder']
-
-# The common logger.
-log = logging.getLogger('beets')
-
-
-CONTENT_TYPES = ('image/jpeg',)
-def _fetch_image(url):
-    """Downloads an image from a URL and checks whether it seems to
-    actually be an image. If so, returns a path to the downloaded image.
-    Otherwise, returns None.
-    """
-    log.debug('Downloading art: %s' % url)
-    try:
-        fn, headers = urllib.urlretrieve(url)
-    except IOError:
-        log.debug('error fetching art')
-        return
-
-    # Make sure it's actually an image.
-    if headers.gettype() in CONTENT_TYPES:
-        log.debug('Downloaded art to: %s' % fn)
-        return fn
-    else:
-        log.debug('Not an image.')
-
-
-# Art from Amazon.
-
-AMAZON_URL = 'http://images.amazon.com/images/P/%s.%02i.LZZZZZZZ.jpg'
-AMAZON_INDICES = (1,2)
-def art_for_asin(asin):
-    """Fetches art for an Amazon ID (ASIN) string."""
-    for index in AMAZON_INDICES:
-        # Fetch the image.
-        url = AMAZON_URL % (asin, index)
-        fn = _fetch_image(url)
-        if fn:
-            return fn
-
-
-# AlbumArt.org scraper.
-
-AAO_URL = 'http://www.albumart.org/index_detail.php'
-AAO_PAT = r'href\s*=\s*"([^>"]*)"[^>]*title\s*=\s*"View larger image"'
-def aao_art(asin):
-    # Get the page from albumart.org.
-    url = '%s?%s' % (AAO_URL, urllib.urlencode({'asin': asin}))
-    try:
-        log.debug('Scraping art URL: %s' % url)
-        page = urllib.urlopen(url).read()
-    except IOError:
-        log.debug('Error scraping art page')
-        return
-
-    # Search the page for the image URL.
-    m = re.search(AAO_PAT, page)
-    if m:
-        image_url = m.group(1)
-        return _fetch_image(image_url)
-    else:
-        log.debug('No image found on page')
-
-
-# Art from the filesystem.
-
-def art_in_path(path):
-    """Look for album art files in a specified directory."""
-    if not os.path.isdir(path):
-        return
-
-    # Find all files that look like images in the directory.
-    images = []
-    for fn in os.listdir(path):
-        for ext in IMAGE_EXTENSIONS:
-            if fn.lower().endswith('.' + ext):
-                images.append(fn)
-
-    # Look for "preferred" filenames.
-    for fn in images:
-        for name in COVER_NAMES:
-            if fn.lower().startswith(name):
-                log.debug('Using well-named art file %s' % fn)
-                return os.path.join(path, fn)
-
-    # Fall back to any image in the folder.
-    if images:
-        log.debug('Using fallback art file %s' % images[0])
-        return os.path.join(path, images[0])
-
-
-# Main interface.
-
-def art_for_album(album, path):
-    """Given an album info dictionary from MusicBrainz, returns a path
-    to downloaded art for the album (or None if no art is found).
-    """
-    if isinstance(path, basestring):
-        out = art_in_path(path)
-        if out:
-            return out
-
-    if album.asin:
-        log.debug('Fetching album art for ASIN %s.' % album.asin)
-        out = art_for_asin(album.asin)
-        if out:
-            return out
-        return aao_art(album.asin)
-    else:
-        log.debug('No ASIN available: no art found.')
-        return None
-
-
-# Smoke test.
-
-if __name__ == '__main__':
-    aid = sys.argv[1]
-    album = album_for_id(aid)
-    if not album:
-        print 'album not found'
-    else:
-        fn = art_for_album(album, None)
-        if fn:
-            print fn
-            print len(open(fn).read())/1024
-        else:
-            print 'no art found'

lib/beets/autotag/hooks.py

@@ -1,190 +0,0 @@
-# This file is part of beets.
-# Copyright 2012, Adrian Sampson.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-
-"""Glue between metadata sources and the matching logic."""
-import logging
-from collections import namedtuple
-
-from lib.beets import plugins
-from lib.beets.autotag import mb
-
-log = logging.getLogger('beets')
-
-
-# Classes used to represent candidate options.
-
-class AlbumInfo(object):
-    """Describes a canonical release that may be used to match a release
-    in the library. Consists of these data members:
-
-    - ``album``: the release title
-    - ``album_id``: MusicBrainz ID; UUID fragment only
-    - ``artist``: name of the release's primary artist
-    - ``artist_id``
-    - ``tracks``: list of TrackInfo objects making up the release
-    - ``asin``: Amazon ASIN
-    - ``albumtype``: string describing the kind of release
-    - ``va``: boolean: whether the release has "various artists"
-    - ``year``: release year
-    - ``month``: release month
-    - ``day``: release day
-    - ``label``: music label responsible for the release
-    - ``mediums``: the number of discs in this release
-    - ``artist_sort``: name of the release's artist for sorting
-    - ``releasegroup_id``: MBID for the album's release group
-    - ``catalognum``: the label's catalog number for the release
-    - ``script``: character set used for metadata
-    - ``language``: human language of the metadata
-    - ``country``: the release country
-    - ``albumstatus``: MusicBrainz release status (Official, etc.)
-    - ``media``: delivery mechanism (Vinyl, etc.)
-    - ``albumdisambig``: MusicBrainz release disambiguation comment
-    - ``artist_credit``: Release-specific artist name
-
-    The fields up through ``tracks`` are required. The others are
-    optional and may be None.
-    """
-    def __init__(self, album, album_id, artist, artist_id, tracks, asin=None,
-                 albumtype=None, va=False, year=None, month=None, day=None,
-                 label=None, mediums=None, artist_sort=None,
-                 releasegroup_id=None, catalognum=None, script=None,
-                 language=None, country=None, albumstatus=None, media=None,
-                 albumdisambig=None, artist_credit=None):
-        self.album = album
-        self.album_id = album_id
-        self.artist = artist
-        self.artist_id = artist_id
-        self.tracks = tracks
-        self.asin = asin
-        self.albumtype = albumtype
-        self.va = va
-        self.year = year
-        self.month = month
-        self.day = day
-        self.label = label
-        self.mediums = mediums
-        self.artist_sort = artist_sort
-        self.releasegroup_id = releasegroup_id
-        self.catalognum = catalognum
-        self.script = script
-        self.language = language
-        self.country = country
-        self.albumstatus = albumstatus
-        self.media = media
-        self.albumdisambig = albumdisambig
-        self.artist_credit = artist_credit
-
-class TrackInfo(object):
-    """Describes a canonical track present on a release. Appears as part
-    of an AlbumInfo's ``tracks`` list. Consists of these data members:
-
-    - ``title``: name of the track
-    - ``track_id``: MusicBrainz ID; UUID fragment only
-    - ``artist``: individual track artist name
-    - ``artist_id``
-    - ``length``: float: duration of the track in seconds
-    - ``index``: position on the entire release
-    - ``medium``: the disc number this track appears on in the album
-    - ``medium_index``: the track's position on the disc
-    - ``artist_sort``: name of the track artist for sorting
-    - ``disctitle``: name of the individual medium (subtitle)
-    - ``artist_credit``: Recording-specific artist name
-
-    Only ``title`` and ``track_id`` are required. The rest of the fields
-    may be None. The indices ``index``, ``medium``, and ``medium_index``
-    are all 1-based.
-    """
-    def __init__(self, title, track_id, artist=None, artist_id=None,
-                 length=None, index=None, medium=None, medium_index=None,
-                 artist_sort=None, disctitle=None, artist_credit=None):
-        self.title = title
-        self.track_id = track_id
-        self.artist = artist
-        self.artist_id = artist_id
-        self.length = length
-        self.index = index
-        self.medium = medium
-        self.medium_index = medium_index
-        self.artist_sort = artist_sort
-        self.disctitle = disctitle
-        self.artist_credit = artist_credit
-
-AlbumMatch = namedtuple('AlbumMatch', ['distance', 'info', 'mapping',
-                                       'extra_items', 'extra_tracks'])
-
-TrackMatch = namedtuple('TrackMatch', ['distance', 'info'])
-
-
-# Aggregation of sources.
-
-def _album_for_id(album_id):
-    """Get an album corresponding to a MusicBrainz release ID."""
-    try:
-        return mb.album_for_id(album_id)
-    except mb.MusicBrainzAPIError as exc:
-        exc.log(log)
-
-def _track_for_id(track_id):
-    """Get an item for a recording MBID."""
-    try:
-        return mb.track_for_id(track_id)
-    except mb.MusicBrainzAPIError as exc:
-        exc.log(log)
-
-def _album_candidates(items, artist, album, va_likely):
-    """Search for album matches. ``items`` is a list of Item objects
-    that make up the album. ``artist`` and ``album`` are the respective
-    names (strings), which may be derived from the item list or may be
-    entered by the user. ``va_likely`` is a boolean indicating whether
-    the album is likely to be a "various artists" release.
-    """
-    out = []
-
-    # Base candidates if we have album and artist to match.
-    if artist and album:
-        try:
-            out.extend(mb.match_album(artist, album, len(items)))
-        except mb.MusicBrainzAPIError as exc:
-            exc.log(log)
-
-    # Also add VA matches from MusicBrainz where appropriate.
-    if va_likely and album:
-        try:
-            out.extend(mb.match_album(None, album, len(items)))
-        except mb.MusicBrainzAPIError as exc:
-            exc.log(log)
-
-    # Candidates from plugins.
-    out.extend(plugins.candidates(items))
-
-    return out
-
-def _item_candidates(item, artist, title):
-    """Search for item matches. ``item`` is the Item to be matched.
-    ``artist`` and ``title`` are strings and either reflect the item or
-    are specified by the user.
-    """
-    out = []
-
-    # MusicBrainz candidates.
-    if artist and title:
-        try:
-            out.extend(mb.match_track(artist, title))
-        except mb.MusicBrainzAPIError as exc:
-            exc.log(log)
-
-    # Plugin candidates.
-    out.extend(plugins.item_candidates(item))
-
-    return out

lib/beets/autotag/match.py

@@ -1,497 +0,0 @@
-# This file is part of beets.
-# Copyright 2012, Adrian Sampson.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-
-"""Matches existing metadata with canonical information to identify
-releases and tracks.
-"""
-from __future__ import division
-
-import logging
-import re
-from lib.munkres import Munkres
-#from unidecode import unidecode
-
-from lib.beets import plugins
-from lib.beets.util import levenshtein, plurality
-from lib.beets.autotag import hooks
-
-# Distance parameters.
-# Text distance weights: proportions on the normalized intuitive edit
-# distance.
-ARTIST_WEIGHT = 3.0
-ALBUM_WEIGHT = 3.0
-# The weight of the entire distance calculated for a given track.
-TRACK_WEIGHT = 1.0
-# The weight of a missing track.
-MISSING_WEIGHT = 0.9
-# The weight of an extra (umatched) track.
-UNMATCHED_WEIGHT = 0.6
-# These distances are components of the track distance (that is, they
-# compete against each other but not ARTIST_WEIGHT and ALBUM_WEIGHT;
-# the overall TRACK_WEIGHT does that).
-TRACK_TITLE_WEIGHT = 3.0
-# Used instead of a global artist penalty for various-artist matches.
-TRACK_ARTIST_WEIGHT = 2.0
-# Added when the indices of tracks don't match.
-TRACK_INDEX_WEIGHT = 1.0
-# Track length weights: no penalty before GRACE, maximum (WEIGHT)
-# penalty at GRACE+MAX discrepancy.
-TRACK_LENGTH_GRACE = 10
-TRACK_LENGTH_MAX = 30
-TRACK_LENGTH_WEIGHT = 2.0
-# MusicBrainz track ID matches.
-TRACK_ID_WEIGHT = 5.0
-
-# Parameters for string distance function.
-# Words that can be moved to the end of a string using a comma.
-SD_END_WORDS = ['the', 'a', 'an']
-# Reduced weights for certain portions of the string.
-SD_PATTERNS = [
-    (r'^the ', 0.1),
-    (r'[\[\(]?(ep|single)[\]\)]?', 0.0),
-    (r'[\[\(]?(featuring|feat|ft)[\. :].+', 0.1),
-    (r'\(.*?\)', 0.3),
-    (r'\[.*?\]', 0.3),
-    (r'(, )?(pt\.|part) .+', 0.2),
-]
-# Replacements to use before testing distance.
-SD_REPLACE = [
-    (r'&', 'and'),
-]
-
-# Recommendation constants.
-RECOMMEND_STRONG = 'RECOMMEND_STRONG'
-RECOMMEND_MEDIUM = 'RECOMMEND_MEDIUM'
-RECOMMEND_NONE = 'RECOMMEND_NONE'
-# Thresholds for recommendations.
-STRONG_REC_THRESH = 0.04
-MEDIUM_REC_THRESH = 0.25
-REC_GAP_THRESH = 0.25
-
-# Artist signals that indicate "various artists". These are used at the
-# album level to determine whether a given release is likely a VA
-# release and also on the track level to to remove the penalty for
-# differing artists.
-VA_ARTISTS = (u'', u'various artists', u'va', u'unknown')
-
-# Autotagging exceptions.
-class AutotagError(Exception):
-    pass
-
-# Global logger.
-log = logging.getLogger('beets')
-
-
-# Primary matching functionality.
-
-def _string_dist_basic(str1, str2):
-    """Basic edit distance between two strings, ignoring
-    non-alphanumeric characters and case. Comparisons are based on a
-    transliteration/lowering to ASCII characters. Normalized by string
-    length.
-    """
-    #str1 = unidecode(str1)
-    #str2 = unidecode(str2)
-    str1 = re.sub(r'[^a-z0-9]', '', str1.lower())
-    str2 = re.sub(r'[^a-z0-9]', '', str2.lower())
-    if not str1 and not str2:
-        return 0.0
-    return levenshtein(str1, str2) / float(max(len(str1), len(str2)))
-
-def string_dist(str1, str2):
-    """Gives an "intuitive" edit distance between two strings. This is
-    an edit distance, normalized by the string length, with a number of
-    tweaks that reflect intuition about text.
-    """
-    str1 = str1.lower()
-    str2 = str2.lower()
-
-    # Don't penalize strings that move certain words to the end. For
-    # example, "the something" should be considered equal to
-    # "something, the".
-    for word in SD_END_WORDS:
-        if str1.endswith(', %s' % word):
-            str1 = '%s %s' % (word, str1[:-len(word)-2])
-        if str2.endswith(', %s' % word):
-            str2 = '%s %s' % (word, str2[:-len(word)-2])
-
-    # Perform a couple of basic normalizing substitutions.
-    for pat, repl in SD_REPLACE:
-        str1 = re.sub(pat, repl, str1)
-        str2 = re.sub(pat, repl, str2)
-
-    # Change the weight for certain string portions matched by a set
-    # of regular expressions. We gradually change the strings and build
-    # up penalties associated with parts of the string that were
-    # deleted.
-    base_dist = _string_dist_basic(str1, str2)
-    penalty = 0.0
-    for pat, weight in SD_PATTERNS:
-        # Get strings that drop the pattern.
-        case_str1 = re.sub(pat, '', str1)
-        case_str2 = re.sub(pat, '', str2)
-
-        if case_str1 != str1 or case_str2 != str2:
-            # If the pattern was present (i.e., it is deleted in the
-            # the current case), recalculate the distances for the
-            # modified strings.
-            case_dist = _string_dist_basic(case_str1, case_str2)
-            case_delta = max(0.0, base_dist - case_dist)
-            if case_delta == 0.0:
-                continue
-
-            # Shift our baseline strings down (to avoid rematching the
-            # same part of the string) and add a scaled distance
-            # amount to the penalties.
-            str1 = case_str1
-            str2 = case_str2
-            base_dist = case_dist
-            penalty += weight * case_delta
-    dist = base_dist + penalty
-
-    return dist
-
-def current_metadata(items):
-    """Returns the most likely artist and album for a set of Items.
-    Each is determined by tag reflected by the plurality of the Items.
-    """
-    keys = 'artist', 'album'
-    likelies = {}
-    consensus = {}
-    for key in keys:
-        values = [getattr(item, key) for item in items if item]
-        likelies[key], freq = plurality(values)
-        consensus[key] = (freq == len(values))
-    return likelies['artist'], likelies['album'], consensus['artist']
-
-def assign_items(items, tracks):
-    """Given a list of Items and a list of TrackInfo objects, find the
-    best mapping between them. Returns a mapping from Items to TrackInfo
-    objects, a set of extra Items, and a set of extra TrackInfo
-    objects. These "extra" objects occur when there is an unequal number
-    of objects of the two types.
-    """
-    # Construct the cost matrix.
-    costs = []
-    for item in items:
-        row = []
-        for i, track in enumerate(tracks):
-            row.append(track_distance(item, track))
-        costs.append(row)
-
-    # Find a minimum-cost bipartite matching.
-    matching = Munkres().compute(costs)
-
-    # Produce the output matching.
-    mapping = dict((items[i], tracks[j]) for (i, j) in matching)
-    extra_items = set(items) - set(mapping.keys())
-    extra_tracks = set(tracks) - set(mapping.values())
-    return mapping, extra_items, extra_tracks
-
-def track_distance(item, track_info, incl_artist=False):
-    """Determines the significance of a track metadata change. Returns a
-    float in [0.0,1.0]. `incl_artist` indicates that a distance
-    component should be included for the track artist (i.e., for
-    various-artist releases).
-    """
-    # Distance and normalization accumulators.
-    dist, dist_max = 0.0, 0.0
-
-    # Check track length.
-    # If there's no length to check, apply no penalty.
-    if track_info.length:
-        diff = abs(item.length - track_info.length)
-        diff = max(diff - TRACK_LENGTH_GRACE, 0.0)
-        diff = min(diff, TRACK_LENGTH_MAX)
-        dist += (diff / TRACK_LENGTH_MAX) * TRACK_LENGTH_WEIGHT
-    dist_max += TRACK_LENGTH_WEIGHT
-
-    # Track title.
-    dist += string_dist(item.title, track_info.title) * TRACK_TITLE_WEIGHT
-    dist_max += TRACK_TITLE_WEIGHT
-
-    # Track artist, if included.
-    # Attention: MB DB does not have artist info for all compilations,
-    # so only check artist distance if there is actually an artist in
-    # the MB track data.
-    if incl_artist and track_info.artist and \
-            item.artist.lower() not in VA_ARTISTS:
-        dist += string_dist(item.artist, track_info.artist) * \
-                TRACK_ARTIST_WEIGHT
-        dist_max += TRACK_ARTIST_WEIGHT
-
-    # Track index.
-    if track_info.index and item.track:
-        if item.track not in (track_info.index, track_info.medium_index):
-            dist += TRACK_INDEX_WEIGHT
-        dist_max += TRACK_INDEX_WEIGHT
-
-    # MusicBrainz track ID.
-    if item.mb_trackid:
-        if item.mb_trackid != track_info.track_id:
-            dist += TRACK_ID_WEIGHT
-        dist_max += TRACK_ID_WEIGHT
-
-    # Plugin distances.
-    plugin_d, plugin_dm = plugins.track_distance(item, track_info)
-    dist += plugin_d
-    dist_max += plugin_dm
-
-    return dist / dist_max
-
-def distance(items, album_info, mapping):
-    """Determines how "significant" an album metadata change would be.
-    Returns a float in [0.0,1.0]. `album_info` is an AlbumInfo object
-    reflecting the album to be compared. `items` is a sequence of all
-    Item objects that will be matched (order is not important).
-    `mapping` is a dictionary mapping Items to TrackInfo objects; the
-    keys are a subset of `items` and the values are a subset of
-    `album_info.tracks`.
-    """
-    cur_artist, cur_album, _ = current_metadata(items)
-    cur_artist = cur_artist or ''
-    cur_album = cur_album or ''
-
-    # These accumulate the possible distance components. The final
-    # distance will be dist/dist_max.
-    dist = 0.0
-    dist_max = 0.0
-
-    # Artist/album metadata.
-    if not album_info.va:
-        dist += string_dist(cur_artist, album_info.artist) * ARTIST_WEIGHT
-        dist_max += ARTIST_WEIGHT
-    dist += string_dist(cur_album,  album_info.album) * ALBUM_WEIGHT
-    dist_max += ALBUM_WEIGHT
-
-    # Matched track distances.
-    for item, track in mapping.iteritems():
-        dist += track_distance(item, track, album_info.va) * TRACK_WEIGHT
-        dist_max += TRACK_WEIGHT
-
-    # Extra and unmatched tracks.
-    for track in set(album_info.tracks) - set(mapping.values()):
-        dist += MISSING_WEIGHT
-        dist_max += MISSING_WEIGHT
-    for item in set(items) - set(mapping.keys()):
-        dist += UNMATCHED_WEIGHT
-        dist_max += UNMATCHED_WEIGHT
-
-    # Plugin distances.
-    plugin_d, plugin_dm = plugins.album_distance(items, album_info)
-    dist += plugin_d
-    dist_max += plugin_dm
-
-    # Normalize distance, avoiding divide-by-zero.
-    if dist_max == 0.0:
-        return 0.0
-    else:
-        return dist / dist_max
-
-def match_by_id(items):
-    """If the items are tagged with a MusicBrainz album ID, returns an
-    AlbumInfo object for the corresponding album. Otherwise, returns
-    None.
-    """
-    # Is there a consensus on the MB album ID?
-    albumids = [item.mb_albumid for item in items if item.mb_albumid]
-    if not albumids:
-        log.debug('No album IDs found.')
-        return None
-
-    # If all album IDs are equal, look up the album.
-    if bool(reduce(lambda x,y: x if x==y else (), albumids)):
-        albumid = albumids[0]
-        log.debug('Searching for discovered album ID: ' + albumid)
-        return hooks._album_for_id(albumid)
-    else:
-        log.debug('No album ID consensus.')
-        return None
-
-    #fixme In the future, at the expense of performance, we could use
-    # other IDs (i.e., track and artist) in case the album tag isn't
-    # present, but that event seems very unlikely.
-
-def recommendation(results):
-    """Given a sorted list of AlbumMatch or TrackMatch objects, return a
-    recommendation flag (RECOMMEND_STRONG, RECOMMEND_MEDIUM,
-    RECOMMEND_NONE) based on the results' distances.
-    """
-    if not results:
-        # No candidates: no recommendation.
-        rec = RECOMMEND_NONE
-    else:
-        min_dist = results[0].distance
-        if min_dist < STRONG_REC_THRESH:
-            # Strong recommendation level.
-            rec = RECOMMEND_STRONG
-        elif len(results) == 1:
-            # Only a single candidate. Medium recommendation.
-            rec = RECOMMEND_MEDIUM
-        elif min_dist <= MEDIUM_REC_THRESH:
-            # Medium recommendation level.
-            rec = RECOMMEND_MEDIUM
-        elif results[1].distance - min_dist >= REC_GAP_THRESH:
-            # Gap between first two candidates is large.
-            rec = RECOMMEND_MEDIUM
-        else:
-            # No conclusion.
-            rec = RECOMMEND_NONE
-    return rec
-
-def _add_candidate(items, results, info):
-    """Given a candidate AlbumInfo object, attempt to add the candidate
-    to the output dictionary of AlbumMatch objects. This involves
-    checking the track count, ordering the items, checking for
-    duplicates, and calculating the distance.
-    """
-    log.debug('Candidate: %s - %s' % (info.artist, info.album))
-
-    # Don't duplicate.
-    if info.album_id in results:
-        log.debug('Duplicate.')
-        return
-
-    # Find mapping between the items and the track info.
-    mapping, extra_items, extra_tracks = assign_items(items, info.tracks)
-
-    # Get the change distance.
-    dist = distance(items, info, mapping)
-    log.debug('Success. Distance: %f' % dist)
-
-    results[info.album_id] = hooks.AlbumMatch(dist, info, mapping,
-                                              extra_items, extra_tracks)
-
-def tag_album(items, timid=False, search_artist=None, search_album=None,
-              search_id=None):
-    """Bundles together the functionality used to infer tags for a
-    set of items comprised by an album. Returns everything relevant:
-        - The current artist.
-        - The current album.
-        - A list of AlbumMatch objects. The candidates are sorted by
-        distance (i.e., best match first).
-        - A recommendation, one of RECOMMEND_STRONG, RECOMMEND_MEDIUM,
-          or RECOMMEND_NONE; indicating that the first candidate is
-          very likely, it is somewhat likely, or no conclusion could
-          be reached.
-    If search_artist and search_album or search_id are provided, then
-    they are used as search terms in place of the current metadata.
-    May raise an AutotagError if existing metadata is insufficient.
-    """
-    # Get current metadata.
-    cur_artist, cur_album, artist_consensus = current_metadata(items)
-    log.debug('Tagging %s - %s' % (cur_artist, cur_album))
-
-    # The output result (distance, AlbumInfo) tuples (keyed by MB album
-    # ID).
-    candidates = {}
-
-    # Try to find album indicated by MusicBrainz IDs.
-    if search_id:
-        log.debug('Searching for album ID: ' + search_id)
-        id_info = hooks._album_for_id(search_id)
-    else:
-        id_info = match_by_id(items)
-    if id_info:
-        _add_candidate(items, candidates, id_info)
-        rec = recommendation(candidates.values())
-        log.debug('Album ID match recommendation is ' + str(rec))
-        if candidates and not timid:
-            # If we have a very good MBID match, return immediately.
-            # Otherwise, this match will compete against metadata-based
-            # matches.
-            if rec == RECOMMEND_STRONG:
-                log.debug('ID match.')
-                return cur_artist, cur_album, candidates.values(), rec
-
-    # If searching by ID, don't continue to metadata search.
-    if search_id is not None:
-        if candidates:
-            return cur_artist, cur_album, candidates.values(), rec
-        else:
-            return cur_artist, cur_album, [], RECOMMEND_NONE
-
-    # Search terms.
-    if not (search_artist and search_album):
-        # No explicit search terms -- use current metadata.
-        search_artist, search_album = cur_artist, cur_album
-    log.debug(u'Search terms: %s - %s' % (search_artist, search_album))
-
-    # Is this album likely to be a "various artist" release?
-    va_likely = ((not artist_consensus) or
-                 (search_artist.lower() in VA_ARTISTS) or
-                 any(item.comp for item in items))
-    log.debug(u'Album might be VA: %s' % str(va_likely))
-
-    # Get the results from the data sources.
-    search_cands = hooks._album_candidates(items, search_artist, search_album,
-                                           va_likely)
-    log.debug(u'Evaluating %i candidates.' % len(search_cands))
-    for info in search_cands:
-        _add_candidate(items, candidates, info)
-
-    # Sort and get the recommendation.
-    candidates = sorted(candidates.itervalues())
-    rec = recommendation(candidates)
-    return cur_artist, cur_album, candidates, rec
-
-def tag_item(item, timid=False, search_artist=None, search_title=None,
-             search_id=None):
-    """Attempts to find metadata for a single track. Returns a
-    `(candidates, recommendation)` pair where `candidates` is a list of
-    TrackMatch objects. `search_artist` and `search_title` may be used
-    to override the current metadata for the purposes of the MusicBrainz
-    title; likewise `search_id`.
-    """
-    # Holds candidates found so far: keys are MBIDs; values are
-    # (distance, TrackInfo) pairs.
-    candidates = {}
-
-    # First, try matching by MusicBrainz ID.
-    trackid = search_id or item.mb_trackid
-    if trackid:
-        log.debug('Searching for track ID: ' + trackid)
-        track_info = hooks._track_for_id(trackid)
-        if track_info:
-            dist = track_distance(item, track_info, incl_artist=True)
-            candidates[track_info.track_id] = \
-                    hooks.TrackMatch(dist, track_info)
-            # If this is a good match, then don't keep searching.
-            rec = recommendation(candidates.values())
-            if rec == RECOMMEND_STRONG and not timid:
-                log.debug('Track ID match.')
-                return candidates.values(), rec
-
-    # If we're searching by ID, don't proceed.
-    if search_id is not None:
-        if candidates:
-            return candidates.values(), rec
-        else:
-            return [], RECOMMEND_NONE
-
-    # Search terms.
-    if not (search_artist and search_title):
-        search_artist, search_title = item.artist, item.title
-    log.debug(u'Item search terms: %s - %s' % (search_artist, search_title))
-
-    # Get and evaluate candidate metadata.
-    for track_info in hooks._item_candidates(item, search_artist, search_title):
-        dist = track_distance(item, track_info, incl_artist=True)
-        candidates[track_info.track_id] = hooks.TrackMatch(dist, track_info)
-
-    # Sort by distance and return with recommendation.
-    log.debug('Found %i candidates.' % len(candidates))
-    candidates = sorted(candidates.itervalues())
-    rec = recommendation(candidates)
-    return candidates, rec

lib/munkres.py

@@ -61,7 +61,7 @@ You could then use that index matrix to loop over the original cost matrix
 and calculate the smallest cost of the combinations::
 
     n = len(matrix)
-    minval = sys.maxint
+    minval = sys.maxsize
     for row in range(n):
         cost = 0
         for col in range(n):
@@ -163,7 +163,7 @@ large value. For example::
     for row in matrix:
         cost_row = []
         for col in row:
-            cost_row += [sys.maxint - col]
+            cost_row += [sys.maxsize - col]
         cost_matrix += [cost_row]
 
     m = Munkres()
@@ -197,7 +197,7 @@ creation of the cost matrix::
     import munkres
 
     cost_matrix = munkres.make_cost_matrix(matrix,
-                                           lambda cost: sys.maxint - cost)
+                                           lambda cost: sys.maxsize - cost)
 
 So, the above profit-calculation program can be recast as::
 
@@ -206,7 +206,7 @@ So, the above profit-calculation program can be recast as::
     matrix = [[5, 9, 1],
               [10, 3, 2],
               [8, 7, 4]]
-    cost_matrix = make_cost_matrix(matrix, lambda cost: sys.maxint - cost)
+    cost_matrix = make_cost_matrix(matrix, lambda cost: sys.maxsize - cost)
     m = Munkres()
     indexes = m.compute(cost_matrix)
     print_matrix(matrix, msg='Lowest cost through this matrix:')
@@ -277,6 +277,7 @@ __docformat__ = 'restructuredtext'
 # ---------------------------------------------------------------------------
 
 import sys
+import copy
 
 # ---------------------------------------------------------------------------
 # Exports
@@ -289,7 +290,7 @@ __all__     = ['Munkres', 'make_cost_matrix']
 # ---------------------------------------------------------------------------
 
 # Info about the module
-__version__   = "1.0.5.4"
+__version__   = "1.0.6"
 __author__    = "Brian Clapper, bmc@clapper.org"
 __url__       = "http://software.clapper.org/munkres/"
 __copyright__ = "(c) 2008 Brian M. Clapper"
@@ -458,8 +459,8 @@ class Munkres:
         for i in range(n):
             for j in range(n):
                 if (self.C[i][j] == 0) and \
-                   (not self.col_covered[j]) and \
-                   (not self.row_covered[i]):
+                        (not self.col_covered[j]) and \
+                        (not self.row_covered[i]):
                     self.marked[i][j] = 1
                     self.col_covered[j] = True
                     self.row_covered[i] = True
@@ -575,7 +576,7 @@ class Munkres:
 
     def __find_smallest(self):
         """Find the smallest uncovered value in the matrix."""
-        minval = sys.maxint
+        minval = sys.maxsize
         for i in range(self.n):
             for j in range(self.n):
                 if (not self.row_covered[i]) and (not self.col_covered[j]):
@@ -595,8 +596,8 @@ class Munkres:
             j = 0
             while True:
                 if (self.C[i][j] == 0) and \
-                   (not self.row_covered[i]) and \
-                   (not self.col_covered[j]):
+                        (not self.row_covered[i]) and \
+                        (not self.col_covered[j]):
                     row = i
                     col = j
                     done = True
@@ -690,7 +691,7 @@ def make_cost_matrix(profit_matrix, inversion_function):
 
     .. python::
 
-        cost_matrix = Munkres.make_cost_matrix(matrix, lambda x : sys.maxint - x)
+        cost_matrix = Munkres.make_cost_matrix(matrix, lambda x : sys.maxsize - x)
 
     :Parameters:
         profit_matrix : list of lists
@@ -721,7 +722,7 @@ def print_matrix(matrix, msg=None):
     import math
 
     if msg is not None:
-        print msg
+        print(msg)
 
     # Calculate the appropriate format width.
     width = 0
@@ -746,36 +747,31 @@ def print_matrix(matrix, msg=None):
 
 if __name__ == '__main__':
 
-
     matrices = [
-                # Square
-                ([[400, 150, 400],
-                  [400, 450, 600],
-                  [300, 225, 300]],
-                 850 # expected cost
-                ),
+        # Square
+        ([[400, 150, 400],
+          [400, 450, 600],
+          [300, 225, 300]],
+         850),  # expected cost
 
-                # Rectangular variant
-                ([[400, 150, 400, 1],
-                  [400, 450, 600, 2],
-                  [300, 225, 300, 3]],
-                 452 # expected cost
-                ),
+        # Rectangular variant
+        ([[400, 150, 400, 1],
+          [400, 450, 600, 2],
+          [300, 225, 300, 3]],
+         452),  # expected cost
 
-                # Square
-                ([[10, 10,  8],
-                  [ 9,  8,  1],
-                  [ 9,  7,  4]],
-                 18
-                ),
 
-                # Rectangular variant
-                ([[10, 10,  8, 11],
-                  [ 9,  8,  1, 1],
-                  [ 9,  7,  4, 10]],
-                 15
-                ),
-               ]
+        # Square
+        ([[10, 10,  8],
+          [9,  8,  1],
+          [9,  7,  4]],
+         18),
+
+        # Rectangular variant
+        ([[10, 10,  8, 11],
+          [9,  8,  1, 1],
+          [9,  7,  4, 10]],
+         15)]
 
     m = Munkres()
     for cost_matrix, expected_total in matrices:
@@ -785,7 +781,6 @@ if __name__ == '__main__':
         for r, c in indexes:
             x = cost_matrix[r][c]
             total_cost += x
-            print '(%d, %d) -> %d' % (r, c, x)
-        print 'lowest cost=%d' % total_cost
+            print('(%d, %d) -> %d' % (r, c, x))
+        print('lowest cost=%d' % total_cost)
         assert expected_total == total_cost
-

lib/mutagen/musepack.py

@@ -1,118 +0,0 @@
-# A Musepack reader/tagger
-#
-# Copyright 2006 Lukas Lalinsky <lalinsky@gmail.com>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# $Id: musepack.py 4013 2007-04-23 09:18:22Z luks $
-
-"""Musepack audio streams with APEv2 tags.
-
-Musepack is an audio format originally based on the MPEG-1 Layer-2
-algorithms. Stream versions 4 through 7 are supported.
-
-For more information, see http://www.musepack.net/.
-"""
-
-__all__ = ["Musepack", "Open", "delete"]
-
-import struct
-
-from lib.mutagen.apev2 import APEv2File, error, delete
-from lib.mutagen.id3 import BitPaddedInt
-from lib.mutagen._util import cdata
-
-class MusepackHeaderError(error): pass
-
-RATES = [44100, 48000, 37800, 32000]
-
-class MusepackInfo(object):
-    """Musepack stream information.
-
-    Attributes:
-    channels -- number of audio channels
-    length -- file length in seconds, as a float
-    sample_rate -- audio sampling rate in Hz
-    bitrate -- audio bitrate, in bits per second 
-    version -- Musepack stream version
-
-    Optional Attributes:
-    title_gain, title_peak -- Replay Gain and peak data for this song
-    album_gain, album_peak -- Replay Gain and peak data for this album
-
-    These attributes are only available in stream version 7. The
-    gains are a float, +/- some dB. The peaks are a percentage [0..1] of
-    the maximum amplitude. This means to get a number comparable to
-    VorbisGain, you must multiply the peak by 2.
-    """
-
-    def __init__(self, fileobj):
-        header = fileobj.read(32)
-        if len(header) != 32:
-            raise MusepackHeaderError("not a Musepack file")
-        # Skip ID3v2 tags
-        if header[:3] == "ID3":
-            size = 10 + BitPaddedInt(header[6:10])
-            fileobj.seek(size)
-            header = fileobj.read(32)
-            if len(header) != 32:
-                raise MusepackHeaderError("not a Musepack file")
-        # SV7
-        if header.startswith("MP+"):
-            self.version = ord(header[3]) & 0xF
-            if self.version < 7:
-                raise MusepackHeaderError("not a Musepack file")
-            frames = cdata.uint_le(header[4:8])
-            flags = cdata.uint_le(header[8:12])
-
-            self.title_peak, self.title_gain = struct.unpack(
-                "<Hh", header[12:16])
-            self.album_peak, self.album_gain = struct.unpack(
-                "<Hh", header[16:20])
-            self.title_gain /= 100.0
-            self.album_gain /= 100.0
-            self.title_peak /= 65535.0
-            self.album_peak /= 65535.0
-
-            self.sample_rate = RATES[(flags >> 16) & 0x0003]
-            self.bitrate = 0
-        # SV4-SV6
-        else:
-            header_dword = cdata.uint_le(header[0:4])
-            self.version = (header_dword >> 11) & 0x03FF;
-            if self.version < 4 or self.version > 6:
-                raise MusepackHeaderError("not a Musepack file")
-            self.bitrate = (header_dword >> 23) & 0x01FF;
-            self.sample_rate = 44100
-            if self.version >= 5:
-                frames = cdata.uint_le(header[4:8])
-            else:
-                frames = cdata.ushort_le(header[6:8])
-            if self.version < 6:
-                frames -= 1
-        self.channels = 2
-        self.length = float(frames * 1152 - 576) / self.sample_rate
-        if not self.bitrate and self.length != 0:
-            fileobj.seek(0, 2)
-            self.bitrate = int(fileobj.tell() * 8 / (self.length * 1000) + 0.5)
-
-    def pprint(self):
-        if self.version >= 7:
-            rg_data = ", Gain: %+0.2f (title), %+0.2f (album)" %(
-                self.title_gain, self.album_gain)
-        else:
-            rg_data = ""
-        return "Musepack, %.2f seconds, %d Hz%s" % (
-            self.length, self.sample_rate, rg_data)
-
-class Musepack(APEv2File):
-    _Info = MusepackInfo
-    _mimes = ["audio/x-musepack", "audio/x-mpc"]
-
-    def score(filename, fileobj, header):
-        return header.startswith("MP+") + filename.endswith(".mpc")
-    score = staticmethod(score)
-
-Open = Musepack

lib/unidecode/__init__.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+"""Transliterate Unicode text into plain 7-bit ASCII.
+
+Example usage:
+>>> from unidecode import unidecode:
+>>> unidecode(u"\u5317\u4EB0")
+"Bei Jing "
+
+The transliteration uses a straightforward map, and doesn't have alternatives
+for the same character based on language, position, or anything else.
+
+In Python 3, a standard string object will be returned. If you need bytes, use:
+>>> unidecode("Κνωσός").encode("ascii")
+b'Knosos'
+"""
+import warnings
+from sys import version_info
+
+Cache = {}
+
+def unidecode(string):
+    """Transliterate an Unicode object into an ASCII string
+
+    >>> unidecode(u"\u5317\u4EB0")
+    "Bei Jing "
+    """
+
+    if version_info[0] < 3 and not isinstance(string, unicode):
+        warnings.warn(  "Argument %r is not an unicode object. "
+                        "Passing an encoded string will likely have "
+                        "unexpected results." % (type(string),),
+			RuntimeWarning, 2)
+
+    retval = []
+
+    for char in string:
+        codepoint = ord(char)
+
+        if codepoint < 0x80: # Basic ASCII
+            retval.append(str(char))
+            continue
+        
+        if codepoint > 0xeffff:
+            continue # Characters in Private Use Area and above are ignored
+
+        section = codepoint >> 8   # Chop off the last two hex digits
+        position = codepoint % 256 # Last two hex digits
+
+        try:
+            table = Cache[section]
+        except KeyError:
+            try:
+                mod = __import__('unidecode.x%03x'%(section), [], [], ['data'])
+            except ImportError:
+                Cache[section] = None
+                continue   # No match: ignore this character and carry on.
+
+            Cache[section] = table = mod.data
+
+        if table and len(table) > position:
+            retval.append( table[position] )
+
+    return ''.join(retval)

lib/unidecode/x000.py

@@ -0,0 +1,258 @@
+data = (
+'\x00',    # 0x00
+'\x01',    # 0x01
+'\x02',    # 0x02
+'\x03',    # 0x03
+'\x04',    # 0x04
+'\x05',    # 0x05
+'\x06',    # 0x06
+'\x07',    # 0x07
+'\x08',    # 0x08
+'\x09',    # 0x09
+'\x0a',    # 0x0a
+'\x0b',    # 0x0b
+'\x0c',    # 0x0c
+'\x0d',    # 0x0d
+'\x0e',    # 0x0e
+'\x0f',    # 0x0f
+'\x10',    # 0x10
+'\x11',    # 0x11
+'\x12',    # 0x12
+'\x13',    # 0x13
+'\x14',    # 0x14
+'\x15',    # 0x15
+'\x16',    # 0x16
+'\x17',    # 0x17
+'\x18',    # 0x18
+'\x19',    # 0x19
+'\x1a',    # 0x1a
+'\x1b',    # 0x1b
+'\x1c',    # 0x1c
+'\x1d',    # 0x1d
+'\x1e',    # 0x1e
+'\x1f',    # 0x1f
+' ',    # 0x20
+'!',    # 0x21
+'"',    # 0x22
+'#',    # 0x23
+'$',    # 0x24
+'%',    # 0x25
+'&',    # 0x26
+'\'',    # 0x27
+'(',    # 0x28
+')',    # 0x29
+'*',    # 0x2a
+'+',    # 0x2b
+',',    # 0x2c
+'-',    # 0x2d
+'.',    # 0x2e
+'/',    # 0x2f
+'0',    # 0x30
+'1',    # 0x31
+'2',    # 0x32
+'3',    # 0x33
+'4',    # 0x34
+'5',    # 0x35
+'6',    # 0x36
+'7',    # 0x37
+'8',    # 0x38
+'9',    # 0x39
+':',    # 0x3a
+';',    # 0x3b
+'<',    # 0x3c
+'=',    # 0x3d
+'>',    # 0x3e
+'?',    # 0x3f
+'@',    # 0x40
+'A',    # 0x41
+'B',    # 0x42
+'C',    # 0x43
+'D',    # 0x44
+'E',    # 0x45
+'F',    # 0x46
+'G',    # 0x47
+'H',    # 0x48
+'I',    # 0x49
+'J',    # 0x4a
+'K',    # 0x4b
+'L',    # 0x4c
+'M',    # 0x4d
+'N',    # 0x4e
+'O',    # 0x4f
+'P',    # 0x50
+'Q',    # 0x51
+'R',    # 0x52
+'S',    # 0x53
+'T',    # 0x54
+'U',    # 0x55
+'V',    # 0x56
+'W',    # 0x57
+'X',    # 0x58
+'Y',    # 0x59
+'Z',    # 0x5a
+']',    # 0x5b
+'\\',    # 0x5c
+']',    # 0x5d
+'^',    # 0x5e
+'_',    # 0x5f
+'`',    # 0x60
+'a',    # 0x61
+'b',    # 0x62
+'c',    # 0x63
+'d',    # 0x64
+'e',    # 0x65
+'f',    # 0x66
+'g',    # 0x67
+'h',    # 0x68
+'i',    # 0x69
+'j',    # 0x6a
+'k',    # 0x6b
+'l',    # 0x6c
+'m',    # 0x6d
+'n',    # 0x6e
+'o',    # 0x6f
+'p',    # 0x70
+'q',    # 0x71
+'r',    # 0x72
+'s',    # 0x73
+'t',    # 0x74
+'u',    # 0x75
+'v',    # 0x76
+'w',    # 0x77
+'x',    # 0x78
+'y',    # 0x79
+'z',    # 0x7a
+'{',    # 0x7b
+'|',    # 0x7c
+'}',    # 0x7d
+'~',    # 0x7e
+'',    # 0x7f
+'',    # 0x80
+'',    # 0x81
+'',    # 0x82
+'',    # 0x83
+'',    # 0x84
+'',    # 0x85
+'',    # 0x86
+'',    # 0x87
+'',    # 0x88
+'',    # 0x89
+'',    # 0x8a
+'',    # 0x8b
+'',    # 0x8c
+'',    # 0x8d
+'',    # 0x8e
+'',    # 0x8f
+'',    # 0x90
+'',    # 0x91
+'',    # 0x92
+'',    # 0x93
+'',    # 0x94
+'',    # 0x95
+'',    # 0x96
+'',    # 0x97
+'',    # 0x98
+'',    # 0x99
+'',    # 0x9a
+'',    # 0x9b
+'',    # 0x9c
+'',    # 0x9d
+'',    # 0x9e
+'',    # 0x9f
+' ',    # 0xa0
+'!',    # 0xa1
+'C/',    # 0xa2
+'PS',    # 0xa3
+'$?',    # 0xa4
+'Y=',    # 0xa5
+'|',    # 0xa6
+'SS',    # 0xa7
+'"',    # 0xa8
+'(c)',    # 0xa9
+'a',    # 0xaa
+'<<',    # 0xab
+'!',    # 0xac
+'',    # 0xad
+'(r)',    # 0xae
+'-',    # 0xaf
+'deg',    # 0xb0
+'+-',    # 0xb1
+'2',    # 0xb2
+'3',    # 0xb3
+'\'',    # 0xb4
+'u',    # 0xb5
+'P',    # 0xb6
+'*',    # 0xb7
+',',    # 0xb8
+'1',    # 0xb9
+'o',    # 0xba
+'>>',    # 0xbb
+'1/4',    # 0xbc
+'1/2',    # 0xbd
+'3/4',    # 0xbe
+'?',    # 0xbf
+'A',    # 0xc0
+'A',    # 0xc1
+'A',    # 0xc2
+'A',    # 0xc3
+'A',    # 0xc4
+'A',    # 0xc5
+'AE',    # 0xc6
+'C',    # 0xc7
+'E',    # 0xc8
+'E',    # 0xc9
+'E',    # 0xca
+'E',    # 0xcb
+'I',    # 0xcc
+'I',    # 0xcd
+'I',    # 0xce
+'I',    # 0xcf
+'D',    # 0xd0
+'N',    # 0xd1
+'O',    # 0xd2
+'O',    # 0xd3
+'O',    # 0xd4
+'O',    # 0xd5
+'O',    # 0xd6
+'x',    # 0xd7
+'O',    # 0xd8
+'U',    # 0xd9
+'U',    # 0xda
+'U',    # 0xdb
+'U',    # 0xdc
+'Y',    # 0xdd
+'Th',    # 0xde
+'ss',    # 0xdf
+'a',    # 0xe0
+'a',    # 0xe1
+'a',    # 0xe2
+'a',    # 0xe3
+'a',    # 0xe4
+'a',    # 0xe5
+'ae',    # 0xe6
+'c',    # 0xe7
+'e',    # 0xe8
+'e',    # 0xe9
+'e',    # 0xea
+'e',    # 0xeb
+'i',    # 0xec
+'i',    # 0xed
+'i',    # 0xee
+'i',    # 0xef
+'d',    # 0xf0
+'n',    # 0xf1
+'o',    # 0xf2
+'o',    # 0xf3
+'o',    # 0xf4
+'o',    # 0xf5
+'o',    # 0xf6
+'/',    # 0xf7
+'o',    # 0xf8
+'u',    # 0xf9
+'u',    # 0xfa
+'u',    # 0xfb
+'u',    # 0xfc
+'y',    # 0xfd
+'th',    # 0xfe
+'y',    # 0xff
+)

lib/unidecode/x001.py

@@ -0,0 +1,258 @@
+data = (
+'A',    # 0x00
+'a',    # 0x01
+'A',    # 0x02
+'a',    # 0x03
+'A',    # 0x04
+'a',    # 0x05
+'C',    # 0x06
+'c',    # 0x07
+'C',    # 0x08
+'c',    # 0x09
+'C',    # 0x0a
+'c',    # 0x0b
+'C',    # 0x0c
+'c',    # 0x0d
+'D',    # 0x0e
+'d',    # 0x0f
+'D',    # 0x10
+'d',    # 0x11
+'E',    # 0x12
+'e',    # 0x13
+'E',    # 0x14
+'e',    # 0x15
+'E',    # 0x16
+'e',    # 0x17
+'E',    # 0x18
+'e',    # 0x19
+'E',    # 0x1a
+'e',    # 0x1b
+'G',    # 0x1c
+'g',    # 0x1d
+'G',    # 0x1e
+'g',    # 0x1f
+'G',    # 0x20
+'g',    # 0x21
+'G',    # 0x22
+'g',    # 0x23
+'H',    # 0x24
+'h',    # 0x25
+'H',    # 0x26
+'h',    # 0x27
+'I',    # 0x28
+'i',    # 0x29
+'I',    # 0x2a
+'i',    # 0x2b
+'I',    # 0x2c
+'i',    # 0x2d
+'I',    # 0x2e
+'i',    # 0x2f
+'I',    # 0x30
+'i',    # 0x31
+'IJ',    # 0x32
+'ij',    # 0x33
+'J',    # 0x34
+'j',    # 0x35
+'K',    # 0x36
+'k',    # 0x37
+'k',    # 0x38
+'L',    # 0x39
+'l',    # 0x3a
+'L',    # 0x3b
+'l',    # 0x3c
+'L',    # 0x3d
+'l',    # 0x3e
+'L',    # 0x3f
+'l',    # 0x40
+'L',    # 0x41
+'l',    # 0x42
+'N',    # 0x43
+'n',    # 0x44
+'N',    # 0x45
+'n',    # 0x46
+'N',    # 0x47
+'n',    # 0x48
+'\'n',    # 0x49
+'ng',    # 0x4a
+'NG',    # 0x4b
+'O',    # 0x4c
+'o',    # 0x4d
+'O',    # 0x4e
+'o',    # 0x4f
+'O',    # 0x50
+'o',    # 0x51
+'OE',    # 0x52
+'oe',    # 0x53
+'R',    # 0x54
+'r',    # 0x55
+'R',    # 0x56
+'r',    # 0x57
+'R',    # 0x58
+'r',    # 0x59
+'S',    # 0x5a
+'s',    # 0x5b
+'S',    # 0x5c
+'s',    # 0x5d
+'S',    # 0x5e
+'s',    # 0x5f
+'S',    # 0x60
+'s',    # 0x61
+'T',    # 0x62
+'t',    # 0x63
+'T',    # 0x64
+'t',    # 0x65
+'T',    # 0x66
+'t',    # 0x67
+'U',    # 0x68
+'u',    # 0x69
+'U',    # 0x6a
+'u',    # 0x6b
+'U',    # 0x6c
+'u',    # 0x6d
+'U',    # 0x6e
+'u',    # 0x6f
+'U',    # 0x70
+'u',    # 0x71
+'U',    # 0x72
+'u',    # 0x73
+'W',    # 0x74
+'w',    # 0x75
+'Y',    # 0x76
+'y',    # 0x77
+'Y',    # 0x78
+'Z',    # 0x79
+'z',    # 0x7a
+'Z',    # 0x7b
+'z',    # 0x7c
+'Z',    # 0x7d
+'z',    # 0x7e
+'s',    # 0x7f
+'b',    # 0x80
+'B',    # 0x81
+'B',    # 0x82
+'b',    # 0x83
+'6',    # 0x84
+'6',    # 0x85
+'O',    # 0x86
+'C',    # 0x87
+'c',    # 0x88
+'D',    # 0x89
+'D',    # 0x8a
+'D',    # 0x8b
+'d',    # 0x8c
+'d',    # 0x8d
+'3',    # 0x8e
+'@',    # 0x8f
+'E',    # 0x90
+'F',    # 0x91
+'f',    # 0x92
+'G',    # 0x93
+'G',    # 0x94
+'hv',    # 0x95
+'I',    # 0x96
+'I',    # 0x97
+'K',    # 0x98
+'k',    # 0x99
+'l',    # 0x9a
+'l',    # 0x9b
+'W',    # 0x9c
+'N',    # 0x9d
+'n',    # 0x9e
+'O',    # 0x9f
+'O',    # 0xa0
+'o',    # 0xa1
+'OI',    # 0xa2
+'oi',    # 0xa3
+'P',    # 0xa4
+'p',    # 0xa5
+'YR',    # 0xa6
+'2',    # 0xa7
+'2',    # 0xa8
+'SH',    # 0xa9
+'sh',    # 0xaa
+'t',    # 0xab
+'T',    # 0xac
+'t',    # 0xad
+'T',    # 0xae
+'U',    # 0xaf
+'u',    # 0xb0
+'Y',    # 0xb1
+'V',    # 0xb2
+'Y',    # 0xb3
+'y',    # 0xb4
+'Z',    # 0xb5
+'z',    # 0xb6
+'ZH',    # 0xb7
+'ZH',    # 0xb8
+'zh',    # 0xb9
+'zh',    # 0xba
+'2',    # 0xbb
+'5',    # 0xbc
+'5',    # 0xbd
+'ts',    # 0xbe
+'w',    # 0xbf
+'|',    # 0xc0
+'||',    # 0xc1
+'|=',    # 0xc2
+'!',    # 0xc3
+'DZ',    # 0xc4
+'Dz',    # 0xc5
+'dz',    # 0xc6
+'LJ',    # 0xc7
+'Lj',    # 0xc8
+'lj',    # 0xc9
+'NJ',    # 0xca
+'Nj',    # 0xcb
+'nj',    # 0xcc
+'A',    # 0xcd
+'a',    # 0xce
+'I',    # 0xcf
+'i',    # 0xd0
+'O',    # 0xd1
+'o',    # 0xd2
+'U',    # 0xd3
+'u',    # 0xd4
+'U',    # 0xd5
+'u',    # 0xd6
+'U',    # 0xd7
+'u',    # 0xd8
+'U',    # 0xd9
+'u',    # 0xda
+'U',    # 0xdb
+'u',    # 0xdc
+'@',    # 0xdd
+'A',    # 0xde
+'a',    # 0xdf
+'A',    # 0xe0
+'a',    # 0xe1
+'AE',    # 0xe2
+'ae',    # 0xe3
+'G',    # 0xe4
+'g',    # 0xe5
+'G',    # 0xe6
+'g',    # 0xe7
+'K',    # 0xe8
+'k',    # 0xe9
+'O',    # 0xea
+'o',    # 0xeb
+'O',    # 0xec
+'o',    # 0xed
+'ZH',    # 0xee
+'zh',    # 0xef
+'j',    # 0xf0
+'DZ',    # 0xf1
+'Dz',    # 0xf2
+'dz',    # 0xf3
+'G',    # 0xf4
+'g',    # 0xf5
+'HV',    # 0xf6
+'W',    # 0xf7
+'N',    # 0xf8
+'n',    # 0xf9
+'A',    # 0xfa
+'a',    # 0xfb
+'AE',    # 0xfc
+'ae',    # 0xfd
+'O',    # 0xfe
+'o',    # 0xff
+)

lib/unidecode/x002.py

@@ -0,0 +1,257 @@
+data = (
+'A',    # 0x00
+'a',    # 0x01
+'A',    # 0x02
+'a',    # 0x03
+'E',    # 0x04
+'e',    # 0x05
+'E',    # 0x06
+'e',    # 0x07
+'I',    # 0x08
+'i',    # 0x09
+'I',    # 0x0a
+'i',    # 0x0b
+'O',    # 0x0c
+'o',    # 0x0d
+'O',    # 0x0e
+'o',    # 0x0f
+'R',    # 0x10
+'r',    # 0x11
+'R',    # 0x12
+'r',    # 0x13
+'U',    # 0x14
+'u',    # 0x15
+'U',    # 0x16
+'u',    # 0x17
+'S',    # 0x18
+'s',    # 0x19
+'T',    # 0x1a
+'t',    # 0x1b
+'Y',    # 0x1c
+'y',    # 0x1d
+'H',    # 0x1e
+'h',    # 0x1f
+'N',    # 0x20
+'d',    # 0x21
+'OU',    # 0x22
+'ou',    # 0x23
+'Z',    # 0x24
+'z',    # 0x25
+'A',    # 0x26
+'a',    # 0x27
+'E',    # 0x28
+'e',    # 0x29
+'O',    # 0x2a
+'o',    # 0x2b
+'O',    # 0x2c
+'o',    # 0x2d
+'O',    # 0x2e
+'o',    # 0x2f
+'O',    # 0x30
+'o',    # 0x31
+'Y',    # 0x32
+'y',    # 0x33
+'l',    # 0x34
+'n',    # 0x35
+'t',    # 0x36
+'j',    # 0x37
+'db',    # 0x38
+'qp',    # 0x39
+'A',    # 0x3a
+'C',    # 0x3b
+'c',    # 0x3c
+'L',    # 0x3d
+'T',    # 0x3e
+'s',    # 0x3f
+'z',    # 0x40
+'[?]',    # 0x41
+'[?]',    # 0x42
+'B',    # 0x43
+'U',    # 0x44
+'^',    # 0x45
+'E',    # 0x46
+'e',    # 0x47
+'J',    # 0x48
+'j',    # 0x49
+'q',    # 0x4a
+'q',    # 0x4b
+'R',    # 0x4c
+'r',    # 0x4d
+'Y',    # 0x4e
+'y',    # 0x4f
+'a',    # 0x50
+'a',    # 0x51
+'a',    # 0x52
+'b',    # 0x53
+'o',    # 0x54
+'c',    # 0x55
+'d',    # 0x56
+'d',    # 0x57
+'e',    # 0x58
+'@',    # 0x59
+'@',    # 0x5a
+'e',    # 0x5b
+'e',    # 0x5c
+'e',    # 0x5d
+'e',    # 0x5e
+'j',    # 0x5f
+'g',    # 0x60
+'g',    # 0x61
+'g',    # 0x62
+'g',    # 0x63
+'u',    # 0x64
+'Y',    # 0x65
+'h',    # 0x66
+'h',    # 0x67
+'i',    # 0x68
+'i',    # 0x69
+'I',    # 0x6a
+'l',    # 0x6b
+'l',    # 0x6c
+'l',    # 0x6d
+'lZ',    # 0x6e
+'W',    # 0x6f
+'W',    # 0x70
+'m',    # 0x71
+'n',    # 0x72
+'n',    # 0x73
+'n',    # 0x74
+'o',    # 0x75
+'OE',    # 0x76
+'O',    # 0x77
+'F',    # 0x78
+'r',    # 0x79
+'r',    # 0x7a
+'r',    # 0x7b
+'r',    # 0x7c
+'r',    # 0x7d
+'r',    # 0x7e
+'r',    # 0x7f
+'R',    # 0x80
+'R',    # 0x81
+'s',    # 0x82
+'S',    # 0x83
+'j',    # 0x84
+'S',    # 0x85
+'S',    # 0x86
+'t',    # 0x87
+'t',    # 0x88
+'u',    # 0x89
+'U',    # 0x8a
+'v',    # 0x8b
+'^',    # 0x8c
+'w',    # 0x8d
+'y',    # 0x8e
+'Y',    # 0x8f
+'z',    # 0x90
+'z',    # 0x91
+'Z',    # 0x92
+'Z',    # 0x93
+'?',    # 0x94
+'?',    # 0x95
+'?',    # 0x96
+'C',    # 0x97
+'@',    # 0x98
+'B',    # 0x99
+'E',    # 0x9a
+'G',    # 0x9b
+'H',    # 0x9c
+'j',    # 0x9d
+'k',    # 0x9e
+'L',    # 0x9f
+'q',    # 0xa0
+'?',    # 0xa1
+'?',    # 0xa2
+'dz',    # 0xa3
+'dZ',    # 0xa4
+'dz',    # 0xa5
+'ts',    # 0xa6
+'tS',    # 0xa7
+'tC',    # 0xa8
+'fN',    # 0xa9
+'ls',    # 0xaa
+'lz',    # 0xab
+'WW',    # 0xac
+']]',    # 0xad
+'h',    # 0xae
+'h',    # 0xaf
+'k',    # 0xb0
+'h',    # 0xb1
+'j',    # 0xb2
+'r',    # 0xb3
+'r',    # 0xb4
+'r',    # 0xb5
+'r',    # 0xb6
+'w',    # 0xb7
+'y',    # 0xb8
+'\'',    # 0xb9
+'"',    # 0xba
+'`',    # 0xbb
+'\'',    # 0xbc
+'`',    # 0xbd
+'`',    # 0xbe
+'\'',    # 0xbf
+'?',    # 0xc0
+'?',    # 0xc1
+'<',    # 0xc2
+'>',    # 0xc3
+'^',    # 0xc4
+'V',    # 0xc5
+'^',    # 0xc6
+'V',    # 0xc7
+'\'',    # 0xc8
+'-',    # 0xc9
+'/',    # 0xca
+'\\',    # 0xcb
+',',    # 0xcc
+'_',    # 0xcd
+'\\',    # 0xce
+'/',    # 0xcf
+':',    # 0xd0
+'.',    # 0xd1
+'`',    # 0xd2
+'\'',    # 0xd3
+'^',    # 0xd4
+'V',    # 0xd5
+'+',    # 0xd6
+'-',    # 0xd7
+'V',    # 0xd8
+'.',    # 0xd9
+'@',    # 0xda
+',',    # 0xdb
+'~',    # 0xdc
+'"',    # 0xdd
+'R',    # 0xde
+'X',    # 0xdf
+'G',    # 0xe0
+'l',    # 0xe1
+'s',    # 0xe2
+'x',    # 0xe3
+'?',    # 0xe4
+'',    # 0xe5
+'',    # 0xe6
+'',    # 0xe7
+'',    # 0xe8
+'',    # 0xe9
+'',    # 0xea
+'',    # 0xeb
+'V',    # 0xec
+'=',    # 0xed
+'"',    # 0xee
+'[?]',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'[?]',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x003.py

@@ -0,0 +1,257 @@
+data = (
+'',    # 0x00
+'',    # 0x01
+'',    # 0x02
+'',    # 0x03
+'',    # 0x04
+'',    # 0x05
+'',    # 0x06
+'',    # 0x07
+'',    # 0x08
+'',    # 0x09
+'',    # 0x0a
+'',    # 0x0b
+'',    # 0x0c
+'',    # 0x0d
+'',    # 0x0e
+'',    # 0x0f
+'',    # 0x10
+'',    # 0x11
+'',    # 0x12
+'',    # 0x13
+'',    # 0x14
+'',    # 0x15
+'',    # 0x16
+'',    # 0x17
+'',    # 0x18
+'',    # 0x19
+'',    # 0x1a
+'',    # 0x1b
+'',    # 0x1c
+'',    # 0x1d
+'',    # 0x1e
+'',    # 0x1f
+'',    # 0x20
+'',    # 0x21
+'',    # 0x22
+'',    # 0x23
+'',    # 0x24
+'',    # 0x25
+'',    # 0x26
+'',    # 0x27
+'',    # 0x28
+'',    # 0x29
+'',    # 0x2a
+'',    # 0x2b
+'',    # 0x2c
+'',    # 0x2d
+'',    # 0x2e
+'',    # 0x2f
+'',    # 0x30
+'',    # 0x31
+'',    # 0x32
+'',    # 0x33
+'',    # 0x34
+'',    # 0x35
+'',    # 0x36
+'',    # 0x37
+'',    # 0x38
+'',    # 0x39
+'',    # 0x3a
+'',    # 0x3b
+'',    # 0x3c
+'',    # 0x3d
+'',    # 0x3e
+'',    # 0x3f
+'',    # 0x40
+'',    # 0x41
+'',    # 0x42
+'',    # 0x43
+'',    # 0x44
+'',    # 0x45
+'',    # 0x46
+'',    # 0x47
+'',    # 0x48
+'',    # 0x49
+'',    # 0x4a
+'',    # 0x4b
+'',    # 0x4c
+'',    # 0x4d
+'',    # 0x4e
+'[?]',    # 0x4f
+'[?]',    # 0x50
+'[?]',    # 0x51
+'[?]',    # 0x52
+'[?]',    # 0x53
+'[?]',    # 0x54
+'[?]',    # 0x55
+'[?]',    # 0x56
+'[?]',    # 0x57
+'[?]',    # 0x58
+'[?]',    # 0x59
+'[?]',    # 0x5a
+'[?]',    # 0x5b
+'[?]',    # 0x5c
+'[?]',    # 0x5d
+'[?]',    # 0x5e
+'[?]',    # 0x5f
+'',    # 0x60
+'',    # 0x61
+'',    # 0x62
+'a',    # 0x63
+'e',    # 0x64
+'i',    # 0x65
+'o',    # 0x66
+'u',    # 0x67
+'c',    # 0x68
+'d',    # 0x69
+'h',    # 0x6a
+'m',    # 0x6b
+'r',    # 0x6c
+'t',    # 0x6d
+'v',    # 0x6e
+'x',    # 0x6f
+'[?]',    # 0x70
+'[?]',    # 0x71
+'[?]',    # 0x72
+'[?]',    # 0x73
+'\'',    # 0x74
+',',    # 0x75
+'[?]',    # 0x76
+'[?]',    # 0x77
+'[?]',    # 0x78
+'[?]',    # 0x79
+'',    # 0x7a
+'[?]',    # 0x7b
+'[?]',    # 0x7c
+'[?]',    # 0x7d
+'?',    # 0x7e
+'[?]',    # 0x7f
+'[?]',    # 0x80
+'[?]',    # 0x81
+'[?]',    # 0x82
+'[?]',    # 0x83
+'',    # 0x84
+'',    # 0x85
+'A',    # 0x86
+';',    # 0x87
+'E',    # 0x88
+'E',    # 0x89
+'I',    # 0x8a
+'[?]',    # 0x8b
+'O',    # 0x8c
+'[?]',    # 0x8d
+'U',    # 0x8e
+'O',    # 0x8f
+'I',    # 0x90
+'A',    # 0x91
+'B',    # 0x92
+'G',    # 0x93
+'D',    # 0x94
+'E',    # 0x95
+'Z',    # 0x96
+'E',    # 0x97
+'Th',    # 0x98
+'I',    # 0x99
+'K',    # 0x9a
+'L',    # 0x9b
+'M',    # 0x9c
+'N',    # 0x9d
+'Ks',    # 0x9e
+'O',    # 0x9f
+'P',    # 0xa0
+'R',    # 0xa1
+'[?]',    # 0xa2
+'S',    # 0xa3
+'T',    # 0xa4
+'U',    # 0xa5
+'Ph',    # 0xa6
+'Kh',    # 0xa7
+'Ps',    # 0xa8
+'O',    # 0xa9
+'I',    # 0xaa
+'U',    # 0xab
+'a',    # 0xac
+'e',    # 0xad
+'e',    # 0xae
+'i',    # 0xaf
+'u',    # 0xb0
+'a',    # 0xb1
+'b',    # 0xb2
+'g',    # 0xb3
+'d',    # 0xb4
+'e',    # 0xb5
+'z',    # 0xb6
+'e',    # 0xb7
+'th',    # 0xb8
+'i',    # 0xb9
+'k',    # 0xba
+'l',    # 0xbb
+'m',    # 0xbc
+'n',    # 0xbd
+'x',    # 0xbe
+'o',    # 0xbf
+'p',    # 0xc0
+'r',    # 0xc1
+'s',    # 0xc2
+'s',    # 0xc3
+'t',    # 0xc4
+'u',    # 0xc5
+'ph',    # 0xc6
+'kh',    # 0xc7
+'ps',    # 0xc8
+'o',    # 0xc9
+'i',    # 0xca
+'u',    # 0xcb
+'o',    # 0xcc
+'u',    # 0xcd
+'o',    # 0xce
+'[?]',    # 0xcf
+'b',    # 0xd0
+'th',    # 0xd1
+'U',    # 0xd2
+'U',    # 0xd3
+'U',    # 0xd4
+'ph',    # 0xd5
+'p',    # 0xd6
+'&',    # 0xd7
+'[?]',    # 0xd8
+'[?]',    # 0xd9
+'St',    # 0xda
+'st',    # 0xdb
+'W',    # 0xdc
+'w',    # 0xdd
+'Q',    # 0xde
+'q',    # 0xdf
+'Sp',    # 0xe0
+'sp',    # 0xe1
+'Sh',    # 0xe2
+'sh',    # 0xe3
+'F',    # 0xe4
+'f',    # 0xe5
+'Kh',    # 0xe6
+'kh',    # 0xe7
+'H',    # 0xe8
+'h',    # 0xe9
+'G',    # 0xea
+'g',    # 0xeb
+'CH',    # 0xec
+'ch',    # 0xed
+'Ti',    # 0xee
+'ti',    # 0xef
+'k',    # 0xf0
+'r',    # 0xf1
+'c',    # 0xf2
+'j',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x004.py

@@ -0,0 +1,257 @@
+data = (
+'Ie',    # 0x00
+'Io',    # 0x01
+'Dj',    # 0x02
+'Gj',    # 0x03
+'Ie',    # 0x04
+'Dz',    # 0x05
+'I',    # 0x06
+'Yi',    # 0x07
+'J',    # 0x08
+'Lj',    # 0x09
+'Nj',    # 0x0a
+'Tsh',    # 0x0b
+'Kj',    # 0x0c
+'I',    # 0x0d
+'U',    # 0x0e
+'Dzh',    # 0x0f
+'A',    # 0x10
+'B',    # 0x11
+'V',    # 0x12
+'G',    # 0x13
+'D',    # 0x14
+'E',    # 0x15
+'Zh',    # 0x16
+'Z',    # 0x17
+'I',    # 0x18
+'I',    # 0x19
+'K',    # 0x1a
+'L',    # 0x1b
+'M',    # 0x1c
+'N',    # 0x1d
+'O',    # 0x1e
+'P',    # 0x1f
+'R',    # 0x20
+'S',    # 0x21
+'T',    # 0x22
+'U',    # 0x23
+'F',    # 0x24
+'Kh',    # 0x25
+'Ts',    # 0x26
+'Ch',    # 0x27
+'Sh',    # 0x28
+'Shch',    # 0x29
+'\'',    # 0x2a
+'Y',    # 0x2b
+'\'',    # 0x2c
+'E',    # 0x2d
+'Iu',    # 0x2e
+'Ia',    # 0x2f
+'a',    # 0x30
+'b',    # 0x31
+'v',    # 0x32
+'g',    # 0x33
+'d',    # 0x34
+'e',    # 0x35
+'zh',    # 0x36
+'z',    # 0x37
+'i',    # 0x38
+'i',    # 0x39
+'k',    # 0x3a
+'l',    # 0x3b
+'m',    # 0x3c
+'n',    # 0x3d
+'o',    # 0x3e
+'p',    # 0x3f
+'r',    # 0x40
+'s',    # 0x41
+'t',    # 0x42
+'u',    # 0x43
+'f',    # 0x44
+'kh',    # 0x45
+'ts',    # 0x46
+'ch',    # 0x47
+'sh',    # 0x48
+'shch',    # 0x49
+'\'',    # 0x4a
+'y',    # 0x4b
+'\'',    # 0x4c
+'e',    # 0x4d
+'iu',    # 0x4e
+'ia',    # 0x4f
+'ie',    # 0x50
+'io',    # 0x51
+'dj',    # 0x52
+'gj',    # 0x53
+'ie',    # 0x54
+'dz',    # 0x55
+'i',    # 0x56
+'yi',    # 0x57
+'j',    # 0x58
+'lj',    # 0x59
+'nj',    # 0x5a
+'tsh',    # 0x5b
+'kj',    # 0x5c
+'i',    # 0x5d
+'u',    # 0x5e
+'dzh',    # 0x5f
+'O',    # 0x60
+'o',    # 0x61
+'E',    # 0x62
+'e',    # 0x63
+'Ie',    # 0x64
+'ie',    # 0x65
+'E',    # 0x66
+'e',    # 0x67
+'Ie',    # 0x68
+'ie',    # 0x69
+'O',    # 0x6a
+'o',    # 0x6b
+'Io',    # 0x6c
+'io',    # 0x6d
+'Ks',    # 0x6e
+'ks',    # 0x6f
+'Ps',    # 0x70
+'ps',    # 0x71
+'F',    # 0x72
+'f',    # 0x73
+'Y',    # 0x74
+'y',    # 0x75
+'Y',    # 0x76
+'y',    # 0x77
+'u',    # 0x78
+'u',    # 0x79
+'O',    # 0x7a
+'o',    # 0x7b
+'O',    # 0x7c
+'o',    # 0x7d
+'Ot',    # 0x7e
+'ot',    # 0x7f
+'Q',    # 0x80
+'q',    # 0x81
+'*1000*',    # 0x82
+'',    # 0x83
+'',    # 0x84
+'',    # 0x85
+'',    # 0x86
+'[?]',    # 0x87
+'*100.000*',    # 0x88
+'*1.000.000*',    # 0x89
+'[?]',    # 0x8a
+'[?]',    # 0x8b
+'"',    # 0x8c
+'"',    # 0x8d
+'R\'',    # 0x8e
+'r\'',    # 0x8f
+'G\'',    # 0x90
+'g\'',    # 0x91
+'G\'',    # 0x92
+'g\'',    # 0x93
+'G\'',    # 0x94
+'g\'',    # 0x95
+'Zh\'',    # 0x96
+'zh\'',    # 0x97
+'Z\'',    # 0x98
+'z\'',    # 0x99
+'K\'',    # 0x9a
+'k\'',    # 0x9b
+'K\'',    # 0x9c
+'k\'',    # 0x9d
+'K\'',    # 0x9e
+'k\'',    # 0x9f
+'K\'',    # 0xa0
+'k\'',    # 0xa1
+'N\'',    # 0xa2
+'n\'',    # 0xa3
+'Ng',    # 0xa4
+'ng',    # 0xa5
+'P\'',    # 0xa6
+'p\'',    # 0xa7
+'Kh',    # 0xa8
+'kh',    # 0xa9
+'S\'',    # 0xaa
+'s\'',    # 0xab
+'T\'',    # 0xac
+'t\'',    # 0xad
+'U',    # 0xae
+'u',    # 0xaf
+'U\'',    # 0xb0
+'u\'',    # 0xb1
+'Kh\'',    # 0xb2
+'kh\'',    # 0xb3
+'Tts',    # 0xb4
+'tts',    # 0xb5
+'Ch\'',    # 0xb6
+'ch\'',    # 0xb7
+'Ch\'',    # 0xb8
+'ch\'',    # 0xb9
+'H',    # 0xba
+'h',    # 0xbb
+'Ch',    # 0xbc
+'ch',    # 0xbd
+'Ch\'',    # 0xbe
+'ch\'',    # 0xbf
+'`',    # 0xc0
+'Zh',    # 0xc1
+'zh',    # 0xc2
+'K\'',    # 0xc3
+'k\'',    # 0xc4
+'[?]',    # 0xc5
+'[?]',    # 0xc6
+'N\'',    # 0xc7
+'n\'',    # 0xc8
+'[?]',    # 0xc9
+'[?]',    # 0xca
+'Ch',    # 0xcb
+'ch',    # 0xcc
+'[?]',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'a',    # 0xd0
+'a',    # 0xd1
+'A',    # 0xd2
+'a',    # 0xd3
+'Ae',    # 0xd4
+'ae',    # 0xd5
+'Ie',    # 0xd6
+'ie',    # 0xd7
+'@',    # 0xd8
+'@',    # 0xd9
+'@',    # 0xda
+'@',    # 0xdb
+'Zh',    # 0xdc
+'zh',    # 0xdd
+'Z',    # 0xde
+'z',    # 0xdf
+'Dz',    # 0xe0
+'dz',    # 0xe1
+'I',    # 0xe2
+'i',    # 0xe3
+'I',    # 0xe4
+'i',    # 0xe5
+'O',    # 0xe6
+'o',    # 0xe7
+'O',    # 0xe8
+'o',    # 0xe9
+'O',    # 0xea
+'o',    # 0xeb
+'E',    # 0xec
+'e',    # 0xed
+'U',    # 0xee
+'u',    # 0xef
+'U',    # 0xf0
+'u',    # 0xf1
+'U',    # 0xf2
+'u',    # 0xf3
+'Ch',    # 0xf4
+'ch',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'Y',    # 0xf8
+'y',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x005.py

@@ -0,0 +1,257 @@
+data = (
+'[?]',    # 0x00
+'[?]',    # 0x01
+'[?]',    # 0x02
+'[?]',    # 0x03
+'[?]',    # 0x04
+'[?]',    # 0x05
+'[?]',    # 0x06
+'[?]',    # 0x07
+'[?]',    # 0x08
+'[?]',    # 0x09
+'[?]',    # 0x0a
+'[?]',    # 0x0b
+'[?]',    # 0x0c
+'[?]',    # 0x0d
+'[?]',    # 0x0e
+'[?]',    # 0x0f
+'[?]',    # 0x10
+'[?]',    # 0x11
+'[?]',    # 0x12
+'[?]',    # 0x13
+'[?]',    # 0x14
+'[?]',    # 0x15
+'[?]',    # 0x16
+'[?]',    # 0x17
+'[?]',    # 0x18
+'[?]',    # 0x19
+'[?]',    # 0x1a
+'[?]',    # 0x1b
+'[?]',    # 0x1c
+'[?]',    # 0x1d
+'[?]',    # 0x1e
+'[?]',    # 0x1f
+'[?]',    # 0x20
+'[?]',    # 0x21
+'[?]',    # 0x22
+'[?]',    # 0x23
+'[?]',    # 0x24
+'[?]',    # 0x25
+'[?]',    # 0x26
+'[?]',    # 0x27
+'[?]',    # 0x28
+'[?]',    # 0x29
+'[?]',    # 0x2a
+'[?]',    # 0x2b
+'[?]',    # 0x2c
+'[?]',    # 0x2d
+'[?]',    # 0x2e
+'[?]',    # 0x2f
+'[?]',    # 0x30
+'A',    # 0x31
+'B',    # 0x32
+'G',    # 0x33
+'D',    # 0x34
+'E',    # 0x35
+'Z',    # 0x36
+'E',    # 0x37
+'E',    # 0x38
+'T`',    # 0x39
+'Zh',    # 0x3a
+'I',    # 0x3b
+'L',    # 0x3c
+'Kh',    # 0x3d
+'Ts',    # 0x3e
+'K',    # 0x3f
+'H',    # 0x40
+'Dz',    # 0x41
+'Gh',    # 0x42
+'Ch',    # 0x43
+'M',    # 0x44
+'Y',    # 0x45
+'N',    # 0x46
+'Sh',    # 0x47
+'O',    # 0x48
+'Ch`',    # 0x49
+'P',    # 0x4a
+'J',    # 0x4b
+'Rh',    # 0x4c
+'S',    # 0x4d
+'V',    # 0x4e
+'T',    # 0x4f
+'R',    # 0x50
+'Ts`',    # 0x51
+'W',    # 0x52
+'P`',    # 0x53
+'K`',    # 0x54
+'O',    # 0x55
+'F',    # 0x56
+'[?]',    # 0x57
+'[?]',    # 0x58
+'<',    # 0x59
+'\'',    # 0x5a
+'/',    # 0x5b
+'!',    # 0x5c
+',',    # 0x5d
+'?',    # 0x5e
+'.',    # 0x5f
+'[?]',    # 0x60
+'a',    # 0x61
+'b',    # 0x62
+'g',    # 0x63
+'d',    # 0x64
+'e',    # 0x65
+'z',    # 0x66
+'e',    # 0x67
+'e',    # 0x68
+'t`',    # 0x69
+'zh',    # 0x6a
+'i',    # 0x6b
+'l',    # 0x6c
+'kh',    # 0x6d
+'ts',    # 0x6e
+'k',    # 0x6f
+'h',    # 0x70
+'dz',    # 0x71
+'gh',    # 0x72
+'ch',    # 0x73
+'m',    # 0x74
+'y',    # 0x75
+'n',    # 0x76
+'sh',    # 0x77
+'o',    # 0x78
+'ch`',    # 0x79
+'p',    # 0x7a
+'j',    # 0x7b
+'rh',    # 0x7c
+'s',    # 0x7d
+'v',    # 0x7e
+'t',    # 0x7f
+'r',    # 0x80
+'ts`',    # 0x81
+'w',    # 0x82
+'p`',    # 0x83
+'k`',    # 0x84
+'o',    # 0x85
+'f',    # 0x86
+'ew',    # 0x87
+'[?]',    # 0x88
+':',    # 0x89
+'-',    # 0x8a
+'[?]',    # 0x8b
+'[?]',    # 0x8c
+'[?]',    # 0x8d
+'[?]',    # 0x8e
+'[?]',    # 0x8f
+'[?]',    # 0x90
+'',    # 0x91
+'',    # 0x92
+'',    # 0x93
+'',    # 0x94
+'',    # 0x95
+'',    # 0x96
+'',    # 0x97
+'',    # 0x98
+'',    # 0x99
+'',    # 0x9a
+'',    # 0x9b
+'',    # 0x9c
+'',    # 0x9d
+'',    # 0x9e
+'',    # 0x9f
+'',    # 0xa0
+'',    # 0xa1
+'[?]',    # 0xa2
+'',    # 0xa3
+'',    # 0xa4
+'',    # 0xa5
+'',    # 0xa6
+'',    # 0xa7
+'',    # 0xa8
+'',    # 0xa9
+'',    # 0xaa
+'',    # 0xab
+'',    # 0xac
+'',    # 0xad
+'',    # 0xae
+'',    # 0xaf
+'@',    # 0xb0
+'e',    # 0xb1
+'a',    # 0xb2
+'o',    # 0xb3
+'i',    # 0xb4
+'e',    # 0xb5
+'e',    # 0xb6
+'a',    # 0xb7
+'a',    # 0xb8
+'o',    # 0xb9
+'[?]',    # 0xba
+'u',    # 0xbb
+'\'',    # 0xbc
+'',    # 0xbd
+'',    # 0xbe
+'',    # 0xbf
+'|',    # 0xc0
+'',    # 0xc1
+'',    # 0xc2
+':',    # 0xc3
+'',    # 0xc4
+'[?]',    # 0xc5
+'[?]',    # 0xc6
+'[?]',    # 0xc7
+'[?]',    # 0xc8
+'[?]',    # 0xc9
+'[?]',    # 0xca
+'[?]',    # 0xcb
+'[?]',    # 0xcc
+'[?]',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'',    # 0xd0
+'b',    # 0xd1
+'g',    # 0xd2
+'d',    # 0xd3
+'h',    # 0xd4
+'v',    # 0xd5
+'z',    # 0xd6
+'kh',    # 0xd7
+'t',    # 0xd8
+'y',    # 0xd9
+'k',    # 0xda
+'k',    # 0xdb
+'l',    # 0xdc
+'m',    # 0xdd
+'m',    # 0xde
+'n',    # 0xdf
+'n',    # 0xe0
+'s',    # 0xe1
+'`',    # 0xe2
+'p',    # 0xe3
+'p',    # 0xe4
+'ts',    # 0xe5
+'ts',    # 0xe6
+'q',    # 0xe7
+'r',    # 0xe8
+'sh',    # 0xe9
+'t',    # 0xea
+'[?]',    # 0xeb
+'[?]',    # 0xec
+'[?]',    # 0xed
+'[?]',    # 0xee
+'[?]',    # 0xef
+'V',    # 0xf0
+'oy',    # 0xf1
+'i',    # 0xf2
+'\'',    # 0xf3
+'"',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x006.py

@@ -0,0 +1,257 @@
+data = (
+'[?]',    # 0x00
+'[?]',    # 0x01
+'[?]',    # 0x02
+'[?]',    # 0x03
+'[?]',    # 0x04
+'[?]',    # 0x05
+'[?]',    # 0x06
+'[?]',    # 0x07
+'[?]',    # 0x08
+'[?]',    # 0x09
+'[?]',    # 0x0a
+'[?]',    # 0x0b
+',',    # 0x0c
+'[?]',    # 0x0d
+'[?]',    # 0x0e
+'[?]',    # 0x0f
+'[?]',    # 0x10
+'[?]',    # 0x11
+'[?]',    # 0x12
+'[?]',    # 0x13
+'[?]',    # 0x14
+'[?]',    # 0x15
+'[?]',    # 0x16
+'[?]',    # 0x17
+'[?]',    # 0x18
+'[?]',    # 0x19
+'[?]',    # 0x1a
+';',    # 0x1b
+'[?]',    # 0x1c
+'[?]',    # 0x1d
+'[?]',    # 0x1e
+'?',    # 0x1f
+'[?]',    # 0x20
+'',    # 0x21
+'a',    # 0x22
+'\'',    # 0x23
+'w\'',    # 0x24
+'',    # 0x25
+'y\'',    # 0x26
+'',    # 0x27
+'b',    # 0x28
+'@',    # 0x29
+'t',    # 0x2a
+'th',    # 0x2b
+'j',    # 0x2c
+'H',    # 0x2d
+'kh',    # 0x2e
+'d',    # 0x2f
+'dh',    # 0x30
+'r',    # 0x31
+'z',    # 0x32
+'s',    # 0x33
+'sh',    # 0x34
+'S',    # 0x35
+'D',    # 0x36
+'T',    # 0x37
+'Z',    # 0x38
+'`',    # 0x39
+'G',    # 0x3a
+'[?]',    # 0x3b
+'[?]',    # 0x3c
+'[?]',    # 0x3d
+'[?]',    # 0x3e
+'[?]',    # 0x3f
+'',    # 0x40
+'f',    # 0x41
+'q',    # 0x42
+'k',    # 0x43
+'l',    # 0x44
+'m',    # 0x45
+'n',    # 0x46
+'h',    # 0x47
+'w',    # 0x48
+'~',    # 0x49
+'y',    # 0x4a
+'an',    # 0x4b
+'un',    # 0x4c
+'in',    # 0x4d
+'a',    # 0x4e
+'u',    # 0x4f
+'i',    # 0x50
+'W',    # 0x51
+'',    # 0x52
+'',    # 0x53
+'\'',    # 0x54
+'\'',    # 0x55
+'[?]',    # 0x56
+'[?]',    # 0x57
+'[?]',    # 0x58
+'[?]',    # 0x59
+'[?]',    # 0x5a
+'[?]',    # 0x5b
+'[?]',    # 0x5c
+'[?]',    # 0x5d
+'[?]',    # 0x5e
+'[?]',    # 0x5f
+'0',    # 0x60
+'1',    # 0x61
+'2',    # 0x62
+'3',    # 0x63
+'4',    # 0x64
+'5',    # 0x65
+'6',    # 0x66
+'7',    # 0x67
+'8',    # 0x68
+'9',    # 0x69
+'%',    # 0x6a
+'.',    # 0x6b
+',',    # 0x6c
+'*',    # 0x6d
+'[?]',    # 0x6e
+'[?]',    # 0x6f
+'',    # 0x70
+'\'',    # 0x71
+'\'',    # 0x72
+'\'',    # 0x73
+'',    # 0x74
+'\'',    # 0x75
+'\'w',    # 0x76
+'\'u',    # 0x77
+'\'y',    # 0x78
+'tt',    # 0x79
+'tth',    # 0x7a
+'b',    # 0x7b
+'t',    # 0x7c
+'T',    # 0x7d
+'p',    # 0x7e
+'th',    # 0x7f
+'bh',    # 0x80
+'\'h',    # 0x81
+'H',    # 0x82
+'ny',    # 0x83
+'dy',    # 0x84
+'H',    # 0x85
+'ch',    # 0x86
+'cch',    # 0x87
+'dd',    # 0x88
+'D',    # 0x89
+'D',    # 0x8a
+'Dt',    # 0x8b
+'dh',    # 0x8c
+'ddh',    # 0x8d
+'d',    # 0x8e
+'D',    # 0x8f
+'D',    # 0x90
+'rr',    # 0x91
+'R',    # 0x92
+'R',    # 0x93
+'R',    # 0x94
+'R',    # 0x95
+'R',    # 0x96
+'R',    # 0x97
+'j',    # 0x98
+'R',    # 0x99
+'S',    # 0x9a
+'S',    # 0x9b
+'S',    # 0x9c
+'S',    # 0x9d
+'S',    # 0x9e
+'T',    # 0x9f
+'GH',    # 0xa0
+'F',    # 0xa1
+'F',    # 0xa2
+'F',    # 0xa3
+'v',    # 0xa4
+'f',    # 0xa5
+'ph',    # 0xa6
+'Q',    # 0xa7
+'Q',    # 0xa8
+'kh',    # 0xa9
+'k',    # 0xaa
+'K',    # 0xab
+'K',    # 0xac
+'ng',    # 0xad
+'K',    # 0xae
+'g',    # 0xaf
+'G',    # 0xb0
+'N',    # 0xb1
+'G',    # 0xb2
+'G',    # 0xb3
+'G',    # 0xb4
+'L',    # 0xb5
+'L',    # 0xb6
+'L',    # 0xb7
+'L',    # 0xb8
+'N',    # 0xb9
+'N',    # 0xba
+'N',    # 0xbb
+'N',    # 0xbc
+'N',    # 0xbd
+'h',    # 0xbe
+'Ch',    # 0xbf
+'hy',    # 0xc0
+'h',    # 0xc1
+'H',    # 0xc2
+'@',    # 0xc3
+'W',    # 0xc4
+'oe',    # 0xc5
+'oe',    # 0xc6
+'u',    # 0xc7
+'yu',    # 0xc8
+'yu',    # 0xc9
+'W',    # 0xca
+'v',    # 0xcb
+'y',    # 0xcc
+'Y',    # 0xcd
+'Y',    # 0xce
+'W',    # 0xcf
+'',    # 0xd0
+'',    # 0xd1
+'y',    # 0xd2
+'y\'',    # 0xd3
+'.',    # 0xd4
+'ae',    # 0xd5
+'',    # 0xd6
+'',    # 0xd7
+'',    # 0xd8
+'',    # 0xd9
+'',    # 0xda
+'',    # 0xdb
+'',    # 0xdc
+'@',    # 0xdd
+'#',    # 0xde
+'',    # 0xdf
+'',    # 0xe0
+'',    # 0xe1
+'',    # 0xe2
+'',    # 0xe3
+'',    # 0xe4
+'',    # 0xe5
+'',    # 0xe6
+'',    # 0xe7
+'',    # 0xe8
+'^',    # 0xe9
+'',    # 0xea
+'',    # 0xeb
+'',    # 0xec
+'',    # 0xed
+'[?]',    # 0xee
+'[?]',    # 0xef
+'0',    # 0xf0
+'1',    # 0xf1
+'2',    # 0xf2
+'3',    # 0xf3
+'4',    # 0xf4
+'5',    # 0xf5
+'6',    # 0xf6
+'7',    # 0xf7
+'8',    # 0xf8
+'9',    # 0xf9
+'Sh',    # 0xfa
+'D',    # 0xfb
+'Gh',    # 0xfc
+'&',    # 0xfd
+'+m',    # 0xfe
+)

lib/unidecode/x007.py

@@ -0,0 +1,257 @@
+data = (
+'//',    # 0x00
+'/',    # 0x01
+',',    # 0x02
+'!',    # 0x03
+'!',    # 0x04
+'-',    # 0x05
+',',    # 0x06
+',',    # 0x07
+';',    # 0x08
+'?',    # 0x09
+'~',    # 0x0a
+'{',    # 0x0b
+'}',    # 0x0c
+'*',    # 0x0d
+'[?]',    # 0x0e
+'',    # 0x0f
+'\'',    # 0x10
+'',    # 0x11
+'b',    # 0x12
+'g',    # 0x13
+'g',    # 0x14
+'d',    # 0x15
+'d',    # 0x16
+'h',    # 0x17
+'w',    # 0x18
+'z',    # 0x19
+'H',    # 0x1a
+'t',    # 0x1b
+'t',    # 0x1c
+'y',    # 0x1d
+'yh',    # 0x1e
+'k',    # 0x1f
+'l',    # 0x20
+'m',    # 0x21
+'n',    # 0x22
+'s',    # 0x23
+'s',    # 0x24
+'`',    # 0x25
+'p',    # 0x26
+'p',    # 0x27
+'S',    # 0x28
+'q',    # 0x29
+'r',    # 0x2a
+'sh',    # 0x2b
+'t',    # 0x2c
+'[?]',    # 0x2d
+'[?]',    # 0x2e
+'[?]',    # 0x2f
+'a',    # 0x30
+'a',    # 0x31
+'a',    # 0x32
+'A',    # 0x33
+'A',    # 0x34
+'A',    # 0x35
+'e',    # 0x36
+'e',    # 0x37
+'e',    # 0x38
+'E',    # 0x39
+'i',    # 0x3a
+'i',    # 0x3b
+'u',    # 0x3c
+'u',    # 0x3d
+'u',    # 0x3e
+'o',    # 0x3f
+'',    # 0x40
+'`',    # 0x41
+'\'',    # 0x42
+'',    # 0x43
+'',    # 0x44
+'X',    # 0x45
+'Q',    # 0x46
+'@',    # 0x47
+'@',    # 0x48
+'|',    # 0x49
+'+',    # 0x4a
+'[?]',    # 0x4b
+'[?]',    # 0x4c
+'[?]',    # 0x4d
+'[?]',    # 0x4e
+'[?]',    # 0x4f
+'[?]',    # 0x50
+'[?]',    # 0x51
+'[?]',    # 0x52
+'[?]',    # 0x53
+'[?]',    # 0x54
+'[?]',    # 0x55
+'[?]',    # 0x56
+'[?]',    # 0x57
+'[?]',    # 0x58
+'[?]',    # 0x59
+'[?]',    # 0x5a
+'[?]',    # 0x5b
+'[?]',    # 0x5c
+'[?]',    # 0x5d
+'[?]',    # 0x5e
+'[?]',    # 0x5f
+'[?]',    # 0x60
+'[?]',    # 0x61
+'[?]',    # 0x62
+'[?]',    # 0x63
+'[?]',    # 0x64
+'[?]',    # 0x65
+'[?]',    # 0x66
+'[?]',    # 0x67
+'[?]',    # 0x68
+'[?]',    # 0x69
+'[?]',    # 0x6a
+'[?]',    # 0x6b
+'[?]',    # 0x6c
+'[?]',    # 0x6d
+'[?]',    # 0x6e
+'[?]',    # 0x6f
+'[?]',    # 0x70
+'[?]',    # 0x71
+'[?]',    # 0x72
+'[?]',    # 0x73
+'[?]',    # 0x74
+'[?]',    # 0x75
+'[?]',    # 0x76
+'[?]',    # 0x77
+'[?]',    # 0x78
+'[?]',    # 0x79
+'[?]',    # 0x7a
+'[?]',    # 0x7b
+'[?]',    # 0x7c
+'[?]',    # 0x7d
+'[?]',    # 0x7e
+'[?]',    # 0x7f
+'h',    # 0x80
+'sh',    # 0x81
+'n',    # 0x82
+'r',    # 0x83
+'b',    # 0x84
+'L',    # 0x85
+'k',    # 0x86
+'\'',    # 0x87
+'v',    # 0x88
+'m',    # 0x89
+'f',    # 0x8a
+'dh',    # 0x8b
+'th',    # 0x8c
+'l',    # 0x8d
+'g',    # 0x8e
+'ny',    # 0x8f
+'s',    # 0x90
+'d',    # 0x91
+'z',    # 0x92
+'t',    # 0x93
+'y',    # 0x94
+'p',    # 0x95
+'j',    # 0x96
+'ch',    # 0x97
+'tt',    # 0x98
+'hh',    # 0x99
+'kh',    # 0x9a
+'th',    # 0x9b
+'z',    # 0x9c
+'sh',    # 0x9d
+'s',    # 0x9e
+'d',    # 0x9f
+'t',    # 0xa0
+'z',    # 0xa1
+'`',    # 0xa2
+'gh',    # 0xa3
+'q',    # 0xa4
+'w',    # 0xa5
+'a',    # 0xa6
+'aa',    # 0xa7
+'i',    # 0xa8
+'ee',    # 0xa9
+'u',    # 0xaa
+'oo',    # 0xab
+'e',    # 0xac
+'ey',    # 0xad
+'o',    # 0xae
+'oa',    # 0xaf
+'',    # 0xb0
+'[?]',    # 0xb1
+'[?]',    # 0xb2
+'[?]',    # 0xb3
+'[?]',    # 0xb4
+'[?]',    # 0xb5
+'[?]',    # 0xb6
+'[?]',    # 0xb7
+'[?]',    # 0xb8
+'[?]',    # 0xb9
+'[?]',    # 0xba
+'[?]',    # 0xbb
+'[?]',    # 0xbc
+'[?]',    # 0xbd
+'[?]',    # 0xbe
+'[?]',    # 0xbf
+'[?]',    # 0xc0
+'[?]',    # 0xc1
+'[?]',    # 0xc2
+'[?]',    # 0xc3
+'[?]',    # 0xc4
+'[?]',    # 0xc5
+'[?]',    # 0xc6
+'[?]',    # 0xc7
+'[?]',    # 0xc8
+'[?]',    # 0xc9
+'[?]',    # 0xca
+'[?]',    # 0xcb
+'[?]',    # 0xcc
+'[?]',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'[?]',    # 0xd0
+'[?]',    # 0xd1
+'[?]',    # 0xd2
+'[?]',    # 0xd3
+'[?]',    # 0xd4
+'[?]',    # 0xd5
+'[?]',    # 0xd6
+'[?]',    # 0xd7
+'[?]',    # 0xd8
+'[?]',    # 0xd9
+'[?]',    # 0xda
+'[?]',    # 0xdb
+'[?]',    # 0xdc
+'[?]',    # 0xdd
+'[?]',    # 0xde
+'[?]',    # 0xdf
+'[?]',    # 0xe0
+'[?]',    # 0xe1
+'[?]',    # 0xe2
+'[?]',    # 0xe3
+'[?]',    # 0xe4
+'[?]',    # 0xe5
+'[?]',    # 0xe6
+'[?]',    # 0xe7
+'[?]',    # 0xe8
+'[?]',    # 0xe9
+'[?]',    # 0xea
+'[?]',    # 0xeb
+'[?]',    # 0xec
+'[?]',    # 0xed
+'[?]',    # 0xee
+'[?]',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'[?]',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x009.py

@@ -0,0 +1,257 @@
+data = (
+'[?]',    # 0x00
+'N',    # 0x01
+'N',    # 0x02
+'H',    # 0x03
+'[?]',    # 0x04
+'a',    # 0x05
+'aa',    # 0x06
+'i',    # 0x07
+'ii',    # 0x08
+'u',    # 0x09
+'uu',    # 0x0a
+'R',    # 0x0b
+'L',    # 0x0c
+'eN',    # 0x0d
+'e',    # 0x0e
+'e',    # 0x0f
+'ai',    # 0x10
+'oN',    # 0x11
+'o',    # 0x12
+'o',    # 0x13
+'au',    # 0x14
+'k',    # 0x15
+'kh',    # 0x16
+'g',    # 0x17
+'gh',    # 0x18
+'ng',    # 0x19
+'c',    # 0x1a
+'ch',    # 0x1b
+'j',    # 0x1c
+'jh',    # 0x1d
+'ny',    # 0x1e
+'tt',    # 0x1f
+'tth',    # 0x20
+'dd',    # 0x21
+'ddh',    # 0x22
+'nn',    # 0x23
+'t',    # 0x24
+'th',    # 0x25
+'d',    # 0x26
+'dh',    # 0x27
+'n',    # 0x28
+'nnn',    # 0x29
+'p',    # 0x2a
+'ph',    # 0x2b
+'b',    # 0x2c
+'bh',    # 0x2d
+'m',    # 0x2e
+'y',    # 0x2f
+'r',    # 0x30
+'rr',    # 0x31
+'l',    # 0x32
+'l',    # 0x33
+'lll',    # 0x34
+'v',    # 0x35
+'sh',    # 0x36
+'ss',    # 0x37
+'s',    # 0x38
+'h',    # 0x39
+'[?]',    # 0x3a
+'[?]',    # 0x3b
+'\'',    # 0x3c
+'\'',    # 0x3d
+'aa',    # 0x3e
+'i',    # 0x3f
+'ii',    # 0x40
+'u',    # 0x41
+'uu',    # 0x42
+'R',    # 0x43
+'RR',    # 0x44
+'eN',    # 0x45
+'e',    # 0x46
+'e',    # 0x47
+'ai',    # 0x48
+'oN',    # 0x49
+'o',    # 0x4a
+'o',    # 0x4b
+'au',    # 0x4c
+'',    # 0x4d
+'[?]',    # 0x4e
+'[?]',    # 0x4f
+'AUM',    # 0x50
+'\'',    # 0x51
+'\'',    # 0x52
+'`',    # 0x53
+'\'',    # 0x54
+'[?]',    # 0x55
+'[?]',    # 0x56
+'[?]',    # 0x57
+'q',    # 0x58
+'khh',    # 0x59
+'ghh',    # 0x5a
+'z',    # 0x5b
+'dddh',    # 0x5c
+'rh',    # 0x5d
+'f',    # 0x5e
+'yy',    # 0x5f
+'RR',    # 0x60
+'LL',    # 0x61
+'L',    # 0x62
+'LL',    # 0x63
+' / ',    # 0x64
+' // ',    # 0x65
+'0',    # 0x66
+'1',    # 0x67
+'2',    # 0x68
+'3',    # 0x69
+'4',    # 0x6a
+'5',    # 0x6b
+'6',    # 0x6c
+'7',    # 0x6d
+'8',    # 0x6e
+'9',    # 0x6f
+'.',    # 0x70
+'[?]',    # 0x71
+'[?]',    # 0x72
+'[?]',    # 0x73
+'[?]',    # 0x74
+'[?]',    # 0x75
+'[?]',    # 0x76
+'[?]',    # 0x77
+'[?]',    # 0x78
+'[?]',    # 0x79
+'[?]',    # 0x7a
+'[?]',    # 0x7b
+'[?]',    # 0x7c
+'[?]',    # 0x7d
+'[?]',    # 0x7e
+'[?]',    # 0x7f
+'[?]',    # 0x80
+'N',    # 0x81
+'N',    # 0x82
+'H',    # 0x83
+'[?]',    # 0x84
+'a',    # 0x85
+'aa',    # 0x86
+'i',    # 0x87
+'ii',    # 0x88
+'u',    # 0x89
+'uu',    # 0x8a
+'R',    # 0x8b
+'RR',    # 0x8c
+'[?]',    # 0x8d
+'[?]',    # 0x8e
+'e',    # 0x8f
+'ai',    # 0x90
+'[?]',    # 0x91
+'[?]',    # 0x92
+'o',    # 0x93
+'au',    # 0x94
+'k',    # 0x95
+'kh',    # 0x96
+'g',    # 0x97
+'gh',    # 0x98
+'ng',    # 0x99
+'c',    # 0x9a
+'ch',    # 0x9b
+'j',    # 0x9c
+'jh',    # 0x9d
+'ny',    # 0x9e
+'tt',    # 0x9f
+'tth',    # 0xa0
+'dd',    # 0xa1
+'ddh',    # 0xa2
+'nn',    # 0xa3
+'t',    # 0xa4
+'th',    # 0xa5
+'d',    # 0xa6
+'dh',    # 0xa7
+'n',    # 0xa8
+'[?]',    # 0xa9
+'p',    # 0xaa
+'ph',    # 0xab
+'b',    # 0xac
+'bh',    # 0xad
+'m',    # 0xae
+'y',    # 0xaf
+'r',    # 0xb0
+'[?]',    # 0xb1
+'l',    # 0xb2
+'[?]',    # 0xb3
+'[?]',    # 0xb4
+'[?]',    # 0xb5
+'sh',    # 0xb6
+'ss',    # 0xb7
+'s',    # 0xb8
+'h',    # 0xb9
+'[?]',    # 0xba
+'[?]',    # 0xbb
+'\'',    # 0xbc
+'[?]',    # 0xbd
+'aa',    # 0xbe
+'i',    # 0xbf
+'ii',    # 0xc0
+'u',    # 0xc1
+'uu',    # 0xc2
+'R',    # 0xc3
+'RR',    # 0xc4
+'[?]',    # 0xc5
+'[?]',    # 0xc6
+'e',    # 0xc7
+'ai',    # 0xc8
+'[?]',    # 0xc9
+'[?]',    # 0xca
+'o',    # 0xcb
+'au',    # 0xcc
+'',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'[?]',    # 0xd0
+'[?]',    # 0xd1
+'[?]',    # 0xd2
+'[?]',    # 0xd3
+'[?]',    # 0xd4
+'[?]',    # 0xd5
+'[?]',    # 0xd6
+'+',    # 0xd7
+'[?]',    # 0xd8
+'[?]',    # 0xd9
+'[?]',    # 0xda
+'[?]',    # 0xdb
+'rr',    # 0xdc
+'rh',    # 0xdd
+'[?]',    # 0xde
+'yy',    # 0xdf
+'RR',    # 0xe0
+'LL',    # 0xe1
+'L',    # 0xe2
+'LL',    # 0xe3
+'[?]',    # 0xe4
+'[?]',    # 0xe5
+'0',    # 0xe6
+'1',    # 0xe7
+'2',    # 0xe8
+'3',    # 0xe9
+'4',    # 0xea
+'5',    # 0xeb
+'6',    # 0xec
+'7',    # 0xed
+'8',    # 0xee
+'9',    # 0xef
+'r\'',    # 0xf0
+'r`',    # 0xf1
+'Rs',    # 0xf2
+'Rs',    # 0xf3
+'1/',    # 0xf4
+'2/',    # 0xf5
+'3/',    # 0xf6
+'4/',    # 0xf7
+' 1 - 1/',    # 0xf8
+'/16',    # 0xf9
+'',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x00a.py

@@ -0,0 +1,257 @@
+data = (
+'[?]',    # 0x00
+'[?]',    # 0x01
+'N',    # 0x02
+'[?]',    # 0x03
+'[?]',    # 0x04
+'a',    # 0x05
+'aa',    # 0x06
+'i',    # 0x07
+'ii',    # 0x08
+'u',    # 0x09
+'uu',    # 0x0a
+'[?]',    # 0x0b
+'[?]',    # 0x0c
+'[?]',    # 0x0d
+'[?]',    # 0x0e
+'ee',    # 0x0f
+'ai',    # 0x10
+'[?]',    # 0x11
+'[?]',    # 0x12
+'oo',    # 0x13
+'au',    # 0x14
+'k',    # 0x15
+'kh',    # 0x16
+'g',    # 0x17
+'gh',    # 0x18
+'ng',    # 0x19
+'c',    # 0x1a
+'ch',    # 0x1b
+'j',    # 0x1c
+'jh',    # 0x1d
+'ny',    # 0x1e
+'tt',    # 0x1f
+'tth',    # 0x20
+'dd',    # 0x21
+'ddh',    # 0x22
+'nn',    # 0x23
+'t',    # 0x24
+'th',    # 0x25
+'d',    # 0x26
+'dh',    # 0x27
+'n',    # 0x28
+'[?]',    # 0x29
+'p',    # 0x2a
+'ph',    # 0x2b
+'b',    # 0x2c
+'bb',    # 0x2d
+'m',    # 0x2e
+'y',    # 0x2f
+'r',    # 0x30
+'[?]',    # 0x31
+'l',    # 0x32
+'ll',    # 0x33
+'[?]',    # 0x34
+'v',    # 0x35
+'sh',    # 0x36
+'[?]',    # 0x37
+'s',    # 0x38
+'h',    # 0x39
+'[?]',    # 0x3a
+'[?]',    # 0x3b
+'\'',    # 0x3c
+'[?]',    # 0x3d
+'aa',    # 0x3e
+'i',    # 0x3f
+'ii',    # 0x40
+'u',    # 0x41
+'uu',    # 0x42
+'[?]',    # 0x43
+'[?]',    # 0x44
+'[?]',    # 0x45
+'[?]',    # 0x46
+'ee',    # 0x47
+'ai',    # 0x48
+'[?]',    # 0x49
+'[?]',    # 0x4a
+'oo',    # 0x4b
+'au',    # 0x4c
+'',    # 0x4d
+'[?]',    # 0x4e
+'[?]',    # 0x4f
+'[?]',    # 0x50
+'[?]',    # 0x51
+'[?]',    # 0x52
+'[?]',    # 0x53
+'[?]',    # 0x54
+'[?]',    # 0x55
+'[?]',    # 0x56
+'[?]',    # 0x57
+'[?]',    # 0x58
+'khh',    # 0x59
+'ghh',    # 0x5a
+'z',    # 0x5b
+'rr',    # 0x5c
+'[?]',    # 0x5d
+'f',    # 0x5e
+'[?]',    # 0x5f
+'[?]',    # 0x60
+'[?]',    # 0x61
+'[?]',    # 0x62
+'[?]',    # 0x63
+'[?]',    # 0x64
+'[?]',    # 0x65
+'0',    # 0x66
+'1',    # 0x67
+'2',    # 0x68
+'3',    # 0x69
+'4',    # 0x6a
+'5',    # 0x6b
+'6',    # 0x6c
+'7',    # 0x6d
+'8',    # 0x6e
+'9',    # 0x6f
+'N',    # 0x70
+'H',    # 0x71
+'',    # 0x72
+'',    # 0x73
+'G.E.O.',    # 0x74
+'[?]',    # 0x75
+'[?]',    # 0x76
+'[?]',    # 0x77
+'[?]',    # 0x78
+'[?]',    # 0x79
+'[?]',    # 0x7a
+'[?]',    # 0x7b
+'[?]',    # 0x7c
+'[?]',    # 0x7d
+'[?]',    # 0x7e
+'[?]',    # 0x7f
+'[?]',    # 0x80
+'N',    # 0x81
+'N',    # 0x82
+'H',    # 0x83
+'[?]',    # 0x84
+'a',    # 0x85
+'aa',    # 0x86
+'i',    # 0x87
+'ii',    # 0x88
+'u',    # 0x89
+'uu',    # 0x8a
+'R',    # 0x8b
+'[?]',    # 0x8c
+'eN',    # 0x8d
+'[?]',    # 0x8e
+'e',    # 0x8f
+'ai',    # 0x90
+'oN',    # 0x91
+'[?]',    # 0x92
+'o',    # 0x93
+'au',    # 0x94
+'k',    # 0x95
+'kh',    # 0x96
+'g',    # 0x97
+'gh',    # 0x98
+'ng',    # 0x99
+'c',    # 0x9a
+'ch',    # 0x9b
+'j',    # 0x9c
+'jh',    # 0x9d
+'ny',    # 0x9e
+'tt',    # 0x9f
+'tth',    # 0xa0
+'dd',    # 0xa1
+'ddh',    # 0xa2
+'nn',    # 0xa3
+'t',    # 0xa4
+'th',    # 0xa5
+'d',    # 0xa6
+'dh',    # 0xa7
+'n',    # 0xa8
+'[?]',    # 0xa9
+'p',    # 0xaa
+'ph',    # 0xab
+'b',    # 0xac
+'bh',    # 0xad
+'m',    # 0xae
+'ya',    # 0xaf
+'r',    # 0xb0
+'[?]',    # 0xb1
+'l',    # 0xb2
+'ll',    # 0xb3
+'[?]',    # 0xb4
+'v',    # 0xb5
+'sh',    # 0xb6
+'ss',    # 0xb7
+'s',    # 0xb8
+'h',    # 0xb9
+'[?]',    # 0xba
+'[?]',    # 0xbb
+'\'',    # 0xbc
+'\'',    # 0xbd
+'aa',    # 0xbe
+'i',    # 0xbf
+'ii',    # 0xc0
+'u',    # 0xc1
+'uu',    # 0xc2
+'R',    # 0xc3
+'RR',    # 0xc4
+'eN',    # 0xc5
+'[?]',    # 0xc6
+'e',    # 0xc7
+'ai',    # 0xc8
+'oN',    # 0xc9
+'[?]',    # 0xca
+'o',    # 0xcb
+'au',    # 0xcc
+'',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'AUM',    # 0xd0
+'[?]',    # 0xd1
+'[?]',    # 0xd2
+'[?]',    # 0xd3
+'[?]',    # 0xd4
+'[?]',    # 0xd5
+'[?]',    # 0xd6
+'[?]',    # 0xd7
+'[?]',    # 0xd8
+'[?]',    # 0xd9
+'[?]',    # 0xda
+'[?]',    # 0xdb
+'[?]',    # 0xdc
+'[?]',    # 0xdd
+'[?]',    # 0xde
+'[?]',    # 0xdf
+'RR',    # 0xe0
+'[?]',    # 0xe1
+'[?]',    # 0xe2
+'[?]',    # 0xe3
+'[?]',    # 0xe4
+'[?]',    # 0xe5
+'0',    # 0xe6
+'1',    # 0xe7
+'2',    # 0xe8
+'3',    # 0xe9
+'4',    # 0xea
+'5',    # 0xeb
+'6',    # 0xec
+'7',    # 0xed
+'8',    # 0xee
+'9',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'[?]',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x00b.py

@@ -0,0 +1,257 @@
+data = (
+'[?]',    # 0x00
+'N',    # 0x01
+'N',    # 0x02
+'H',    # 0x03
+'[?]',    # 0x04
+'a',    # 0x05
+'aa',    # 0x06
+'i',    # 0x07
+'ii',    # 0x08
+'u',    # 0x09
+'uu',    # 0x0a
+'R',    # 0x0b
+'L',    # 0x0c
+'[?]',    # 0x0d
+'[?]',    # 0x0e
+'e',    # 0x0f
+'ai',    # 0x10
+'[?]',    # 0x11
+'[?]',    # 0x12
+'o',    # 0x13
+'au',    # 0x14
+'k',    # 0x15
+'kh',    # 0x16
+'g',    # 0x17
+'gh',    # 0x18
+'ng',    # 0x19
+'c',    # 0x1a
+'ch',    # 0x1b
+'j',    # 0x1c
+'jh',    # 0x1d
+'ny',    # 0x1e
+'tt',    # 0x1f
+'tth',    # 0x20
+'dd',    # 0x21
+'ddh',    # 0x22
+'nn',    # 0x23
+'t',    # 0x24
+'th',    # 0x25
+'d',    # 0x26
+'dh',    # 0x27
+'n',    # 0x28
+'[?]',    # 0x29
+'p',    # 0x2a
+'ph',    # 0x2b
+'b',    # 0x2c
+'bh',    # 0x2d
+'m',    # 0x2e
+'y',    # 0x2f
+'r',    # 0x30
+'[?]',    # 0x31
+'l',    # 0x32
+'ll',    # 0x33
+'[?]',    # 0x34
+'',    # 0x35
+'sh',    # 0x36
+'ss',    # 0x37
+'s',    # 0x38
+'h',    # 0x39
+'[?]',    # 0x3a
+'[?]',    # 0x3b
+'\'',    # 0x3c
+'\'',    # 0x3d
+'aa',    # 0x3e
+'i',    # 0x3f
+'ii',    # 0x40
+'u',    # 0x41
+'uu',    # 0x42
+'R',    # 0x43
+'[?]',    # 0x44
+'[?]',    # 0x45
+'[?]',    # 0x46
+'e',    # 0x47
+'ai',    # 0x48
+'[?]',    # 0x49
+'[?]',    # 0x4a
+'o',    # 0x4b
+'au',    # 0x4c
+'',    # 0x4d
+'[?]',    # 0x4e
+'[?]',    # 0x4f
+'[?]',    # 0x50
+'[?]',    # 0x51
+'[?]',    # 0x52
+'[?]',    # 0x53
+'[?]',    # 0x54
+'[?]',    # 0x55
+'+',    # 0x56
+'+',    # 0x57
+'[?]',    # 0x58
+'[?]',    # 0x59
+'[?]',    # 0x5a
+'[?]',    # 0x5b
+'rr',    # 0x5c
+'rh',    # 0x5d
+'[?]',    # 0x5e
+'yy',    # 0x5f
+'RR',    # 0x60
+'LL',    # 0x61
+'[?]',    # 0x62
+'[?]',    # 0x63
+'[?]',    # 0x64
+'[?]',    # 0x65
+'0',    # 0x66
+'1',    # 0x67
+'2',    # 0x68
+'3',    # 0x69
+'4',    # 0x6a
+'5',    # 0x6b
+'6',    # 0x6c
+'7',    # 0x6d
+'8',    # 0x6e
+'9',    # 0x6f
+'',    # 0x70
+'[?]',    # 0x71
+'[?]',    # 0x72
+'[?]',    # 0x73
+'[?]',    # 0x74
+'[?]',    # 0x75
+'[?]',    # 0x76
+'[?]',    # 0x77
+'[?]',    # 0x78
+'[?]',    # 0x79
+'[?]',    # 0x7a
+'[?]',    # 0x7b
+'[?]',    # 0x7c
+'[?]',    # 0x7d
+'[?]',    # 0x7e
+'[?]',    # 0x7f
+'[?]',    # 0x80
+'[?]',    # 0x81
+'N',    # 0x82
+'H',    # 0x83
+'[?]',    # 0x84
+'a',    # 0x85
+'aa',    # 0x86
+'i',    # 0x87
+'ii',    # 0x88
+'u',    # 0x89
+'uu',    # 0x8a
+'[?]',    # 0x8b
+'[?]',    # 0x8c
+'[?]',    # 0x8d
+'e',    # 0x8e
+'ee',    # 0x8f
+'ai',    # 0x90
+'[?]',    # 0x91
+'o',    # 0x92
+'oo',    # 0x93
+'au',    # 0x94
+'k',    # 0x95
+'[?]',    # 0x96
+'[?]',    # 0x97
+'[?]',    # 0x98
+'ng',    # 0x99
+'c',    # 0x9a
+'[?]',    # 0x9b
+'j',    # 0x9c
+'[?]',    # 0x9d
+'ny',    # 0x9e
+'tt',    # 0x9f
+'[?]',    # 0xa0
+'[?]',    # 0xa1
+'[?]',    # 0xa2
+'nn',    # 0xa3
+'t',    # 0xa4
+'[?]',    # 0xa5
+'[?]',    # 0xa6
+'[?]',    # 0xa7
+'n',    # 0xa8
+'nnn',    # 0xa9
+'p',    # 0xaa
+'[?]',    # 0xab
+'[?]',    # 0xac
+'[?]',    # 0xad
+'m',    # 0xae
+'y',    # 0xaf
+'r',    # 0xb0
+'rr',    # 0xb1
+'l',    # 0xb2
+'ll',    # 0xb3
+'lll',    # 0xb4
+'v',    # 0xb5
+'[?]',    # 0xb6
+'ss',    # 0xb7
+'s',    # 0xb8
+'h',    # 0xb9
+'[?]',    # 0xba
+'[?]',    # 0xbb
+'[?]',    # 0xbc
+'[?]',    # 0xbd
+'aa',    # 0xbe
+'i',    # 0xbf
+'ii',    # 0xc0
+'u',    # 0xc1
+'uu',    # 0xc2
+'[?]',    # 0xc3
+'[?]',    # 0xc4
+'[?]',    # 0xc5
+'e',    # 0xc6
+'ee',    # 0xc7
+'ai',    # 0xc8
+'[?]',    # 0xc9
+'o',    # 0xca
+'oo',    # 0xcb
+'au',    # 0xcc
+'',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'[?]',    # 0xd0
+'[?]',    # 0xd1
+'[?]',    # 0xd2
+'[?]',    # 0xd3
+'[?]',    # 0xd4
+'[?]',    # 0xd5
+'[?]',    # 0xd6
+'+',    # 0xd7
+'[?]',    # 0xd8
+'[?]',    # 0xd9
+'[?]',    # 0xda
+'[?]',    # 0xdb
+'[?]',    # 0xdc
+'[?]',    # 0xdd
+'[?]',    # 0xde
+'[?]',    # 0xdf
+'[?]',    # 0xe0
+'[?]',    # 0xe1
+'[?]',    # 0xe2
+'[?]',    # 0xe3
+'[?]',    # 0xe4
+'[?]',    # 0xe5
+'0',    # 0xe6
+'1',    # 0xe7
+'2',    # 0xe8
+'3',    # 0xe9
+'4',    # 0xea
+'5',    # 0xeb
+'6',    # 0xec
+'7',    # 0xed
+'8',    # 0xee
+'9',    # 0xef
+'+10+',    # 0xf0
+'+100+',    # 0xf1
+'+1000+',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x00c.py

@@ -0,0 +1,257 @@
+data = (
+'[?]',    # 0x00
+'N',    # 0x01
+'N',    # 0x02
+'H',    # 0x03
+'[?]',    # 0x04
+'a',    # 0x05
+'aa',    # 0x06
+'i',    # 0x07
+'ii',    # 0x08
+'u',    # 0x09
+'uu',    # 0x0a
+'R',    # 0x0b
+'L',    # 0x0c
+'[?]',    # 0x0d
+'e',    # 0x0e
+'ee',    # 0x0f
+'ai',    # 0x10
+'[?]',    # 0x11
+'o',    # 0x12
+'oo',    # 0x13
+'au',    # 0x14
+'k',    # 0x15
+'kh',    # 0x16
+'g',    # 0x17
+'gh',    # 0x18
+'ng',    # 0x19
+'c',    # 0x1a
+'ch',    # 0x1b
+'j',    # 0x1c
+'jh',    # 0x1d
+'ny',    # 0x1e
+'tt',    # 0x1f
+'tth',    # 0x20
+'dd',    # 0x21
+'ddh',    # 0x22
+'nn',    # 0x23
+'t',    # 0x24
+'th',    # 0x25
+'d',    # 0x26
+'dh',    # 0x27
+'n',    # 0x28
+'[?]',    # 0x29
+'p',    # 0x2a
+'ph',    # 0x2b
+'b',    # 0x2c
+'bh',    # 0x2d
+'m',    # 0x2e
+'y',    # 0x2f
+'r',    # 0x30
+'rr',    # 0x31
+'l',    # 0x32
+'ll',    # 0x33
+'[?]',    # 0x34
+'v',    # 0x35
+'sh',    # 0x36
+'ss',    # 0x37
+'s',    # 0x38
+'h',    # 0x39
+'[?]',    # 0x3a
+'[?]',    # 0x3b
+'[?]',    # 0x3c
+'[?]',    # 0x3d
+'aa',    # 0x3e
+'i',    # 0x3f
+'ii',    # 0x40
+'u',    # 0x41
+'uu',    # 0x42
+'R',    # 0x43
+'RR',    # 0x44
+'[?]',    # 0x45
+'e',    # 0x46
+'ee',    # 0x47
+'ai',    # 0x48
+'[?]',    # 0x49
+'o',    # 0x4a
+'oo',    # 0x4b
+'au',    # 0x4c
+'',    # 0x4d
+'[?]',    # 0x4e
+'[?]',    # 0x4f
+'[?]',    # 0x50
+'[?]',    # 0x51
+'[?]',    # 0x52
+'[?]',    # 0x53
+'[?]',    # 0x54
+'+',    # 0x55
+'+',    # 0x56
+'[?]',    # 0x57
+'[?]',    # 0x58
+'[?]',    # 0x59
+'[?]',    # 0x5a
+'[?]',    # 0x5b
+'[?]',    # 0x5c
+'[?]',    # 0x5d
+'[?]',    # 0x5e
+'[?]',    # 0x5f
+'RR',    # 0x60
+'LL',    # 0x61
+'[?]',    # 0x62
+'[?]',    # 0x63
+'[?]',    # 0x64
+'[?]',    # 0x65
+'0',    # 0x66
+'1',    # 0x67
+'2',    # 0x68
+'3',    # 0x69
+'4',    # 0x6a
+'5',    # 0x6b
+'6',    # 0x6c
+'7',    # 0x6d
+'8',    # 0x6e
+'9',    # 0x6f
+'[?]',    # 0x70
+'[?]',    # 0x71
+'[?]',    # 0x72
+'[?]',    # 0x73
+'[?]',    # 0x74
+'[?]',    # 0x75
+'[?]',    # 0x76
+'[?]',    # 0x77
+'[?]',    # 0x78
+'[?]',    # 0x79
+'[?]',    # 0x7a
+'[?]',    # 0x7b
+'[?]',    # 0x7c
+'[?]',    # 0x7d
+'[?]',    # 0x7e
+'[?]',    # 0x7f
+'[?]',    # 0x80
+'[?]',    # 0x81
+'N',    # 0x82
+'H',    # 0x83
+'[?]',    # 0x84
+'a',    # 0x85
+'aa',    # 0x86
+'i',    # 0x87
+'ii',    # 0x88
+'u',    # 0x89
+'uu',    # 0x8a
+'R',    # 0x8b
+'L',    # 0x8c
+'[?]',    # 0x8d
+'e',    # 0x8e
+'ee',    # 0x8f
+'ai',    # 0x90
+'[?]',    # 0x91
+'o',    # 0x92
+'oo',    # 0x93
+'au',    # 0x94
+'k',    # 0x95
+'kh',    # 0x96
+'g',    # 0x97
+'gh',    # 0x98
+'ng',    # 0x99
+'c',    # 0x9a
+'ch',    # 0x9b
+'j',    # 0x9c
+'jh',    # 0x9d
+'ny',    # 0x9e
+'tt',    # 0x9f
+'tth',    # 0xa0
+'dd',    # 0xa1
+'ddh',    # 0xa2
+'nn',    # 0xa3
+'t',    # 0xa4
+'th',    # 0xa5
+'d',    # 0xa6
+'dh',    # 0xa7
+'n',    # 0xa8
+'[?]',    # 0xa9
+'p',    # 0xaa
+'ph',    # 0xab
+'b',    # 0xac
+'bh',    # 0xad
+'m',    # 0xae
+'y',    # 0xaf
+'r',    # 0xb0
+'rr',    # 0xb1
+'l',    # 0xb2
+'ll',    # 0xb3
+'[?]',    # 0xb4
+'v',    # 0xb5
+'sh',    # 0xb6
+'ss',    # 0xb7
+'s',    # 0xb8
+'h',    # 0xb9
+'[?]',    # 0xba
+'[?]',    # 0xbb
+'[?]',    # 0xbc
+'[?]',    # 0xbd
+'aa',    # 0xbe
+'i',    # 0xbf
+'ii',    # 0xc0
+'u',    # 0xc1
+'uu',    # 0xc2
+'R',    # 0xc3
+'RR',    # 0xc4
+'[?]',    # 0xc5
+'e',    # 0xc6
+'ee',    # 0xc7
+'ai',    # 0xc8
+'[?]',    # 0xc9
+'o',    # 0xca
+'oo',    # 0xcb
+'au',    # 0xcc
+'',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'[?]',    # 0xd0
+'[?]',    # 0xd1
+'[?]',    # 0xd2
+'[?]',    # 0xd3
+'[?]',    # 0xd4
+'+',    # 0xd5
+'+',    # 0xd6
+'[?]',    # 0xd7
+'[?]',    # 0xd8
+'[?]',    # 0xd9
+'[?]',    # 0xda
+'[?]',    # 0xdb
+'[?]',    # 0xdc
+'[?]',    # 0xdd
+'lll',    # 0xde
+'[?]',    # 0xdf
+'RR',    # 0xe0
+'LL',    # 0xe1
+'[?]',    # 0xe2
+'[?]',    # 0xe3
+'[?]',    # 0xe4
+'[?]',    # 0xe5
+'0',    # 0xe6
+'1',    # 0xe7
+'2',    # 0xe8
+'3',    # 0xe9
+'4',    # 0xea
+'5',    # 0xeb
+'6',    # 0xec
+'7',    # 0xed
+'8',    # 0xee
+'9',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'[?]',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x00d.py

@@ -0,0 +1,257 @@
+data = (
+'[?]',    # 0x00
+'[?]',    # 0x01
+'N',    # 0x02
+'H',    # 0x03
+'[?]',    # 0x04
+'a',    # 0x05
+'aa',    # 0x06
+'i',    # 0x07
+'ii',    # 0x08
+'u',    # 0x09
+'uu',    # 0x0a
+'R',    # 0x0b
+'L',    # 0x0c
+'[?]',    # 0x0d
+'e',    # 0x0e
+'ee',    # 0x0f
+'ai',    # 0x10
+'[?]',    # 0x11
+'o',    # 0x12
+'oo',    # 0x13
+'au',    # 0x14
+'k',    # 0x15
+'kh',    # 0x16
+'g',    # 0x17
+'gh',    # 0x18
+'ng',    # 0x19
+'c',    # 0x1a
+'ch',    # 0x1b
+'j',    # 0x1c
+'jh',    # 0x1d
+'ny',    # 0x1e
+'tt',    # 0x1f
+'tth',    # 0x20
+'dd',    # 0x21
+'ddh',    # 0x22
+'nn',    # 0x23
+'t',    # 0x24
+'th',    # 0x25
+'d',    # 0x26
+'dh',    # 0x27
+'n',    # 0x28
+'[?]',    # 0x29
+'p',    # 0x2a
+'ph',    # 0x2b
+'b',    # 0x2c
+'bh',    # 0x2d
+'m',    # 0x2e
+'y',    # 0x2f
+'r',    # 0x30
+'rr',    # 0x31
+'l',    # 0x32
+'ll',    # 0x33
+'lll',    # 0x34
+'v',    # 0x35
+'sh',    # 0x36
+'ss',    # 0x37
+'s',    # 0x38
+'h',    # 0x39
+'[?]',    # 0x3a
+'[?]',    # 0x3b
+'[?]',    # 0x3c
+'[?]',    # 0x3d
+'aa',    # 0x3e
+'i',    # 0x3f
+'ii',    # 0x40
+'u',    # 0x41
+'uu',    # 0x42
+'R',    # 0x43
+'[?]',    # 0x44
+'[?]',    # 0x45
+'e',    # 0x46
+'ee',    # 0x47
+'ai',    # 0x48
+'',    # 0x49
+'o',    # 0x4a
+'oo',    # 0x4b
+'au',    # 0x4c
+'',    # 0x4d
+'[?]',    # 0x4e
+'[?]',    # 0x4f
+'[?]',    # 0x50
+'[?]',    # 0x51
+'[?]',    # 0x52
+'[?]',    # 0x53
+'[?]',    # 0x54
+'[?]',    # 0x55
+'[?]',    # 0x56
+'+',    # 0x57
+'[?]',    # 0x58
+'[?]',    # 0x59
+'[?]',    # 0x5a
+'[?]',    # 0x5b
+'[?]',    # 0x5c
+'[?]',    # 0x5d
+'[?]',    # 0x5e
+'[?]',    # 0x5f
+'RR',    # 0x60
+'LL',    # 0x61
+'[?]',    # 0x62
+'[?]',    # 0x63
+'[?]',    # 0x64
+'[?]',    # 0x65
+'0',    # 0x66
+'1',    # 0x67
+'2',    # 0x68
+'3',    # 0x69
+'4',    # 0x6a
+'5',    # 0x6b
+'6',    # 0x6c
+'7',    # 0x6d
+'8',    # 0x6e
+'9',    # 0x6f
+'[?]',    # 0x70
+'[?]',    # 0x71
+'[?]',    # 0x72
+'[?]',    # 0x73
+'[?]',    # 0x74
+'[?]',    # 0x75
+'[?]',    # 0x76
+'[?]',    # 0x77
+'[?]',    # 0x78
+'[?]',    # 0x79
+'[?]',    # 0x7a
+'[?]',    # 0x7b
+'[?]',    # 0x7c
+'[?]',    # 0x7d
+'[?]',    # 0x7e
+'[?]',    # 0x7f
+'[?]',    # 0x80
+'[?]',    # 0x81
+'N',    # 0x82
+'H',    # 0x83
+'[?]',    # 0x84
+'a',    # 0x85
+'aa',    # 0x86
+'ae',    # 0x87
+'aae',    # 0x88
+'i',    # 0x89
+'ii',    # 0x8a
+'u',    # 0x8b
+'uu',    # 0x8c
+'R',    # 0x8d
+'RR',    # 0x8e
+'L',    # 0x8f
+'LL',    # 0x90
+'e',    # 0x91
+'ee',    # 0x92
+'ai',    # 0x93
+'o',    # 0x94
+'oo',    # 0x95
+'au',    # 0x96
+'[?]',    # 0x97
+'[?]',    # 0x98
+'[?]',    # 0x99
+'k',    # 0x9a
+'kh',    # 0x9b
+'g',    # 0x9c
+'gh',    # 0x9d
+'ng',    # 0x9e
+'nng',    # 0x9f
+'c',    # 0xa0
+'ch',    # 0xa1
+'j',    # 0xa2
+'jh',    # 0xa3
+'ny',    # 0xa4
+'jny',    # 0xa5
+'nyj',    # 0xa6
+'tt',    # 0xa7
+'tth',    # 0xa8
+'dd',    # 0xa9
+'ddh',    # 0xaa
+'nn',    # 0xab
+'nndd',    # 0xac
+'t',    # 0xad
+'th',    # 0xae
+'d',    # 0xaf
+'dh',    # 0xb0
+'n',    # 0xb1
+'[?]',    # 0xb2
+'nd',    # 0xb3
+'p',    # 0xb4
+'ph',    # 0xb5
+'b',    # 0xb6
+'bh',    # 0xb7
+'m',    # 0xb8
+'mb',    # 0xb9
+'y',    # 0xba
+'r',    # 0xbb
+'[?]',    # 0xbc
+'l',    # 0xbd
+'[?]',    # 0xbe
+'[?]',    # 0xbf
+'v',    # 0xc0
+'sh',    # 0xc1
+'ss',    # 0xc2
+'s',    # 0xc3
+'h',    # 0xc4
+'ll',    # 0xc5
+'f',    # 0xc6
+'[?]',    # 0xc7
+'[?]',    # 0xc8
+'[?]',    # 0xc9
+'',    # 0xca
+'[?]',    # 0xcb
+'[?]',    # 0xcc
+'[?]',    # 0xcd
+'[?]',    # 0xce
+'aa',    # 0xcf
+'ae',    # 0xd0
+'aae',    # 0xd1
+'i',    # 0xd2
+'ii',    # 0xd3
+'u',    # 0xd4
+'[?]',    # 0xd5
+'uu',    # 0xd6
+'[?]',    # 0xd7
+'R',    # 0xd8
+'e',    # 0xd9
+'ee',    # 0xda
+'ai',    # 0xdb
+'o',    # 0xdc
+'oo',    # 0xdd
+'au',    # 0xde
+'L',    # 0xdf
+'[?]',    # 0xe0
+'[?]',    # 0xe1
+'[?]',    # 0xe2
+'[?]',    # 0xe3
+'[?]',    # 0xe4
+'[?]',    # 0xe5
+'[?]',    # 0xe6
+'[?]',    # 0xe7
+'[?]',    # 0xe8
+'[?]',    # 0xe9
+'[?]',    # 0xea
+'[?]',    # 0xeb
+'[?]',    # 0xec
+'[?]',    # 0xed
+'[?]',    # 0xee
+'[?]',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'RR',    # 0xf2
+'LL',    # 0xf3
+' . ',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x00e.py

@@ -0,0 +1,257 @@
+data = (
+'[?]',    # 0x00
+'k',    # 0x01
+'kh',    # 0x02
+'kh',    # 0x03
+'kh',    # 0x04
+'kh',    # 0x05
+'kh',    # 0x06
+'ng',    # 0x07
+'cch',    # 0x08
+'ch',    # 0x09
+'ch',    # 0x0a
+'ch',    # 0x0b
+'ch',    # 0x0c
+'y',    # 0x0d
+'d',    # 0x0e
+'t',    # 0x0f
+'th',    # 0x10
+'th',    # 0x11
+'th',    # 0x12
+'n',    # 0x13
+'d',    # 0x14
+'t',    # 0x15
+'th',    # 0x16
+'th',    # 0x17
+'th',    # 0x18
+'n',    # 0x19
+'b',    # 0x1a
+'p',    # 0x1b
+'ph',    # 0x1c
+'f',    # 0x1d
+'ph',    # 0x1e
+'f',    # 0x1f
+'ph',    # 0x20
+'m',    # 0x21
+'y',    # 0x22
+'r',    # 0x23
+'R',    # 0x24
+'l',    # 0x25
+'L',    # 0x26
+'w',    # 0x27
+'s',    # 0x28
+'s',    # 0x29
+'s',    # 0x2a
+'h',    # 0x2b
+'l',    # 0x2c
+'`',    # 0x2d
+'h',    # 0x2e
+'~',    # 0x2f
+'a',    # 0x30
+'a',    # 0x31
+'aa',    # 0x32
+'am',    # 0x33
+'i',    # 0x34
+'ii',    # 0x35
+'ue',    # 0x36
+'uue',    # 0x37
+'u',    # 0x38
+'uu',    # 0x39
+'\'',    # 0x3a
+'[?]',    # 0x3b
+'[?]',    # 0x3c
+'[?]',    # 0x3d
+'[?]',    # 0x3e
+'Bh.',    # 0x3f
+'e',    # 0x40
+'ae',    # 0x41
+'o',    # 0x42
+'ai',    # 0x43
+'ai',    # 0x44
+'ao',    # 0x45
+'+',    # 0x46
+'',    # 0x47
+'',    # 0x48
+'',    # 0x49
+'',    # 0x4a
+'',    # 0x4b
+'',    # 0x4c
+'M',    # 0x4d
+'',    # 0x4e
+' * ',    # 0x4f
+'0',    # 0x50
+'1',    # 0x51
+'2',    # 0x52
+'3',    # 0x53
+'4',    # 0x54
+'5',    # 0x55
+'6',    # 0x56
+'7',    # 0x57
+'8',    # 0x58
+'9',    # 0x59
+' // ',    # 0x5a
+' /// ',    # 0x5b
+'[?]',    # 0x5c
+'[?]',    # 0x5d
+'[?]',    # 0x5e
+'[?]',    # 0x5f
+'[?]',    # 0x60
+'[?]',    # 0x61
+'[?]',    # 0x62
+'[?]',    # 0x63
+'[?]',    # 0x64
+'[?]',    # 0x65
+'[?]',    # 0x66
+'[?]',    # 0x67
+'[?]',    # 0x68
+'[?]',    # 0x69
+'[?]',    # 0x6a
+'[?]',    # 0x6b
+'[?]',    # 0x6c
+'[?]',    # 0x6d
+'[?]',    # 0x6e
+'[?]',    # 0x6f
+'[?]',    # 0x70
+'[?]',    # 0x71
+'[?]',    # 0x72
+'[?]',    # 0x73
+'[?]',    # 0x74
+'[?]',    # 0x75
+'[?]',    # 0x76
+'[?]',    # 0x77
+'[?]',    # 0x78
+'[?]',    # 0x79
+'[?]',    # 0x7a
+'[?]',    # 0x7b
+'[?]',    # 0x7c
+'[?]',    # 0x7d
+'[?]',    # 0x7e
+'[?]',    # 0x7f
+'[?]',    # 0x80
+'k',    # 0x81
+'kh',    # 0x82
+'[?]',    # 0x83
+'kh',    # 0x84
+'[?]',    # 0x85
+'[?]',    # 0x86
+'ng',    # 0x87
+'ch',    # 0x88
+'[?]',    # 0x89
+'s',    # 0x8a
+'[?]',    # 0x8b
+'[?]',    # 0x8c
+'ny',    # 0x8d
+'[?]',    # 0x8e
+'[?]',    # 0x8f
+'[?]',    # 0x90
+'[?]',    # 0x91
+'[?]',    # 0x92
+'[?]',    # 0x93
+'d',    # 0x94
+'h',    # 0x95
+'th',    # 0x96
+'th',    # 0x97
+'[?]',    # 0x98
+'n',    # 0x99
+'b',    # 0x9a
+'p',    # 0x9b
+'ph',    # 0x9c
+'f',    # 0x9d
+'ph',    # 0x9e
+'f',    # 0x9f
+'[?]',    # 0xa0
+'m',    # 0xa1
+'y',    # 0xa2
+'r',    # 0xa3
+'[?]',    # 0xa4
+'l',    # 0xa5
+'[?]',    # 0xa6
+'w',    # 0xa7
+'[?]',    # 0xa8
+'[?]',    # 0xa9
+'s',    # 0xaa
+'h',    # 0xab
+'[?]',    # 0xac
+'`',    # 0xad
+'',    # 0xae
+'~',    # 0xaf
+'a',    # 0xb0
+'',    # 0xb1
+'aa',    # 0xb2
+'am',    # 0xb3
+'i',    # 0xb4
+'ii',    # 0xb5
+'y',    # 0xb6
+'yy',    # 0xb7
+'u',    # 0xb8
+'uu',    # 0xb9
+'[?]',    # 0xba
+'o',    # 0xbb
+'l',    # 0xbc
+'ny',    # 0xbd
+'[?]',    # 0xbe
+'[?]',    # 0xbf
+'e',    # 0xc0
+'ei',    # 0xc1
+'o',    # 0xc2
+'ay',    # 0xc3
+'ai',    # 0xc4
+'[?]',    # 0xc5
+'+',    # 0xc6
+'[?]',    # 0xc7
+'',    # 0xc8
+'',    # 0xc9
+'',    # 0xca
+'',    # 0xcb
+'',    # 0xcc
+'M',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'0',    # 0xd0
+'1',    # 0xd1
+'2',    # 0xd2
+'3',    # 0xd3
+'4',    # 0xd4
+'5',    # 0xd5
+'6',    # 0xd6
+'7',    # 0xd7
+'8',    # 0xd8
+'9',    # 0xd9
+'[?]',    # 0xda
+'[?]',    # 0xdb
+'hn',    # 0xdc
+'hm',    # 0xdd
+'[?]',    # 0xde
+'[?]',    # 0xdf
+'[?]',    # 0xe0
+'[?]',    # 0xe1
+'[?]',    # 0xe2
+'[?]',    # 0xe3
+'[?]',    # 0xe4
+'[?]',    # 0xe5
+'[?]',    # 0xe6
+'[?]',    # 0xe7
+'[?]',    # 0xe8
+'[?]',    # 0xe9
+'[?]',    # 0xea
+'[?]',    # 0xeb
+'[?]',    # 0xec
+'[?]',    # 0xed
+'[?]',    # 0xee
+'[?]',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'[?]',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x00f.py

@@ -0,0 +1,257 @@
+data = (
+'AUM',    # 0x00
+'',    # 0x01
+'',    # 0x02
+'',    # 0x03
+'',    # 0x04
+'',    # 0x05
+'',    # 0x06
+'',    # 0x07
+' // ',    # 0x08
+' * ',    # 0x09
+'',    # 0x0a
+'-',    # 0x0b
+' / ',    # 0x0c
+' / ',    # 0x0d
+' // ',    # 0x0e
+' -/ ',    # 0x0f
+' +/ ',    # 0x10
+' X/ ',    # 0x11
+' /XX/ ',    # 0x12
+' /X/ ',    # 0x13
+', ',    # 0x14
+'',    # 0x15
+'',    # 0x16
+'',    # 0x17
+'',    # 0x18
+'',    # 0x19
+'',    # 0x1a
+'',    # 0x1b
+'',    # 0x1c
+'',    # 0x1d
+'',    # 0x1e
+'',    # 0x1f
+'0',    # 0x20
+'1',    # 0x21
+'2',    # 0x22
+'3',    # 0x23
+'4',    # 0x24
+'5',    # 0x25
+'6',    # 0x26
+'7',    # 0x27
+'8',    # 0x28
+'9',    # 0x29
+'.5',    # 0x2a
+'1.5',    # 0x2b
+'2.5',    # 0x2c
+'3.5',    # 0x2d
+'4.5',    # 0x2e
+'5.5',    # 0x2f
+'6.5',    # 0x30
+'7.5',    # 0x31
+'8.5',    # 0x32
+'-.5',    # 0x33
+'+',    # 0x34
+'*',    # 0x35
+'^',    # 0x36
+'_',    # 0x37
+'',    # 0x38
+'~',    # 0x39
+'[?]',    # 0x3a
+']',    # 0x3b
+'[[',    # 0x3c
+']]',    # 0x3d
+'',    # 0x3e
+'',    # 0x3f
+'k',    # 0x40
+'kh',    # 0x41
+'g',    # 0x42
+'gh',    # 0x43
+'ng',    # 0x44
+'c',    # 0x45
+'ch',    # 0x46
+'j',    # 0x47
+'[?]',    # 0x48
+'ny',    # 0x49
+'tt',    # 0x4a
+'tth',    # 0x4b
+'dd',    # 0x4c
+'ddh',    # 0x4d
+'nn',    # 0x4e
+'t',    # 0x4f
+'th',    # 0x50
+'d',    # 0x51
+'dh',    # 0x52
+'n',    # 0x53
+'p',    # 0x54
+'ph',    # 0x55
+'b',    # 0x56
+'bh',    # 0x57
+'m',    # 0x58
+'ts',    # 0x59
+'tsh',    # 0x5a
+'dz',    # 0x5b
+'dzh',    # 0x5c
+'w',    # 0x5d
+'zh',    # 0x5e
+'z',    # 0x5f
+'\'',    # 0x60
+'y',    # 0x61
+'r',    # 0x62
+'l',    # 0x63
+'sh',    # 0x64
+'ssh',    # 0x65
+'s',    # 0x66
+'h',    # 0x67
+'a',    # 0x68
+'kss',    # 0x69
+'r',    # 0x6a
+'[?]',    # 0x6b
+'[?]',    # 0x6c
+'[?]',    # 0x6d
+'[?]',    # 0x6e
+'[?]',    # 0x6f
+'[?]',    # 0x70
+'aa',    # 0x71
+'i',    # 0x72
+'ii',    # 0x73
+'u',    # 0x74
+'uu',    # 0x75
+'R',    # 0x76
+'RR',    # 0x77
+'L',    # 0x78
+'LL',    # 0x79
+'e',    # 0x7a
+'ee',    # 0x7b
+'o',    # 0x7c
+'oo',    # 0x7d
+'M',    # 0x7e
+'H',    # 0x7f
+'i',    # 0x80
+'ii',    # 0x81
+'',    # 0x82
+'',    # 0x83
+'',    # 0x84
+'',    # 0x85
+'',    # 0x86
+'',    # 0x87
+'',    # 0x88
+'',    # 0x89
+'',    # 0x8a
+'',    # 0x8b
+'[?]',    # 0x8c
+'[?]',    # 0x8d
+'[?]',    # 0x8e
+'[?]',    # 0x8f
+'k',    # 0x90
+'kh',    # 0x91
+'g',    # 0x92
+'gh',    # 0x93
+'ng',    # 0x94
+'c',    # 0x95
+'ch',    # 0x96
+'j',    # 0x97
+'[?]',    # 0x98
+'ny',    # 0x99
+'tt',    # 0x9a
+'tth',    # 0x9b
+'dd',    # 0x9c
+'ddh',    # 0x9d
+'nn',    # 0x9e
+'t',    # 0x9f
+'th',    # 0xa0
+'d',    # 0xa1
+'dh',    # 0xa2
+'n',    # 0xa3
+'p',    # 0xa4
+'ph',    # 0xa5
+'b',    # 0xa6
+'bh',    # 0xa7
+'m',    # 0xa8
+'ts',    # 0xa9
+'tsh',    # 0xaa
+'dz',    # 0xab
+'dzh',    # 0xac
+'w',    # 0xad
+'zh',    # 0xae
+'z',    # 0xaf
+'\'',    # 0xb0
+'y',    # 0xb1
+'r',    # 0xb2
+'l',    # 0xb3
+'sh',    # 0xb4
+'ss',    # 0xb5
+'s',    # 0xb6
+'h',    # 0xb7
+'a',    # 0xb8
+'kss',    # 0xb9
+'w',    # 0xba
+'y',    # 0xbb
+'r',    # 0xbc
+'[?]',    # 0xbd
+'X',    # 0xbe
+' :X: ',    # 0xbf
+' /O/ ',    # 0xc0
+' /o/ ',    # 0xc1
+' \\o\\ ',    # 0xc2
+' (O) ',    # 0xc3
+'',    # 0xc4
+'',    # 0xc5
+'',    # 0xc6
+'',    # 0xc7
+'',    # 0xc8
+'',    # 0xc9
+'',    # 0xca
+'',    # 0xcb
+'',    # 0xcc
+'[?]',    # 0xcd
+'[?]',    # 0xce
+'',    # 0xcf
+'[?]',    # 0xd0
+'[?]',    # 0xd1
+'[?]',    # 0xd2
+'[?]',    # 0xd3
+'[?]',    # 0xd4
+'[?]',    # 0xd5
+'[?]',    # 0xd6
+'[?]',    # 0xd7
+'[?]',    # 0xd8
+'[?]',    # 0xd9
+'[?]',    # 0xda
+'[?]',    # 0xdb
+'[?]',    # 0xdc
+'[?]',    # 0xdd
+'[?]',    # 0xde
+'[?]',    # 0xdf
+'[?]',    # 0xe0
+'[?]',    # 0xe1
+'[?]',    # 0xe2
+'[?]',    # 0xe3
+'[?]',    # 0xe4
+'[?]',    # 0xe5
+'[?]',    # 0xe6
+'[?]',    # 0xe7
+'[?]',    # 0xe8
+'[?]',    # 0xe9
+'[?]',    # 0xea
+'[?]',    # 0xeb
+'[?]',    # 0xec
+'[?]',    # 0xed
+'[?]',    # 0xee
+'[?]',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'[?]',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x010.py

@@ -0,0 +1,257 @@
+data = (
+'k',    # 0x00
+'kh',    # 0x01
+'g',    # 0x02
+'gh',    # 0x03
+'ng',    # 0x04
+'c',    # 0x05
+'ch',    # 0x06
+'j',    # 0x07
+'jh',    # 0x08
+'ny',    # 0x09
+'nny',    # 0x0a
+'tt',    # 0x0b
+'tth',    # 0x0c
+'dd',    # 0x0d
+'ddh',    # 0x0e
+'nn',    # 0x0f
+'tt',    # 0x10
+'th',    # 0x11
+'d',    # 0x12
+'dh',    # 0x13
+'n',    # 0x14
+'p',    # 0x15
+'ph',    # 0x16
+'b',    # 0x17
+'bh',    # 0x18
+'m',    # 0x19
+'y',    # 0x1a
+'r',    # 0x1b
+'l',    # 0x1c
+'w',    # 0x1d
+'s',    # 0x1e
+'h',    # 0x1f
+'ll',    # 0x20
+'a',    # 0x21
+'[?]',    # 0x22
+'i',    # 0x23
+'ii',    # 0x24
+'u',    # 0x25
+'uu',    # 0x26
+'e',    # 0x27
+'[?]',    # 0x28
+'o',    # 0x29
+'au',    # 0x2a
+'[?]',    # 0x2b
+'aa',    # 0x2c
+'i',    # 0x2d
+'ii',    # 0x2e
+'u',    # 0x2f
+'uu',    # 0x30
+'e',    # 0x31
+'ai',    # 0x32
+'[?]',    # 0x33
+'[?]',    # 0x34
+'[?]',    # 0x35
+'N',    # 0x36
+'\'',    # 0x37
+':',    # 0x38
+'',    # 0x39
+'[?]',    # 0x3a
+'[?]',    # 0x3b
+'[?]',    # 0x3c
+'[?]',    # 0x3d
+'[?]',    # 0x3e
+'[?]',    # 0x3f
+'0',    # 0x40
+'1',    # 0x41
+'2',    # 0x42
+'3',    # 0x43
+'4',    # 0x44
+'5',    # 0x45
+'6',    # 0x46
+'7',    # 0x47
+'8',    # 0x48
+'9',    # 0x49
+' / ',    # 0x4a
+' // ',    # 0x4b
+'n*',    # 0x4c
+'r*',    # 0x4d
+'l*',    # 0x4e
+'e*',    # 0x4f
+'sh',    # 0x50
+'ss',    # 0x51
+'R',    # 0x52
+'RR',    # 0x53
+'L',    # 0x54
+'LL',    # 0x55
+'R',    # 0x56
+'RR',    # 0x57
+'L',    # 0x58
+'LL',    # 0x59
+'[?]',    # 0x5a
+'[?]',    # 0x5b
+'[?]',    # 0x5c
+'[?]',    # 0x5d
+'[?]',    # 0x5e
+'[?]',    # 0x5f
+'[?]',    # 0x60
+'[?]',    # 0x61
+'[?]',    # 0x62
+'[?]',    # 0x63
+'[?]',    # 0x64
+'[?]',    # 0x65
+'[?]',    # 0x66
+'[?]',    # 0x67
+'[?]',    # 0x68
+'[?]',    # 0x69
+'[?]',    # 0x6a
+'[?]',    # 0x6b
+'[?]',    # 0x6c
+'[?]',    # 0x6d
+'[?]',    # 0x6e
+'[?]',    # 0x6f
+'[?]',    # 0x70
+'[?]',    # 0x71
+'[?]',    # 0x72
+'[?]',    # 0x73
+'[?]',    # 0x74
+'[?]',    # 0x75
+'[?]',    # 0x76
+'[?]',    # 0x77
+'[?]',    # 0x78
+'[?]',    # 0x79
+'[?]',    # 0x7a
+'[?]',    # 0x7b
+'[?]',    # 0x7c
+'[?]',    # 0x7d
+'[?]',    # 0x7e
+'[?]',    # 0x7f
+'[?]',    # 0x80
+'[?]',    # 0x81
+'[?]',    # 0x82
+'[?]',    # 0x83
+'[?]',    # 0x84
+'[?]',    # 0x85
+'[?]',    # 0x86
+'[?]',    # 0x87
+'[?]',    # 0x88
+'[?]',    # 0x89
+'[?]',    # 0x8a
+'[?]',    # 0x8b
+'[?]',    # 0x8c
+'[?]',    # 0x8d
+'[?]',    # 0x8e
+'[?]',    # 0x8f
+'[?]',    # 0x90
+'[?]',    # 0x91
+'[?]',    # 0x92
+'[?]',    # 0x93
+'[?]',    # 0x94
+'[?]',    # 0x95
+'[?]',    # 0x96
+'[?]',    # 0x97
+'[?]',    # 0x98
+'[?]',    # 0x99
+'[?]',    # 0x9a
+'[?]',    # 0x9b
+'[?]',    # 0x9c
+'[?]',    # 0x9d
+'[?]',    # 0x9e
+'[?]',    # 0x9f
+'A',    # 0xa0
+'B',    # 0xa1
+'G',    # 0xa2
+'D',    # 0xa3
+'E',    # 0xa4
+'V',    # 0xa5
+'Z',    # 0xa6
+'T`',    # 0xa7
+'I',    # 0xa8
+'K',    # 0xa9
+'L',    # 0xaa
+'M',    # 0xab
+'N',    # 0xac
+'O',    # 0xad
+'P',    # 0xae
+'Zh',    # 0xaf
+'R',    # 0xb0
+'S',    # 0xb1
+'T',    # 0xb2
+'U',    # 0xb3
+'P`',    # 0xb4
+'K`',    # 0xb5
+'G\'',    # 0xb6
+'Q',    # 0xb7
+'Sh',    # 0xb8
+'Ch`',    # 0xb9
+'C`',    # 0xba
+'Z\'',    # 0xbb
+'C',    # 0xbc
+'Ch',    # 0xbd
+'X',    # 0xbe
+'J',    # 0xbf
+'H',    # 0xc0
+'E',    # 0xc1
+'Y',    # 0xc2
+'W',    # 0xc3
+'Xh',    # 0xc4
+'OE',    # 0xc5
+'[?]',    # 0xc6
+'[?]',    # 0xc7
+'[?]',    # 0xc8
+'[?]',    # 0xc9
+'[?]',    # 0xca
+'[?]',    # 0xcb
+'[?]',    # 0xcc
+'[?]',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'a',    # 0xd0
+'b',    # 0xd1
+'g',    # 0xd2
+'d',    # 0xd3
+'e',    # 0xd4
+'v',    # 0xd5
+'z',    # 0xd6
+'t`',    # 0xd7
+'i',    # 0xd8
+'k',    # 0xd9
+'l',    # 0xda
+'m',    # 0xdb
+'n',    # 0xdc
+'o',    # 0xdd
+'p',    # 0xde
+'zh',    # 0xdf
+'r',    # 0xe0
+'s',    # 0xe1
+'t',    # 0xe2
+'u',    # 0xe3
+'p`',    # 0xe4
+'k`',    # 0xe5
+'g\'',    # 0xe6
+'q',    # 0xe7
+'sh',    # 0xe8
+'ch`',    # 0xe9
+'c`',    # 0xea
+'z\'',    # 0xeb
+'c',    # 0xec
+'ch',    # 0xed
+'x',    # 0xee
+'j',    # 0xef
+'h',    # 0xf0
+'e',    # 0xf1
+'y',    # 0xf2
+'w',    # 0xf3
+'xh',    # 0xf4
+'oe',    # 0xf5
+'f',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+' // ',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x011.py

@@ -0,0 +1,257 @@
+data = (
+'g',    # 0x00
+'gg',    # 0x01
+'n',    # 0x02
+'d',    # 0x03
+'dd',    # 0x04
+'r',    # 0x05
+'m',    # 0x06
+'b',    # 0x07
+'bb',    # 0x08
+'s',    # 0x09
+'ss',    # 0x0a
+'',    # 0x0b
+'j',    # 0x0c
+'jj',    # 0x0d
+'c',    # 0x0e
+'k',    # 0x0f
+'t',    # 0x10
+'p',    # 0x11
+'h',    # 0x12
+'ng',    # 0x13
+'nn',    # 0x14
+'nd',    # 0x15
+'nb',    # 0x16
+'dg',    # 0x17
+'rn',    # 0x18
+'rr',    # 0x19
+'rh',    # 0x1a
+'rN',    # 0x1b
+'mb',    # 0x1c
+'mN',    # 0x1d
+'bg',    # 0x1e
+'bn',    # 0x1f
+'',    # 0x20
+'bs',    # 0x21
+'bsg',    # 0x22
+'bst',    # 0x23
+'bsb',    # 0x24
+'bss',    # 0x25
+'bsj',    # 0x26
+'bj',    # 0x27
+'bc',    # 0x28
+'bt',    # 0x29
+'bp',    # 0x2a
+'bN',    # 0x2b
+'bbN',    # 0x2c
+'sg',    # 0x2d
+'sn',    # 0x2e
+'sd',    # 0x2f
+'sr',    # 0x30
+'sm',    # 0x31
+'sb',    # 0x32
+'sbg',    # 0x33
+'sss',    # 0x34
+'s',    # 0x35
+'sj',    # 0x36
+'sc',    # 0x37
+'sk',    # 0x38
+'st',    # 0x39
+'sp',    # 0x3a
+'sh',    # 0x3b
+'',    # 0x3c
+'',    # 0x3d
+'',    # 0x3e
+'',    # 0x3f
+'Z',    # 0x40
+'g',    # 0x41
+'d',    # 0x42
+'m',    # 0x43
+'b',    # 0x44
+'s',    # 0x45
+'Z',    # 0x46
+'',    # 0x47
+'j',    # 0x48
+'c',    # 0x49
+'t',    # 0x4a
+'p',    # 0x4b
+'N',    # 0x4c
+'j',    # 0x4d
+'',    # 0x4e
+'',    # 0x4f
+'',    # 0x50
+'',    # 0x51
+'ck',    # 0x52
+'ch',    # 0x53
+'',    # 0x54
+'',    # 0x55
+'pb',    # 0x56
+'pN',    # 0x57
+'hh',    # 0x58
+'Q',    # 0x59
+'[?]',    # 0x5a
+'[?]',    # 0x5b
+'[?]',    # 0x5c
+'[?]',    # 0x5d
+'[?]',    # 0x5e
+'',    # 0x5f
+'',    # 0x60
+'a',    # 0x61
+'ae',    # 0x62
+'ya',    # 0x63
+'yae',    # 0x64
+'eo',    # 0x65
+'e',    # 0x66
+'yeo',    # 0x67
+'ye',    # 0x68
+'o',    # 0x69
+'wa',    # 0x6a
+'wae',    # 0x6b
+'oe',    # 0x6c
+'yo',    # 0x6d
+'u',    # 0x6e
+'weo',    # 0x6f
+'we',    # 0x70
+'wi',    # 0x71
+'yu',    # 0x72
+'eu',    # 0x73
+'yi',    # 0x74
+'i',    # 0x75
+'a-o',    # 0x76
+'a-u',    # 0x77
+'ya-o',    # 0x78
+'ya-yo',    # 0x79
+'eo-o',    # 0x7a
+'eo-u',    # 0x7b
+'eo-eu',    # 0x7c
+'yeo-o',    # 0x7d
+'yeo-u',    # 0x7e
+'o-eo',    # 0x7f
+'o-e',    # 0x80
+'o-ye',    # 0x81
+'o-o',    # 0x82
+'o-u',    # 0x83
+'yo-ya',    # 0x84
+'yo-yae',    # 0x85
+'yo-yeo',    # 0x86
+'yo-o',    # 0x87
+'yo-i',    # 0x88
+'u-a',    # 0x89
+'u-ae',    # 0x8a
+'u-eo-eu',    # 0x8b
+'u-ye',    # 0x8c
+'u-u',    # 0x8d
+'yu-a',    # 0x8e
+'yu-eo',    # 0x8f
+'yu-e',    # 0x90
+'yu-yeo',    # 0x91
+'yu-ye',    # 0x92
+'yu-u',    # 0x93
+'yu-i',    # 0x94
+'eu-u',    # 0x95
+'eu-eu',    # 0x96
+'yi-u',    # 0x97
+'i-a',    # 0x98
+'i-ya',    # 0x99
+'i-o',    # 0x9a
+'i-u',    # 0x9b
+'i-eu',    # 0x9c
+'i-U',    # 0x9d
+'U',    # 0x9e
+'U-eo',    # 0x9f
+'U-u',    # 0xa0
+'U-i',    # 0xa1
+'UU',    # 0xa2
+'[?]',    # 0xa3
+'[?]',    # 0xa4
+'[?]',    # 0xa5
+'[?]',    # 0xa6
+'[?]',    # 0xa7
+'g',    # 0xa8
+'gg',    # 0xa9
+'gs',    # 0xaa
+'n',    # 0xab
+'nj',    # 0xac
+'nh',    # 0xad
+'d',    # 0xae
+'l',    # 0xaf
+'lg',    # 0xb0
+'lm',    # 0xb1
+'lb',    # 0xb2
+'ls',    # 0xb3
+'lt',    # 0xb4
+'lp',    # 0xb5
+'lh',    # 0xb6
+'m',    # 0xb7
+'b',    # 0xb8
+'bs',    # 0xb9
+'s',    # 0xba
+'ss',    # 0xbb
+'ng',    # 0xbc
+'j',    # 0xbd
+'c',    # 0xbe
+'k',    # 0xbf
+'t',    # 0xc0
+'p',    # 0xc1
+'h',    # 0xc2
+'gl',    # 0xc3
+'gsg',    # 0xc4
+'ng',    # 0xc5
+'nd',    # 0xc6
+'ns',    # 0xc7
+'nZ',    # 0xc8
+'nt',    # 0xc9
+'dg',    # 0xca
+'tl',    # 0xcb
+'lgs',    # 0xcc
+'ln',    # 0xcd
+'ld',    # 0xce
+'lth',    # 0xcf
+'ll',    # 0xd0
+'lmg',    # 0xd1
+'lms',    # 0xd2
+'lbs',    # 0xd3
+'lbh',    # 0xd4
+'rNp',    # 0xd5
+'lss',    # 0xd6
+'lZ',    # 0xd7
+'lk',    # 0xd8
+'lQ',    # 0xd9
+'mg',    # 0xda
+'ml',    # 0xdb
+'mb',    # 0xdc
+'ms',    # 0xdd
+'mss',    # 0xde
+'mZ',    # 0xdf
+'mc',    # 0xe0
+'mh',    # 0xe1
+'mN',    # 0xe2
+'bl',    # 0xe3
+'bp',    # 0xe4
+'ph',    # 0xe5
+'pN',    # 0xe6
+'sg',    # 0xe7
+'sd',    # 0xe8
+'sl',    # 0xe9
+'sb',    # 0xea
+'Z',    # 0xeb
+'g',    # 0xec
+'ss',    # 0xed
+'',    # 0xee
+'kh',    # 0xef
+'N',    # 0xf0
+'Ns',    # 0xf1
+'NZ',    # 0xf2
+'pb',    # 0xf3
+'pN',    # 0xf4
+'hn',    # 0xf5
+'hl',    # 0xf6
+'hm',    # 0xf7
+'hb',    # 0xf8
+'Q',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x012.py

@@ -0,0 +1,258 @@
+data = (
+'ha',    # 0x00
+'hu',    # 0x01
+'hi',    # 0x02
+'haa',    # 0x03
+'hee',    # 0x04
+'he',    # 0x05
+'ho',    # 0x06
+'[?]',    # 0x07
+'la',    # 0x08
+'lu',    # 0x09
+'li',    # 0x0a
+'laa',    # 0x0b
+'lee',    # 0x0c
+'le',    # 0x0d
+'lo',    # 0x0e
+'lwa',    # 0x0f
+'hha',    # 0x10
+'hhu',    # 0x11
+'hhi',    # 0x12
+'hhaa',    # 0x13
+'hhee',    # 0x14
+'hhe',    # 0x15
+'hho',    # 0x16
+'hhwa',    # 0x17
+'ma',    # 0x18
+'mu',    # 0x19
+'mi',    # 0x1a
+'maa',    # 0x1b
+'mee',    # 0x1c
+'me',    # 0x1d
+'mo',    # 0x1e
+'mwa',    # 0x1f
+'sza',    # 0x20
+'szu',    # 0x21
+'szi',    # 0x22
+'szaa',    # 0x23
+'szee',    # 0x24
+'sze',    # 0x25
+'szo',    # 0x26
+'szwa',    # 0x27
+'ra',    # 0x28
+'ru',    # 0x29
+'ri',    # 0x2a
+'raa',    # 0x2b
+'ree',    # 0x2c
+'re',    # 0x2d
+'ro',    # 0x2e
+'rwa',    # 0x2f
+'sa',    # 0x30
+'su',    # 0x31
+'si',    # 0x32
+'saa',    # 0x33
+'see',    # 0x34
+'se',    # 0x35
+'so',    # 0x36
+'swa',    # 0x37
+'sha',    # 0x38
+'shu',    # 0x39
+'shi',    # 0x3a
+'shaa',    # 0x3b
+'shee',    # 0x3c
+'she',    # 0x3d
+'sho',    # 0x3e
+'shwa',    # 0x3f
+'qa',    # 0x40
+'qu',    # 0x41
+'qi',    # 0x42
+'qaa',    # 0x43
+'qee',    # 0x44
+'qe',    # 0x45
+'qo',    # 0x46
+'[?]',    # 0x47
+'qwa',    # 0x48
+'[?]',    # 0x49
+'qwi',    # 0x4a
+'qwaa',    # 0x4b
+'qwee',    # 0x4c
+'qwe',    # 0x4d
+'[?]',    # 0x4e
+'[?]',    # 0x4f
+'qha',    # 0x50
+'qhu',    # 0x51
+'qhi',    # 0x52
+'qhaa',    # 0x53
+'qhee',    # 0x54
+'qhe',    # 0x55
+'qho',    # 0x56
+'[?]',    # 0x57
+'qhwa',    # 0x58
+'[?]',    # 0x59
+'qhwi',    # 0x5a
+'qhwaa',    # 0x5b
+'qhwee',    # 0x5c
+'qhwe',    # 0x5d
+'[?]',    # 0x5e
+'[?]',    # 0x5f
+'ba',    # 0x60
+'bu',    # 0x61
+'bi',    # 0x62
+'baa',    # 0x63
+'bee',    # 0x64
+'be',    # 0x65
+'bo',    # 0x66
+'bwa',    # 0x67
+'va',    # 0x68
+'vu',    # 0x69
+'vi',    # 0x6a
+'vaa',    # 0x6b
+'vee',    # 0x6c
+'ve',    # 0x6d
+'vo',    # 0x6e
+'vwa',    # 0x6f
+'ta',    # 0x70
+'tu',    # 0x71
+'ti',    # 0x72
+'taa',    # 0x73
+'tee',    # 0x74
+'te',    # 0x75
+'to',    # 0x76
+'twa',    # 0x77
+'ca',    # 0x78
+'cu',    # 0x79
+'ci',    # 0x7a
+'caa',    # 0x7b
+'cee',    # 0x7c
+'ce',    # 0x7d
+'co',    # 0x7e
+'cwa',    # 0x7f
+'xa',    # 0x80
+'xu',    # 0x81
+'xi',    # 0x82
+'xaa',    # 0x83
+'xee',    # 0x84
+'xe',    # 0x85
+'xo',    # 0x86
+'[?]',    # 0x87
+'xwa',    # 0x88
+'[?]',    # 0x89
+'xwi',    # 0x8a
+'xwaa',    # 0x8b
+'xwee',    # 0x8c
+'xwe',    # 0x8d
+'[?]',    # 0x8e
+'[?]',    # 0x8f
+'na',    # 0x90
+'nu',    # 0x91
+'ni',    # 0x92
+'naa',    # 0x93
+'nee',    # 0x94
+'ne',    # 0x95
+'no',    # 0x96
+'nwa',    # 0x97
+'nya',    # 0x98
+'nyu',    # 0x99
+'nyi',    # 0x9a
+'nyaa',    # 0x9b
+'nyee',    # 0x9c
+'nye',    # 0x9d
+'nyo',    # 0x9e
+'nywa',    # 0x9f
+'\'a',    # 0xa0
+'\'u',    # 0xa1
+'[?]',    # 0xa2
+'\'aa',    # 0xa3
+'\'ee',    # 0xa4
+'\'e',    # 0xa5
+'\'o',    # 0xa6
+'\'wa',    # 0xa7
+'ka',    # 0xa8
+'ku',    # 0xa9
+'ki',    # 0xaa
+'kaa',    # 0xab
+'kee',    # 0xac
+'ke',    # 0xad
+'ko',    # 0xae
+'[?]',    # 0xaf
+'kwa',    # 0xb0
+'[?]',    # 0xb1
+'kwi',    # 0xb2
+'kwaa',    # 0xb3
+'kwee',    # 0xb4
+'kwe',    # 0xb5
+'[?]',    # 0xb6
+'[?]',    # 0xb7
+'kxa',    # 0xb8
+'kxu',    # 0xb9
+'kxi',    # 0xba
+'kxaa',    # 0xbb
+'kxee',    # 0xbc
+'kxe',    # 0xbd
+'kxo',    # 0xbe
+'[?]',    # 0xbf
+'kxwa',    # 0xc0
+'[?]',    # 0xc1
+'kxwi',    # 0xc2
+'kxwaa',    # 0xc3
+'kxwee',    # 0xc4
+'kxwe',    # 0xc5
+'[?]',    # 0xc6
+'[?]',    # 0xc7
+'wa',    # 0xc8
+'wu',    # 0xc9
+'wi',    # 0xca
+'waa',    # 0xcb
+'wee',    # 0xcc
+'we',    # 0xcd
+'wo',    # 0xce
+'[?]',    # 0xcf
+'`a',    # 0xd0
+'`u',    # 0xd1
+'`i',    # 0xd2
+'`aa',    # 0xd3
+'`ee',    # 0xd4
+'`e',    # 0xd5
+'`o',    # 0xd6
+'[?]',    # 0xd7
+'za',    # 0xd8
+'zu',    # 0xd9
+'zi',    # 0xda
+'zaa',    # 0xdb
+'zee',    # 0xdc
+'ze',    # 0xdd
+'zo',    # 0xde
+'zwa',    # 0xdf
+'zha',    # 0xe0
+'zhu',    # 0xe1
+'zhi',    # 0xe2
+'zhaa',    # 0xe3
+'zhee',    # 0xe4
+'zhe',    # 0xe5
+'zho',    # 0xe6
+'zhwa',    # 0xe7
+'ya',    # 0xe8
+'yu',    # 0xe9
+'yi',    # 0xea
+'yaa',    # 0xeb
+'yee',    # 0xec
+'ye',    # 0xed
+'yo',    # 0xee
+'[?]',    # 0xef
+'da',    # 0xf0
+'du',    # 0xf1
+'di',    # 0xf2
+'daa',    # 0xf3
+'dee',    # 0xf4
+'de',    # 0xf5
+'do',    # 0xf6
+'dwa',    # 0xf7
+'dda',    # 0xf8
+'ddu',    # 0xf9
+'ddi',    # 0xfa
+'ddaa',    # 0xfb
+'ddee',    # 0xfc
+'dde',    # 0xfd
+'ddo',    # 0xfe
+'ddwa',    # 0xff
+)

lib/unidecode/x013.py

@@ -0,0 +1,257 @@
+data = (
+'ja',    # 0x00
+'ju',    # 0x01
+'ji',    # 0x02
+'jaa',    # 0x03
+'jee',    # 0x04
+'je',    # 0x05
+'jo',    # 0x06
+'jwa',    # 0x07
+'ga',    # 0x08
+'gu',    # 0x09
+'gi',    # 0x0a
+'gaa',    # 0x0b
+'gee',    # 0x0c
+'ge',    # 0x0d
+'go',    # 0x0e
+'[?]',    # 0x0f
+'gwa',    # 0x10
+'[?]',    # 0x11
+'gwi',    # 0x12
+'gwaa',    # 0x13
+'gwee',    # 0x14
+'gwe',    # 0x15
+'[?]',    # 0x16
+'[?]',    # 0x17
+'gga',    # 0x18
+'ggu',    # 0x19
+'ggi',    # 0x1a
+'ggaa',    # 0x1b
+'ggee',    # 0x1c
+'gge',    # 0x1d
+'ggo',    # 0x1e
+'[?]',    # 0x1f
+'tha',    # 0x20
+'thu',    # 0x21
+'thi',    # 0x22
+'thaa',    # 0x23
+'thee',    # 0x24
+'the',    # 0x25
+'tho',    # 0x26
+'thwa',    # 0x27
+'cha',    # 0x28
+'chu',    # 0x29
+'chi',    # 0x2a
+'chaa',    # 0x2b
+'chee',    # 0x2c
+'che',    # 0x2d
+'cho',    # 0x2e
+'chwa',    # 0x2f
+'pha',    # 0x30
+'phu',    # 0x31
+'phi',    # 0x32
+'phaa',    # 0x33
+'phee',    # 0x34
+'phe',    # 0x35
+'pho',    # 0x36
+'phwa',    # 0x37
+'tsa',    # 0x38
+'tsu',    # 0x39
+'tsi',    # 0x3a
+'tsaa',    # 0x3b
+'tsee',    # 0x3c
+'tse',    # 0x3d
+'tso',    # 0x3e
+'tswa',    # 0x3f
+'tza',    # 0x40
+'tzu',    # 0x41
+'tzi',    # 0x42
+'tzaa',    # 0x43
+'tzee',    # 0x44
+'tze',    # 0x45
+'tzo',    # 0x46
+'[?]',    # 0x47
+'fa',    # 0x48
+'fu',    # 0x49
+'fi',    # 0x4a
+'faa',    # 0x4b
+'fee',    # 0x4c
+'fe',    # 0x4d
+'fo',    # 0x4e
+'fwa',    # 0x4f
+'pa',    # 0x50
+'pu',    # 0x51
+'pi',    # 0x52
+'paa',    # 0x53
+'pee',    # 0x54
+'pe',    # 0x55
+'po',    # 0x56
+'pwa',    # 0x57
+'rya',    # 0x58
+'mya',    # 0x59
+'fya',    # 0x5a
+'[?]',    # 0x5b
+'[?]',    # 0x5c
+'[?]',    # 0x5d
+'[?]',    # 0x5e
+'[?]',    # 0x5f
+'[?]',    # 0x60
+' ',    # 0x61
+'.',    # 0x62
+',',    # 0x63
+';',    # 0x64
+':',    # 0x65
+':: ',    # 0x66
+'?',    # 0x67
+'//',    # 0x68
+'1',    # 0x69
+'2',    # 0x6a
+'3',    # 0x6b
+'4',    # 0x6c
+'5',    # 0x6d
+'6',    # 0x6e
+'7',    # 0x6f
+'8',    # 0x70
+'9',    # 0x71
+'10+',    # 0x72
+'20+',    # 0x73
+'30+',    # 0x74
+'40+',    # 0x75
+'50+',    # 0x76
+'60+',    # 0x77
+'70+',    # 0x78
+'80+',    # 0x79
+'90+',    # 0x7a
+'100+',    # 0x7b
+'10,000+',    # 0x7c
+'[?]',    # 0x7d
+'[?]',    # 0x7e
+'[?]',    # 0x7f
+'[?]',    # 0x80
+'[?]',    # 0x81
+'[?]',    # 0x82
+'[?]',    # 0x83
+'[?]',    # 0x84
+'[?]',    # 0x85
+'[?]',    # 0x86
+'[?]',    # 0x87
+'[?]',    # 0x88
+'[?]',    # 0x89
+'[?]',    # 0x8a
+'[?]',    # 0x8b
+'[?]',    # 0x8c
+'[?]',    # 0x8d
+'[?]',    # 0x8e
+'[?]',    # 0x8f
+'[?]',    # 0x90
+'[?]',    # 0x91
+'[?]',    # 0x92
+'[?]',    # 0x93
+'[?]',    # 0x94
+'[?]',    # 0x95
+'[?]',    # 0x96
+'[?]',    # 0x97
+'[?]',    # 0x98
+'[?]',    # 0x99
+'[?]',    # 0x9a
+'[?]',    # 0x9b
+'[?]',    # 0x9c
+'[?]',    # 0x9d
+'[?]',    # 0x9e
+'[?]',    # 0x9f
+'a',    # 0xa0
+'e',    # 0xa1
+'i',    # 0xa2
+'o',    # 0xa3
+'u',    # 0xa4
+'v',    # 0xa5
+'ga',    # 0xa6
+'ka',    # 0xa7
+'ge',    # 0xa8
+'gi',    # 0xa9
+'go',    # 0xaa
+'gu',    # 0xab
+'gv',    # 0xac
+'ha',    # 0xad
+'he',    # 0xae
+'hi',    # 0xaf
+'ho',    # 0xb0
+'hu',    # 0xb1
+'hv',    # 0xb2
+'la',    # 0xb3
+'le',    # 0xb4
+'li',    # 0xb5
+'lo',    # 0xb6
+'lu',    # 0xb7
+'lv',    # 0xb8
+'ma',    # 0xb9
+'me',    # 0xba
+'mi',    # 0xbb
+'mo',    # 0xbc
+'mu',    # 0xbd
+'na',    # 0xbe
+'hna',    # 0xbf
+'nah',    # 0xc0
+'ne',    # 0xc1
+'ni',    # 0xc2
+'no',    # 0xc3
+'nu',    # 0xc4
+'nv',    # 0xc5
+'qua',    # 0xc6
+'que',    # 0xc7
+'qui',    # 0xc8
+'quo',    # 0xc9
+'quu',    # 0xca
+'quv',    # 0xcb
+'sa',    # 0xcc
+'s',    # 0xcd
+'se',    # 0xce
+'si',    # 0xcf
+'so',    # 0xd0
+'su',    # 0xd1
+'sv',    # 0xd2
+'da',    # 0xd3
+'ta',    # 0xd4
+'de',    # 0xd5
+'te',    # 0xd6
+'di',    # 0xd7
+'ti',    # 0xd8
+'do',    # 0xd9
+'du',    # 0xda
+'dv',    # 0xdb
+'dla',    # 0xdc
+'tla',    # 0xdd
+'tle',    # 0xde
+'tli',    # 0xdf
+'tlo',    # 0xe0
+'tlu',    # 0xe1
+'tlv',    # 0xe2
+'tsa',    # 0xe3
+'tse',    # 0xe4
+'tsi',    # 0xe5
+'tso',    # 0xe6
+'tsu',    # 0xe7
+'tsv',    # 0xe8
+'wa',    # 0xe9
+'we',    # 0xea
+'wi',    # 0xeb
+'wo',    # 0xec
+'wu',    # 0xed
+'wv',    # 0xee
+'ya',    # 0xef
+'ye',    # 0xf0
+'yi',    # 0xf1
+'yo',    # 0xf2
+'yu',    # 0xf3
+'yv',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x014.py

@@ -0,0 +1,258 @@
+data = (
+'[?]',    # 0x00
+'e',    # 0x01
+'aai',    # 0x02
+'i',    # 0x03
+'ii',    # 0x04
+'o',    # 0x05
+'oo',    # 0x06
+'oo',    # 0x07
+'ee',    # 0x08
+'i',    # 0x09
+'a',    # 0x0a
+'aa',    # 0x0b
+'we',    # 0x0c
+'we',    # 0x0d
+'wi',    # 0x0e
+'wi',    # 0x0f
+'wii',    # 0x10
+'wii',    # 0x11
+'wo',    # 0x12
+'wo',    # 0x13
+'woo',    # 0x14
+'woo',    # 0x15
+'woo',    # 0x16
+'wa',    # 0x17
+'wa',    # 0x18
+'waa',    # 0x19
+'waa',    # 0x1a
+'waa',    # 0x1b
+'ai',    # 0x1c
+'w',    # 0x1d
+'\'',    # 0x1e
+'t',    # 0x1f
+'k',    # 0x20
+'sh',    # 0x21
+'s',    # 0x22
+'n',    # 0x23
+'w',    # 0x24
+'n',    # 0x25
+'[?]',    # 0x26
+'w',    # 0x27
+'c',    # 0x28
+'?',    # 0x29
+'l',    # 0x2a
+'en',    # 0x2b
+'in',    # 0x2c
+'on',    # 0x2d
+'an',    # 0x2e
+'pe',    # 0x2f
+'paai',    # 0x30
+'pi',    # 0x31
+'pii',    # 0x32
+'po',    # 0x33
+'poo',    # 0x34
+'poo',    # 0x35
+'hee',    # 0x36
+'hi',    # 0x37
+'pa',    # 0x38
+'paa',    # 0x39
+'pwe',    # 0x3a
+'pwe',    # 0x3b
+'pwi',    # 0x3c
+'pwi',    # 0x3d
+'pwii',    # 0x3e
+'pwii',    # 0x3f
+'pwo',    # 0x40
+'pwo',    # 0x41
+'pwoo',    # 0x42
+'pwoo',    # 0x43
+'pwa',    # 0x44
+'pwa',    # 0x45
+'pwaa',    # 0x46
+'pwaa',    # 0x47
+'pwaa',    # 0x48
+'p',    # 0x49
+'p',    # 0x4a
+'h',    # 0x4b
+'te',    # 0x4c
+'taai',    # 0x4d
+'ti',    # 0x4e
+'tii',    # 0x4f
+'to',    # 0x50
+'too',    # 0x51
+'too',    # 0x52
+'dee',    # 0x53
+'di',    # 0x54
+'ta',    # 0x55
+'taa',    # 0x56
+'twe',    # 0x57
+'twe',    # 0x58
+'twi',    # 0x59
+'twi',    # 0x5a
+'twii',    # 0x5b
+'twii',    # 0x5c
+'two',    # 0x5d
+'two',    # 0x5e
+'twoo',    # 0x5f
+'twoo',    # 0x60
+'twa',    # 0x61
+'twa',    # 0x62
+'twaa',    # 0x63
+'twaa',    # 0x64
+'twaa',    # 0x65
+'t',    # 0x66
+'tte',    # 0x67
+'tti',    # 0x68
+'tto',    # 0x69
+'tta',    # 0x6a
+'ke',    # 0x6b
+'kaai',    # 0x6c
+'ki',    # 0x6d
+'kii',    # 0x6e
+'ko',    # 0x6f
+'koo',    # 0x70
+'koo',    # 0x71
+'ka',    # 0x72
+'kaa',    # 0x73
+'kwe',    # 0x74
+'kwe',    # 0x75
+'kwi',    # 0x76
+'kwi',    # 0x77
+'kwii',    # 0x78
+'kwii',    # 0x79
+'kwo',    # 0x7a
+'kwo',    # 0x7b
+'kwoo',    # 0x7c
+'kwoo',    # 0x7d
+'kwa',    # 0x7e
+'kwa',    # 0x7f
+'kwaa',    # 0x80
+'kwaa',    # 0x81
+'kwaa',    # 0x82
+'k',    # 0x83
+'kw',    # 0x84
+'keh',    # 0x85
+'kih',    # 0x86
+'koh',    # 0x87
+'kah',    # 0x88
+'ce',    # 0x89
+'caai',    # 0x8a
+'ci',    # 0x8b
+'cii',    # 0x8c
+'co',    # 0x8d
+'coo',    # 0x8e
+'coo',    # 0x8f
+'ca',    # 0x90
+'caa',    # 0x91
+'cwe',    # 0x92
+'cwe',    # 0x93
+'cwi',    # 0x94
+'cwi',    # 0x95
+'cwii',    # 0x96
+'cwii',    # 0x97
+'cwo',    # 0x98
+'cwo',    # 0x99
+'cwoo',    # 0x9a
+'cwoo',    # 0x9b
+'cwa',    # 0x9c
+'cwa',    # 0x9d
+'cwaa',    # 0x9e
+'cwaa',    # 0x9f
+'cwaa',    # 0xa0
+'c',    # 0xa1
+'th',    # 0xa2
+'me',    # 0xa3
+'maai',    # 0xa4
+'mi',    # 0xa5
+'mii',    # 0xa6
+'mo',    # 0xa7
+'moo',    # 0xa8
+'moo',    # 0xa9
+'ma',    # 0xaa
+'maa',    # 0xab
+'mwe',    # 0xac
+'mwe',    # 0xad
+'mwi',    # 0xae
+'mwi',    # 0xaf
+'mwii',    # 0xb0
+'mwii',    # 0xb1
+'mwo',    # 0xb2
+'mwo',    # 0xb3
+'mwoo',    # 0xb4
+'mwoo',    # 0xb5
+'mwa',    # 0xb6
+'mwa',    # 0xb7
+'mwaa',    # 0xb8
+'mwaa',    # 0xb9
+'mwaa',    # 0xba
+'m',    # 0xbb
+'m',    # 0xbc
+'mh',    # 0xbd
+'m',    # 0xbe
+'m',    # 0xbf
+'ne',    # 0xc0
+'naai',    # 0xc1
+'ni',    # 0xc2
+'nii',    # 0xc3
+'no',    # 0xc4
+'noo',    # 0xc5
+'noo',    # 0xc6
+'na',    # 0xc7
+'naa',    # 0xc8
+'nwe',    # 0xc9
+'nwe',    # 0xca
+'nwa',    # 0xcb
+'nwa',    # 0xcc
+'nwaa',    # 0xcd
+'nwaa',    # 0xce
+'nwaa',    # 0xcf
+'n',    # 0xd0
+'ng',    # 0xd1
+'nh',    # 0xd2
+'le',    # 0xd3
+'laai',    # 0xd4
+'li',    # 0xd5
+'lii',    # 0xd6
+'lo',    # 0xd7
+'loo',    # 0xd8
+'loo',    # 0xd9
+'la',    # 0xda
+'laa',    # 0xdb
+'lwe',    # 0xdc
+'lwe',    # 0xdd
+'lwi',    # 0xde
+'lwi',    # 0xdf
+'lwii',    # 0xe0
+'lwii',    # 0xe1
+'lwo',    # 0xe2
+'lwo',    # 0xe3
+'lwoo',    # 0xe4
+'lwoo',    # 0xe5
+'lwa',    # 0xe6
+'lwa',    # 0xe7
+'lwaa',    # 0xe8
+'lwaa',    # 0xe9
+'l',    # 0xea
+'l',    # 0xeb
+'l',    # 0xec
+'se',    # 0xed
+'saai',    # 0xee
+'si',    # 0xef
+'sii',    # 0xf0
+'so',    # 0xf1
+'soo',    # 0xf2
+'soo',    # 0xf3
+'sa',    # 0xf4
+'saa',    # 0xf5
+'swe',    # 0xf6
+'swe',    # 0xf7
+'swi',    # 0xf8
+'swi',    # 0xf9
+'swii',    # 0xfa
+'swii',    # 0xfb
+'swo',    # 0xfc
+'swo',    # 0xfd
+'swoo',    # 0xfe
+'swoo',    # 0xff
+)

lib/unidecode/x015.py

@@ -0,0 +1,258 @@
+data = (
+'swa',    # 0x00
+'swa',    # 0x01
+'swaa',    # 0x02
+'swaa',    # 0x03
+'swaa',    # 0x04
+'s',    # 0x05
+'s',    # 0x06
+'sw',    # 0x07
+'s',    # 0x08
+'sk',    # 0x09
+'skw',    # 0x0a
+'sW',    # 0x0b
+'spwa',    # 0x0c
+'stwa',    # 0x0d
+'skwa',    # 0x0e
+'scwa',    # 0x0f
+'she',    # 0x10
+'shi',    # 0x11
+'shii',    # 0x12
+'sho',    # 0x13
+'shoo',    # 0x14
+'sha',    # 0x15
+'shaa',    # 0x16
+'shwe',    # 0x17
+'shwe',    # 0x18
+'shwi',    # 0x19
+'shwi',    # 0x1a
+'shwii',    # 0x1b
+'shwii',    # 0x1c
+'shwo',    # 0x1d
+'shwo',    # 0x1e
+'shwoo',    # 0x1f
+'shwoo',    # 0x20
+'shwa',    # 0x21
+'shwa',    # 0x22
+'shwaa',    # 0x23
+'shwaa',    # 0x24
+'sh',    # 0x25
+'ye',    # 0x26
+'yaai',    # 0x27
+'yi',    # 0x28
+'yii',    # 0x29
+'yo',    # 0x2a
+'yoo',    # 0x2b
+'yoo',    # 0x2c
+'ya',    # 0x2d
+'yaa',    # 0x2e
+'ywe',    # 0x2f
+'ywe',    # 0x30
+'ywi',    # 0x31
+'ywi',    # 0x32
+'ywii',    # 0x33
+'ywii',    # 0x34
+'ywo',    # 0x35
+'ywo',    # 0x36
+'ywoo',    # 0x37
+'ywoo',    # 0x38
+'ywa',    # 0x39
+'ywa',    # 0x3a
+'ywaa',    # 0x3b
+'ywaa',    # 0x3c
+'ywaa',    # 0x3d
+'y',    # 0x3e
+'y',    # 0x3f
+'y',    # 0x40
+'yi',    # 0x41
+'re',    # 0x42
+'re',    # 0x43
+'le',    # 0x44
+'raai',    # 0x45
+'ri',    # 0x46
+'rii',    # 0x47
+'ro',    # 0x48
+'roo',    # 0x49
+'lo',    # 0x4a
+'ra',    # 0x4b
+'raa',    # 0x4c
+'la',    # 0x4d
+'rwaa',    # 0x4e
+'rwaa',    # 0x4f
+'r',    # 0x50
+'r',    # 0x51
+'r',    # 0x52
+'fe',    # 0x53
+'faai',    # 0x54
+'fi',    # 0x55
+'fii',    # 0x56
+'fo',    # 0x57
+'foo',    # 0x58
+'fa',    # 0x59
+'faa',    # 0x5a
+'fwaa',    # 0x5b
+'fwaa',    # 0x5c
+'f',    # 0x5d
+'the',    # 0x5e
+'the',    # 0x5f
+'thi',    # 0x60
+'thi',    # 0x61
+'thii',    # 0x62
+'thii',    # 0x63
+'tho',    # 0x64
+'thoo',    # 0x65
+'tha',    # 0x66
+'thaa',    # 0x67
+'thwaa',    # 0x68
+'thwaa',    # 0x69
+'th',    # 0x6a
+'tthe',    # 0x6b
+'tthi',    # 0x6c
+'ttho',    # 0x6d
+'ttha',    # 0x6e
+'tth',    # 0x6f
+'tye',    # 0x70
+'tyi',    # 0x71
+'tyo',    # 0x72
+'tya',    # 0x73
+'he',    # 0x74
+'hi',    # 0x75
+'hii',    # 0x76
+'ho',    # 0x77
+'hoo',    # 0x78
+'ha',    # 0x79
+'haa',    # 0x7a
+'h',    # 0x7b
+'h',    # 0x7c
+'hk',    # 0x7d
+'qaai',    # 0x7e
+'qi',    # 0x7f
+'qii',    # 0x80
+'qo',    # 0x81
+'qoo',    # 0x82
+'qa',    # 0x83
+'qaa',    # 0x84
+'q',    # 0x85
+'tlhe',    # 0x86
+'tlhi',    # 0x87
+'tlho',    # 0x88
+'tlha',    # 0x89
+'re',    # 0x8a
+'ri',    # 0x8b
+'ro',    # 0x8c
+'ra',    # 0x8d
+'ngaai',    # 0x8e
+'ngi',    # 0x8f
+'ngii',    # 0x90
+'ngo',    # 0x91
+'ngoo',    # 0x92
+'nga',    # 0x93
+'ngaa',    # 0x94
+'ng',    # 0x95
+'nng',    # 0x96
+'she',    # 0x97
+'shi',    # 0x98
+'sho',    # 0x99
+'sha',    # 0x9a
+'the',    # 0x9b
+'thi',    # 0x9c
+'tho',    # 0x9d
+'tha',    # 0x9e
+'th',    # 0x9f
+'lhi',    # 0xa0
+'lhii',    # 0xa1
+'lho',    # 0xa2
+'lhoo',    # 0xa3
+'lha',    # 0xa4
+'lhaa',    # 0xa5
+'lh',    # 0xa6
+'the',    # 0xa7
+'thi',    # 0xa8
+'thii',    # 0xa9
+'tho',    # 0xaa
+'thoo',    # 0xab
+'tha',    # 0xac
+'thaa',    # 0xad
+'th',    # 0xae
+'b',    # 0xaf
+'e',    # 0xb0
+'i',    # 0xb1
+'o',    # 0xb2
+'a',    # 0xb3
+'we',    # 0xb4
+'wi',    # 0xb5
+'wo',    # 0xb6
+'wa',    # 0xb7
+'ne',    # 0xb8
+'ni',    # 0xb9
+'no',    # 0xba
+'na',    # 0xbb
+'ke',    # 0xbc
+'ki',    # 0xbd
+'ko',    # 0xbe
+'ka',    # 0xbf
+'he',    # 0xc0
+'hi',    # 0xc1
+'ho',    # 0xc2
+'ha',    # 0xc3
+'ghu',    # 0xc4
+'gho',    # 0xc5
+'ghe',    # 0xc6
+'ghee',    # 0xc7
+'ghi',    # 0xc8
+'gha',    # 0xc9
+'ru',    # 0xca
+'ro',    # 0xcb
+'re',    # 0xcc
+'ree',    # 0xcd
+'ri',    # 0xce
+'ra',    # 0xcf
+'wu',    # 0xd0
+'wo',    # 0xd1
+'we',    # 0xd2
+'wee',    # 0xd3
+'wi',    # 0xd4
+'wa',    # 0xd5
+'hwu',    # 0xd6
+'hwo',    # 0xd7
+'hwe',    # 0xd8
+'hwee',    # 0xd9
+'hwi',    # 0xda
+'hwa',    # 0xdb
+'thu',    # 0xdc
+'tho',    # 0xdd
+'the',    # 0xde
+'thee',    # 0xdf
+'thi',    # 0xe0
+'tha',    # 0xe1
+'ttu',    # 0xe2
+'tto',    # 0xe3
+'tte',    # 0xe4
+'ttee',    # 0xe5
+'tti',    # 0xe6
+'tta',    # 0xe7
+'pu',    # 0xe8
+'po',    # 0xe9
+'pe',    # 0xea
+'pee',    # 0xeb
+'pi',    # 0xec
+'pa',    # 0xed
+'p',    # 0xee
+'gu',    # 0xef
+'go',    # 0xf0
+'ge',    # 0xf1
+'gee',    # 0xf2
+'gi',    # 0xf3
+'ga',    # 0xf4
+'khu',    # 0xf5
+'kho',    # 0xf6
+'khe',    # 0xf7
+'khee',    # 0xf8
+'khi',    # 0xf9
+'kha',    # 0xfa
+'kku',    # 0xfb
+'kko',    # 0xfc
+'kke',    # 0xfd
+'kkee',    # 0xfe
+'kki',    # 0xff
+)

lib/unidecode/x016.py

@@ -0,0 +1,257 @@
+data = (
+'kka',    # 0x00
+'kk',    # 0x01
+'nu',    # 0x02
+'no',    # 0x03
+'ne',    # 0x04
+'nee',    # 0x05
+'ni',    # 0x06
+'na',    # 0x07
+'mu',    # 0x08
+'mo',    # 0x09
+'me',    # 0x0a
+'mee',    # 0x0b
+'mi',    # 0x0c
+'ma',    # 0x0d
+'yu',    # 0x0e
+'yo',    # 0x0f
+'ye',    # 0x10
+'yee',    # 0x11
+'yi',    # 0x12
+'ya',    # 0x13
+'ju',    # 0x14
+'ju',    # 0x15
+'jo',    # 0x16
+'je',    # 0x17
+'jee',    # 0x18
+'ji',    # 0x19
+'ji',    # 0x1a
+'ja',    # 0x1b
+'jju',    # 0x1c
+'jjo',    # 0x1d
+'jje',    # 0x1e
+'jjee',    # 0x1f
+'jji',    # 0x20
+'jja',    # 0x21
+'lu',    # 0x22
+'lo',    # 0x23
+'le',    # 0x24
+'lee',    # 0x25
+'li',    # 0x26
+'la',    # 0x27
+'dlu',    # 0x28
+'dlo',    # 0x29
+'dle',    # 0x2a
+'dlee',    # 0x2b
+'dli',    # 0x2c
+'dla',    # 0x2d
+'lhu',    # 0x2e
+'lho',    # 0x2f
+'lhe',    # 0x30
+'lhee',    # 0x31
+'lhi',    # 0x32
+'lha',    # 0x33
+'tlhu',    # 0x34
+'tlho',    # 0x35
+'tlhe',    # 0x36
+'tlhee',    # 0x37
+'tlhi',    # 0x38
+'tlha',    # 0x39
+'tlu',    # 0x3a
+'tlo',    # 0x3b
+'tle',    # 0x3c
+'tlee',    # 0x3d
+'tli',    # 0x3e
+'tla',    # 0x3f
+'zu',    # 0x40
+'zo',    # 0x41
+'ze',    # 0x42
+'zee',    # 0x43
+'zi',    # 0x44
+'za',    # 0x45
+'z',    # 0x46
+'z',    # 0x47
+'dzu',    # 0x48
+'dzo',    # 0x49
+'dze',    # 0x4a
+'dzee',    # 0x4b
+'dzi',    # 0x4c
+'dza',    # 0x4d
+'su',    # 0x4e
+'so',    # 0x4f
+'se',    # 0x50
+'see',    # 0x51
+'si',    # 0x52
+'sa',    # 0x53
+'shu',    # 0x54
+'sho',    # 0x55
+'she',    # 0x56
+'shee',    # 0x57
+'shi',    # 0x58
+'sha',    # 0x59
+'sh',    # 0x5a
+'tsu',    # 0x5b
+'tso',    # 0x5c
+'tse',    # 0x5d
+'tsee',    # 0x5e
+'tsi',    # 0x5f
+'tsa',    # 0x60
+'chu',    # 0x61
+'cho',    # 0x62
+'che',    # 0x63
+'chee',    # 0x64
+'chi',    # 0x65
+'cha',    # 0x66
+'ttsu',    # 0x67
+'ttso',    # 0x68
+'ttse',    # 0x69
+'ttsee',    # 0x6a
+'ttsi',    # 0x6b
+'ttsa',    # 0x6c
+'X',    # 0x6d
+'.',    # 0x6e
+'qai',    # 0x6f
+'ngai',    # 0x70
+'nngi',    # 0x71
+'nngii',    # 0x72
+'nngo',    # 0x73
+'nngoo',    # 0x74
+'nnga',    # 0x75
+'nngaa',    # 0x76
+'[?]',    # 0x77
+'[?]',    # 0x78
+'[?]',    # 0x79
+'[?]',    # 0x7a
+'[?]',    # 0x7b
+'[?]',    # 0x7c
+'[?]',    # 0x7d
+'[?]',    # 0x7e
+'[?]',    # 0x7f
+' ',    # 0x80
+'b',    # 0x81
+'l',    # 0x82
+'f',    # 0x83
+'s',    # 0x84
+'n',    # 0x85
+'h',    # 0x86
+'d',    # 0x87
+'t',    # 0x88
+'c',    # 0x89
+'q',    # 0x8a
+'m',    # 0x8b
+'g',    # 0x8c
+'ng',    # 0x8d
+'z',    # 0x8e
+'r',    # 0x8f
+'a',    # 0x90
+'o',    # 0x91
+'u',    # 0x92
+'e',    # 0x93
+'i',    # 0x94
+'ch',    # 0x95
+'th',    # 0x96
+'ph',    # 0x97
+'p',    # 0x98
+'x',    # 0x99
+'p',    # 0x9a
+'<',    # 0x9b
+'>',    # 0x9c
+'[?]',    # 0x9d
+'[?]',    # 0x9e
+'[?]',    # 0x9f
+'f',    # 0xa0
+'v',    # 0xa1
+'u',    # 0xa2
+'yr',    # 0xa3
+'y',    # 0xa4
+'w',    # 0xa5
+'th',    # 0xa6
+'th',    # 0xa7
+'a',    # 0xa8
+'o',    # 0xa9
+'ac',    # 0xaa
+'ae',    # 0xab
+'o',    # 0xac
+'o',    # 0xad
+'o',    # 0xae
+'oe',    # 0xaf
+'on',    # 0xb0
+'r',    # 0xb1
+'k',    # 0xb2
+'c',    # 0xb3
+'k',    # 0xb4
+'g',    # 0xb5
+'ng',    # 0xb6
+'g',    # 0xb7
+'g',    # 0xb8
+'w',    # 0xb9
+'h',    # 0xba
+'h',    # 0xbb
+'h',    # 0xbc
+'h',    # 0xbd
+'n',    # 0xbe
+'n',    # 0xbf
+'n',    # 0xc0
+'i',    # 0xc1
+'e',    # 0xc2
+'j',    # 0xc3
+'g',    # 0xc4
+'ae',    # 0xc5
+'a',    # 0xc6
+'eo',    # 0xc7
+'p',    # 0xc8
+'z',    # 0xc9
+'s',    # 0xca
+'s',    # 0xcb
+'s',    # 0xcc
+'c',    # 0xcd
+'z',    # 0xce
+'t',    # 0xcf
+'t',    # 0xd0
+'d',    # 0xd1
+'b',    # 0xd2
+'b',    # 0xd3
+'p',    # 0xd4
+'p',    # 0xd5
+'e',    # 0xd6
+'m',    # 0xd7
+'m',    # 0xd8
+'m',    # 0xd9
+'l',    # 0xda
+'l',    # 0xdb
+'ng',    # 0xdc
+'ng',    # 0xdd
+'d',    # 0xde
+'o',    # 0xdf
+'ear',    # 0xe0
+'ior',    # 0xe1
+'qu',    # 0xe2
+'qu',    # 0xe3
+'qu',    # 0xe4
+'s',    # 0xe5
+'yr',    # 0xe6
+'yr',    # 0xe7
+'yr',    # 0xe8
+'q',    # 0xe9
+'x',    # 0xea
+'.',    # 0xeb
+':',    # 0xec
+'+',    # 0xed
+'17',    # 0xee
+'18',    # 0xef
+'19',    # 0xf0
+'[?]',    # 0xf1
+'[?]',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x017.py

@@ -0,0 +1,257 @@
+data = (
+'[?]',    # 0x00
+'[?]',    # 0x01
+'[?]',    # 0x02
+'[?]',    # 0x03
+'[?]',    # 0x04
+'[?]',    # 0x05
+'[?]',    # 0x06
+'[?]',    # 0x07
+'[?]',    # 0x08
+'[?]',    # 0x09
+'[?]',    # 0x0a
+'[?]',    # 0x0b
+'[?]',    # 0x0c
+'[?]',    # 0x0d
+'[?]',    # 0x0e
+'[?]',    # 0x0f
+'[?]',    # 0x10
+'[?]',    # 0x11
+'[?]',    # 0x12
+'[?]',    # 0x13
+'[?]',    # 0x14
+'[?]',    # 0x15
+'[?]',    # 0x16
+'[?]',    # 0x17
+'[?]',    # 0x18
+'[?]',    # 0x19
+'[?]',    # 0x1a
+'[?]',    # 0x1b
+'[?]',    # 0x1c
+'[?]',    # 0x1d
+'[?]',    # 0x1e
+'[?]',    # 0x1f
+'[?]',    # 0x20
+'[?]',    # 0x21
+'[?]',    # 0x22
+'[?]',    # 0x23
+'[?]',    # 0x24
+'[?]',    # 0x25
+'[?]',    # 0x26
+'[?]',    # 0x27
+'[?]',    # 0x28
+'[?]',    # 0x29
+'[?]',    # 0x2a
+'[?]',    # 0x2b
+'[?]',    # 0x2c
+'[?]',    # 0x2d
+'[?]',    # 0x2e
+'[?]',    # 0x2f
+'[?]',    # 0x30
+'[?]',    # 0x31
+'[?]',    # 0x32
+'[?]',    # 0x33
+'[?]',    # 0x34
+'[?]',    # 0x35
+'[?]',    # 0x36
+'[?]',    # 0x37
+'[?]',    # 0x38
+'[?]',    # 0x39
+'[?]',    # 0x3a
+'[?]',    # 0x3b
+'[?]',    # 0x3c
+'[?]',    # 0x3d
+'[?]',    # 0x3e
+'[?]',    # 0x3f
+'[?]',    # 0x40
+'[?]',    # 0x41
+'[?]',    # 0x42
+'[?]',    # 0x43
+'[?]',    # 0x44
+'[?]',    # 0x45
+'[?]',    # 0x46
+'[?]',    # 0x47
+'[?]',    # 0x48
+'[?]',    # 0x49
+'[?]',    # 0x4a
+'[?]',    # 0x4b
+'[?]',    # 0x4c
+'[?]',    # 0x4d
+'[?]',    # 0x4e
+'[?]',    # 0x4f
+'[?]',    # 0x50
+'[?]',    # 0x51
+'[?]',    # 0x52
+'[?]',    # 0x53
+'[?]',    # 0x54
+'[?]',    # 0x55
+'[?]',    # 0x56
+'[?]',    # 0x57
+'[?]',    # 0x58
+'[?]',    # 0x59
+'[?]',    # 0x5a
+'[?]',    # 0x5b
+'[?]',    # 0x5c
+'[?]',    # 0x5d
+'[?]',    # 0x5e
+'[?]',    # 0x5f
+'[?]',    # 0x60
+'[?]',    # 0x61
+'[?]',    # 0x62
+'[?]',    # 0x63
+'[?]',    # 0x64
+'[?]',    # 0x65
+'[?]',    # 0x66
+'[?]',    # 0x67
+'[?]',    # 0x68
+'[?]',    # 0x69
+'[?]',    # 0x6a
+'[?]',    # 0x6b
+'[?]',    # 0x6c
+'[?]',    # 0x6d
+'[?]',    # 0x6e
+'[?]',    # 0x6f
+'[?]',    # 0x70
+'[?]',    # 0x71
+'[?]',    # 0x72
+'[?]',    # 0x73
+'[?]',    # 0x74
+'[?]',    # 0x75
+'[?]',    # 0x76
+'[?]',    # 0x77
+'[?]',    # 0x78
+'[?]',    # 0x79
+'[?]',    # 0x7a
+'[?]',    # 0x7b
+'[?]',    # 0x7c
+'[?]',    # 0x7d
+'[?]',    # 0x7e
+'[?]',    # 0x7f
+'k',    # 0x80
+'kh',    # 0x81
+'g',    # 0x82
+'gh',    # 0x83
+'ng',    # 0x84
+'c',    # 0x85
+'ch',    # 0x86
+'j',    # 0x87
+'jh',    # 0x88
+'ny',    # 0x89
+'t',    # 0x8a
+'tth',    # 0x8b
+'d',    # 0x8c
+'ddh',    # 0x8d
+'nn',    # 0x8e
+'t',    # 0x8f
+'th',    # 0x90
+'d',    # 0x91
+'dh',    # 0x92
+'n',    # 0x93
+'p',    # 0x94
+'ph',    # 0x95
+'b',    # 0x96
+'bh',    # 0x97
+'m',    # 0x98
+'y',    # 0x99
+'r',    # 0x9a
+'l',    # 0x9b
+'v',    # 0x9c
+'sh',    # 0x9d
+'ss',    # 0x9e
+'s',    # 0x9f
+'h',    # 0xa0
+'l',    # 0xa1
+'q',    # 0xa2
+'a',    # 0xa3
+'aa',    # 0xa4
+'i',    # 0xa5
+'ii',    # 0xa6
+'u',    # 0xa7
+'uk',    # 0xa8
+'uu',    # 0xa9
+'uuv',    # 0xaa
+'ry',    # 0xab
+'ryy',    # 0xac
+'ly',    # 0xad
+'lyy',    # 0xae
+'e',    # 0xaf
+'ai',    # 0xb0
+'oo',    # 0xb1
+'oo',    # 0xb2
+'au',    # 0xb3
+'a',    # 0xb4
+'aa',    # 0xb5
+'aa',    # 0xb6
+'i',    # 0xb7
+'ii',    # 0xb8
+'y',    # 0xb9
+'yy',    # 0xba
+'u',    # 0xbb
+'uu',    # 0xbc
+'ua',    # 0xbd
+'oe',    # 0xbe
+'ya',    # 0xbf
+'ie',    # 0xc0
+'e',    # 0xc1
+'ae',    # 0xc2
+'ai',    # 0xc3
+'oo',    # 0xc4
+'au',    # 0xc5
+'M',    # 0xc6
+'H',    # 0xc7
+'a`',    # 0xc8
+'',    # 0xc9
+'',    # 0xca
+'',    # 0xcb
+'r',    # 0xcc
+'',    # 0xcd
+'!',    # 0xce
+'',    # 0xcf
+'',    # 0xd0
+'',    # 0xd1
+'',    # 0xd2
+'',    # 0xd3
+'.',    # 0xd4
+' // ',    # 0xd5
+':',    # 0xd6
+'+',    # 0xd7
+'++',    # 0xd8
+' * ',    # 0xd9
+' /// ',    # 0xda
+'KR',    # 0xdb
+'\'',    # 0xdc
+'[?]',    # 0xdd
+'[?]',    # 0xde
+'[?]',    # 0xdf
+'0',    # 0xe0
+'1',    # 0xe1
+'2',    # 0xe2
+'3',    # 0xe3
+'4',    # 0xe4
+'5',    # 0xe5
+'6',    # 0xe6
+'7',    # 0xe7
+'8',    # 0xe8
+'9',    # 0xe9
+'[?]',    # 0xea
+'[?]',    # 0xeb
+'[?]',    # 0xec
+'[?]',    # 0xed
+'[?]',    # 0xee
+'[?]',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'[?]',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x018.py

@@ -0,0 +1,257 @@
+data = (
+' @ ',    # 0x00
+' ... ',    # 0x01
+', ',    # 0x02
+'. ',    # 0x03
+': ',    # 0x04
+' // ',    # 0x05
+'',    # 0x06
+'-',    # 0x07
+', ',    # 0x08
+'. ',    # 0x09
+'',    # 0x0a
+'',    # 0x0b
+'',    # 0x0c
+'',    # 0x0d
+'',    # 0x0e
+'[?]',    # 0x0f
+'0',    # 0x10
+'1',    # 0x11
+'2',    # 0x12
+'3',    # 0x13
+'4',    # 0x14
+'5',    # 0x15
+'6',    # 0x16
+'7',    # 0x17
+'8',    # 0x18
+'9',    # 0x19
+'[?]',    # 0x1a
+'[?]',    # 0x1b
+'[?]',    # 0x1c
+'[?]',    # 0x1d
+'[?]',    # 0x1e
+'[?]',    # 0x1f
+'a',    # 0x20
+'e',    # 0x21
+'i',    # 0x22
+'o',    # 0x23
+'u',    # 0x24
+'O',    # 0x25
+'U',    # 0x26
+'ee',    # 0x27
+'n',    # 0x28
+'ng',    # 0x29
+'b',    # 0x2a
+'p',    # 0x2b
+'q',    # 0x2c
+'g',    # 0x2d
+'m',    # 0x2e
+'l',    # 0x2f
+'s',    # 0x30
+'sh',    # 0x31
+'t',    # 0x32
+'d',    # 0x33
+'ch',    # 0x34
+'j',    # 0x35
+'y',    # 0x36
+'r',    # 0x37
+'w',    # 0x38
+'f',    # 0x39
+'k',    # 0x3a
+'kha',    # 0x3b
+'ts',    # 0x3c
+'z',    # 0x3d
+'h',    # 0x3e
+'zr',    # 0x3f
+'lh',    # 0x40
+'zh',    # 0x41
+'ch',    # 0x42
+'-',    # 0x43
+'e',    # 0x44
+'i',    # 0x45
+'o',    # 0x46
+'u',    # 0x47
+'O',    # 0x48
+'U',    # 0x49
+'ng',    # 0x4a
+'b',    # 0x4b
+'p',    # 0x4c
+'q',    # 0x4d
+'g',    # 0x4e
+'m',    # 0x4f
+'t',    # 0x50
+'d',    # 0x51
+'ch',    # 0x52
+'j',    # 0x53
+'ts',    # 0x54
+'y',    # 0x55
+'w',    # 0x56
+'k',    # 0x57
+'g',    # 0x58
+'h',    # 0x59
+'jy',    # 0x5a
+'ny',    # 0x5b
+'dz',    # 0x5c
+'e',    # 0x5d
+'i',    # 0x5e
+'iy',    # 0x5f
+'U',    # 0x60
+'u',    # 0x61
+'ng',    # 0x62
+'k',    # 0x63
+'g',    # 0x64
+'h',    # 0x65
+'p',    # 0x66
+'sh',    # 0x67
+'t',    # 0x68
+'d',    # 0x69
+'j',    # 0x6a
+'f',    # 0x6b
+'g',    # 0x6c
+'h',    # 0x6d
+'ts',    # 0x6e
+'z',    # 0x6f
+'r',    # 0x70
+'ch',    # 0x71
+'zh',    # 0x72
+'i',    # 0x73
+'k',    # 0x74
+'r',    # 0x75
+'f',    # 0x76
+'zh',    # 0x77
+'[?]',    # 0x78
+'[?]',    # 0x79
+'[?]',    # 0x7a
+'[?]',    # 0x7b
+'[?]',    # 0x7c
+'[?]',    # 0x7d
+'[?]',    # 0x7e
+'[?]',    # 0x7f
+'[?]',    # 0x80
+'H',    # 0x81
+'X',    # 0x82
+'W',    # 0x83
+'M',    # 0x84
+' 3 ',    # 0x85
+' 333 ',    # 0x86
+'a',    # 0x87
+'i',    # 0x88
+'k',    # 0x89
+'ng',    # 0x8a
+'c',    # 0x8b
+'tt',    # 0x8c
+'tth',    # 0x8d
+'dd',    # 0x8e
+'nn',    # 0x8f
+'t',    # 0x90
+'d',    # 0x91
+'p',    # 0x92
+'ph',    # 0x93
+'ss',    # 0x94
+'zh',    # 0x95
+'z',    # 0x96
+'a',    # 0x97
+'t',    # 0x98
+'zh',    # 0x99
+'gh',    # 0x9a
+'ng',    # 0x9b
+'c',    # 0x9c
+'jh',    # 0x9d
+'tta',    # 0x9e
+'ddh',    # 0x9f
+'t',    # 0xa0
+'dh',    # 0xa1
+'ss',    # 0xa2
+'cy',    # 0xa3
+'zh',    # 0xa4
+'z',    # 0xa5
+'u',    # 0xa6
+'y',    # 0xa7
+'bh',    # 0xa8
+'\'',    # 0xa9
+'[?]',    # 0xaa
+'[?]',    # 0xab
+'[?]',    # 0xac
+'[?]',    # 0xad
+'[?]',    # 0xae
+'[?]',    # 0xaf
+'[?]',    # 0xb0
+'[?]',    # 0xb1
+'[?]',    # 0xb2
+'[?]',    # 0xb3
+'[?]',    # 0xb4
+'[?]',    # 0xb5
+'[?]',    # 0xb6
+'[?]',    # 0xb7
+'[?]',    # 0xb8
+'[?]',    # 0xb9
+'[?]',    # 0xba
+'[?]',    # 0xbb
+'[?]',    # 0xbc
+'[?]',    # 0xbd
+'[?]',    # 0xbe
+'[?]',    # 0xbf
+'[?]',    # 0xc0
+'[?]',    # 0xc1
+'[?]',    # 0xc2
+'[?]',    # 0xc3
+'[?]',    # 0xc4
+'[?]',    # 0xc5
+'[?]',    # 0xc6
+'[?]',    # 0xc7
+'[?]',    # 0xc8
+'[?]',    # 0xc9
+'[?]',    # 0xca
+'[?]',    # 0xcb
+'[?]',    # 0xcc
+'[?]',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'[?]',    # 0xd0
+'[?]',    # 0xd1
+'[?]',    # 0xd2
+'[?]',    # 0xd3
+'[?]',    # 0xd4
+'[?]',    # 0xd5
+'[?]',    # 0xd6
+'[?]',    # 0xd7
+'[?]',    # 0xd8
+'[?]',    # 0xd9
+'[?]',    # 0xda
+'[?]',    # 0xdb
+'[?]',    # 0xdc
+'[?]',    # 0xdd
+'[?]',    # 0xde
+'[?]',    # 0xdf
+'[?]',    # 0xe0
+'[?]',    # 0xe1
+'[?]',    # 0xe2
+'[?]',    # 0xe3
+'[?]',    # 0xe4
+'[?]',    # 0xe5
+'[?]',    # 0xe6
+'[?]',    # 0xe7
+'[?]',    # 0xe8
+'[?]',    # 0xe9
+'[?]',    # 0xea
+'[?]',    # 0xeb
+'[?]',    # 0xec
+'[?]',    # 0xed
+'[?]',    # 0xee
+'[?]',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'[?]',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x01d.py

@@ -0,0 +1,257 @@
+data = (
+'',    # 0x00
+'',    # 0x01
+'',    # 0x02
+'',    # 0x03
+'',    # 0x04
+'',    # 0x05
+'',    # 0x06
+'',    # 0x07
+'',    # 0x08
+'',    # 0x09
+'',    # 0x0a
+'',    # 0x0b
+'',    # 0x0c
+'',    # 0x0d
+'',    # 0x0e
+'',    # 0x0f
+'',    # 0x10
+'',    # 0x11
+'',    # 0x12
+'',    # 0x13
+'',    # 0x14
+'',    # 0x15
+'',    # 0x16
+'',    # 0x17
+'',    # 0x18
+'',    # 0x19
+'',    # 0x1a
+'',    # 0x1b
+'',    # 0x1c
+'',    # 0x1d
+'',    # 0x1e
+'',    # 0x1f
+'',    # 0x20
+'',    # 0x21
+'',    # 0x22
+'',    # 0x23
+'',    # 0x24
+'',    # 0x25
+'',    # 0x26
+'',    # 0x27
+'',    # 0x28
+'',    # 0x29
+'',    # 0x2a
+'',    # 0x2b
+'',    # 0x2c
+'',    # 0x2d
+'',    # 0x2e
+'',    # 0x2f
+'',    # 0x30
+'',    # 0x31
+'',    # 0x32
+'',    # 0x33
+'',    # 0x34
+'',    # 0x35
+'',    # 0x36
+'',    # 0x37
+'',    # 0x38
+'',    # 0x39
+'',    # 0x3a
+'',    # 0x3b
+'',    # 0x3c
+'',    # 0x3d
+'',    # 0x3e
+'',    # 0x3f
+'',    # 0x40
+'',    # 0x41
+'',    # 0x42
+'',    # 0x43
+'',    # 0x44
+'',    # 0x45
+'',    # 0x46
+'',    # 0x47
+'',    # 0x48
+'',    # 0x49
+'',    # 0x4a
+'',    # 0x4b
+'',    # 0x4c
+'',    # 0x4d
+'',    # 0x4e
+'',    # 0x4f
+'',    # 0x50
+'',    # 0x51
+'',    # 0x52
+'',    # 0x53
+'',    # 0x54
+'',    # 0x55
+'',    # 0x56
+'',    # 0x57
+'',    # 0x58
+'',    # 0x59
+'',    # 0x5a
+'',    # 0x5b
+'',    # 0x5c
+'',    # 0x5d
+'',    # 0x5e
+'',    # 0x5f
+'',    # 0x60
+'',    # 0x61
+'',    # 0x62
+'',    # 0x63
+'',    # 0x64
+'',    # 0x65
+'',    # 0x66
+'',    # 0x67
+'',    # 0x68
+'',    # 0x69
+'',    # 0x6a
+'',    # 0x6b
+'b',    # 0x6c
+'d',    # 0x6d
+'f',    # 0x6e
+'m',    # 0x6f
+'n',    # 0x70
+'p',    # 0x71
+'r',    # 0x72
+'r',    # 0x73
+'s',    # 0x74
+'t',    # 0x75
+'z',    # 0x76
+'g',    # 0x77
+'',    # 0x78
+'',    # 0x79
+'',    # 0x7a
+'',    # 0x7b
+'',    # 0x7c
+'p',    # 0x7d
+'',    # 0x7e
+'',    # 0x7f
+'b',    # 0x80
+'d',    # 0x81
+'f',    # 0x82
+'g',    # 0x83
+'k',    # 0x84
+'l',    # 0x85
+'m',    # 0x86
+'n',    # 0x87
+'p',    # 0x88
+'r',    # 0x89
+'s',    # 0x8a
+'',    # 0x8b
+'v',    # 0x8c
+'x',    # 0x8d
+'z',    # 0x8e
+'',    # 0x8f
+'',    # 0x90
+'',    # 0x91
+'',    # 0x92
+'',    # 0x93
+'',    # 0x94
+'',    # 0x95
+'',    # 0x96
+'',    # 0x97
+'',    # 0x98
+'',    # 0x99
+'',    # 0x9a
+'',    # 0x9b
+'',    # 0x9c
+'',    # 0x9d
+'',    # 0x9e
+'',    # 0x9f
+'',    # 0xa0
+'',    # 0xa1
+'',    # 0xa2
+'',    # 0xa3
+'',    # 0xa4
+'',    # 0xa5
+'',    # 0xa6
+'',    # 0xa7
+'',    # 0xa8
+'',    # 0xa9
+'',    # 0xaa
+'',    # 0xab
+'',    # 0xac
+'',    # 0xad
+'',    # 0xae
+'',    # 0xaf
+'',    # 0xb0
+'',    # 0xb1
+'',    # 0xb2
+'',    # 0xb3
+'',    # 0xb4
+'',    # 0xb5
+'',    # 0xb6
+'',    # 0xb7
+'',    # 0xb8
+'',    # 0xb9
+'',    # 0xba
+'',    # 0xbb
+'',    # 0xbc
+'',    # 0xbd
+'',    # 0xbe
+'',    # 0xbf
+'',    # 0xc0
+'',    # 0xc1
+'',    # 0xc2
+'',    # 0xc3
+'',    # 0xc4
+'',    # 0xc5
+'',    # 0xc6
+'',    # 0xc7
+'',    # 0xc8
+'',    # 0xc9
+'',    # 0xca
+'',    # 0xcb
+'',    # 0xcc
+'',    # 0xcd
+'',    # 0xce
+'',    # 0xcf
+'',    # 0xd0
+'',    # 0xd1
+'',    # 0xd2
+'',    # 0xd3
+'',    # 0xd4
+'',    # 0xd5
+'',    # 0xd6
+'',    # 0xd7
+'',    # 0xd8
+'',    # 0xd9
+'',    # 0xda
+'',    # 0xdb
+'',    # 0xdc
+'',    # 0xdd
+'',    # 0xde
+'',    # 0xdf
+'',    # 0xe0
+'',    # 0xe1
+'',    # 0xe2
+'',    # 0xe3
+'',    # 0xe4
+'',    # 0xe5
+'',    # 0xe6
+'',    # 0xe7
+'',    # 0xe8
+'',    # 0xe9
+'',    # 0xea
+'',    # 0xeb
+'',    # 0xec
+'',    # 0xed
+'',    # 0xee
+'',    # 0xef
+'',    # 0xf0
+'',    # 0xf1
+'',    # 0xf2
+'',    # 0xf3
+'',    # 0xf4
+'',    # 0xf5
+'',    # 0xf6
+'',    # 0xf7
+'',    # 0xf8
+'',    # 0xf9
+'',    # 0xfa
+'',    # 0xfb
+'',    # 0xfc
+'',    # 0xfd
+'',    # 0xfe
+)

lib/unidecode/x01e.py

@@ -0,0 +1,257 @@
+data = (
+'A',    # 0x00
+'a',    # 0x01
+'B',    # 0x02
+'b',    # 0x03
+'B',    # 0x04
+'b',    # 0x05
+'B',    # 0x06
+'b',    # 0x07
+'C',    # 0x08
+'c',    # 0x09
+'D',    # 0x0a
+'d',    # 0x0b
+'D',    # 0x0c
+'d',    # 0x0d
+'D',    # 0x0e
+'d',    # 0x0f
+'D',    # 0x10
+'d',    # 0x11
+'D',    # 0x12
+'d',    # 0x13
+'E',    # 0x14
+'e',    # 0x15
+'E',    # 0x16
+'e',    # 0x17
+'E',    # 0x18
+'e',    # 0x19
+'E',    # 0x1a
+'e',    # 0x1b
+'E',    # 0x1c
+'e',    # 0x1d
+'F',    # 0x1e
+'f',    # 0x1f
+'G',    # 0x20
+'g',    # 0x21
+'H',    # 0x22
+'h',    # 0x23
+'H',    # 0x24
+'h',    # 0x25
+'H',    # 0x26
+'h',    # 0x27
+'H',    # 0x28
+'h',    # 0x29
+'H',    # 0x2a
+'h',    # 0x2b
+'I',    # 0x2c
+'i',    # 0x2d
+'I',    # 0x2e
+'i',    # 0x2f
+'K',    # 0x30
+'k',    # 0x31
+'K',    # 0x32
+'k',    # 0x33
+'K',    # 0x34
+'k',    # 0x35
+'L',    # 0x36
+'l',    # 0x37
+'L',    # 0x38
+'l',    # 0x39
+'L',    # 0x3a
+'l',    # 0x3b
+'L',    # 0x3c
+'l',    # 0x3d
+'M',    # 0x3e
+'m',    # 0x3f
+'M',    # 0x40
+'m',    # 0x41
+'M',    # 0x42
+'m',    # 0x43
+'N',    # 0x44
+'n',    # 0x45
+'N',    # 0x46
+'n',    # 0x47
+'N',    # 0x48
+'n',    # 0x49
+'N',    # 0x4a
+'n',    # 0x4b
+'O',    # 0x4c
+'o',    # 0x4d
+'O',    # 0x4e
+'o',    # 0x4f
+'O',    # 0x50
+'o',    # 0x51
+'O',    # 0x52
+'o',    # 0x53
+'P',    # 0x54
+'p',    # 0x55
+'P',    # 0x56
+'p',    # 0x57
+'R',    # 0x58
+'r',    # 0x59
+'R',    # 0x5a
+'r',    # 0x5b
+'R',    # 0x5c
+'r',    # 0x5d
+'R',    # 0x5e
+'r',    # 0x5f
+'S',    # 0x60
+'s',    # 0x61
+'S',    # 0x62
+'s',    # 0x63
+'S',    # 0x64
+'s',    # 0x65
+'S',    # 0x66
+'s',    # 0x67
+'S',    # 0x68
+'s',    # 0x69
+'T',    # 0x6a
+'t',    # 0x6b
+'T',    # 0x6c
+'t',    # 0x6d
+'T',    # 0x6e
+'t',    # 0x6f
+'T',    # 0x70
+'t',    # 0x71
+'U',    # 0x72
+'u',    # 0x73
+'U',    # 0x74
+'u',    # 0x75
+'U',    # 0x76
+'u',    # 0x77
+'U',    # 0x78
+'u',    # 0x79
+'U',    # 0x7a
+'u',    # 0x7b
+'V',    # 0x7c
+'v',    # 0x7d
+'V',    # 0x7e
+'v',    # 0x7f
+'W',    # 0x80
+'w',    # 0x81
+'W',    # 0x82
+'w',    # 0x83
+'W',    # 0x84
+'w',    # 0x85
+'W',    # 0x86
+'w',    # 0x87
+'W',    # 0x88
+'w',    # 0x89
+'X',    # 0x8a
+'x',    # 0x8b
+'X',    # 0x8c
+'x',    # 0x8d
+'Y',    # 0x8e
+'y',    # 0x8f
+'Z',    # 0x90
+'z',    # 0x91
+'Z',    # 0x92
+'z',    # 0x93
+'Z',    # 0x94
+'z',    # 0x95
+'h',    # 0x96
+'t',    # 0x97
+'w',    # 0x98
+'y',    # 0x99
+'a',    # 0x9a
+'S',    # 0x9b
+'[?]',    # 0x9c
+'[?]',    # 0x9d
+'Ss',    # 0x9e
+'[?]',    # 0x9f
+'A',    # 0xa0
+'a',    # 0xa1
+'A',    # 0xa2
+'a',    # 0xa3
+'A',    # 0xa4
+'a',    # 0xa5
+'A',    # 0xa6
+'a',    # 0xa7
+'A',    # 0xa8
+'a',    # 0xa9
+'A',    # 0xaa
+'a',    # 0xab
+'A',    # 0xac
+'a',    # 0xad
+'A',    # 0xae
+'a',    # 0xaf
+'A',    # 0xb0
+'a',    # 0xb1
+'A',    # 0xb2
+'a',    # 0xb3
+'A',    # 0xb4
+'a',    # 0xb5
+'A',    # 0xb6
+'a',    # 0xb7
+'E',    # 0xb8
+'e',    # 0xb9
+'E',    # 0xba
+'e',    # 0xbb
+'E',    # 0xbc
+'e',    # 0xbd
+'E',    # 0xbe
+'e',    # 0xbf
+'E',    # 0xc0
+'e',    # 0xc1
+'E',    # 0xc2
+'e',    # 0xc3
+'E',    # 0xc4
+'e',    # 0xc5
+'E',    # 0xc6
+'e',    # 0xc7
+'I',    # 0xc8
+'i',    # 0xc9
+'I',    # 0xca
+'i',    # 0xcb
+'O',    # 0xcc
+'o',    # 0xcd
+'O',    # 0xce
+'o',    # 0xcf
+'O',    # 0xd0
+'o',    # 0xd1
+'O',    # 0xd2
+'o',    # 0xd3
+'O',    # 0xd4
+'o',    # 0xd5
+'O',    # 0xd6
+'o',    # 0xd7
+'O',    # 0xd8
+'o',    # 0xd9
+'O',    # 0xda
+'o',    # 0xdb
+'O',    # 0xdc
+'o',    # 0xdd
+'O',    # 0xde
+'o',    # 0xdf
+'O',    # 0xe0
+'o',    # 0xe1
+'O',    # 0xe2
+'o',    # 0xe3
+'U',    # 0xe4
+'u',    # 0xe5
+'U',    # 0xe6
+'u',    # 0xe7
+'U',    # 0xe8
+'u',    # 0xe9
+'U',    # 0xea
+'u',    # 0xeb
+'U',    # 0xec
+'u',    # 0xed
+'U',    # 0xee
+'u',    # 0xef
+'U',    # 0xf0
+'u',    # 0xf1
+'Y',    # 0xf2
+'y',    # 0xf3
+'Y',    # 0xf4
+'y',    # 0xf5
+'Y',    # 0xf6
+'y',    # 0xf7
+'Y',    # 0xf8
+'y',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x01f.py

@@ -0,0 +1,257 @@
+data = (
+'a',    # 0x00
+'a',    # 0x01
+'a',    # 0x02
+'a',    # 0x03
+'a',    # 0x04
+'a',    # 0x05
+'a',    # 0x06
+'a',    # 0x07
+'A',    # 0x08
+'A',    # 0x09
+'A',    # 0x0a
+'A',    # 0x0b
+'A',    # 0x0c
+'A',    # 0x0d
+'A',    # 0x0e
+'A',    # 0x0f
+'e',    # 0x10
+'e',    # 0x11
+'e',    # 0x12
+'e',    # 0x13
+'e',    # 0x14
+'e',    # 0x15
+'[?]',    # 0x16
+'[?]',    # 0x17
+'E',    # 0x18
+'E',    # 0x19
+'E',    # 0x1a
+'E',    # 0x1b
+'E',    # 0x1c
+'E',    # 0x1d
+'[?]',    # 0x1e
+'[?]',    # 0x1f
+'e',    # 0x20
+'e',    # 0x21
+'e',    # 0x22
+'e',    # 0x23
+'e',    # 0x24
+'e',    # 0x25
+'e',    # 0x26
+'e',    # 0x27
+'E',    # 0x28
+'E',    # 0x29
+'E',    # 0x2a
+'E',    # 0x2b
+'E',    # 0x2c
+'E',    # 0x2d
+'E',    # 0x2e
+'E',    # 0x2f
+'i',    # 0x30
+'i',    # 0x31
+'i',    # 0x32
+'i',    # 0x33
+'i',    # 0x34
+'i',    # 0x35
+'i',    # 0x36
+'i',    # 0x37
+'I',    # 0x38
+'I',    # 0x39
+'I',    # 0x3a
+'I',    # 0x3b
+'I',    # 0x3c
+'I',    # 0x3d
+'I',    # 0x3e
+'I',    # 0x3f
+'o',    # 0x40
+'o',    # 0x41
+'o',    # 0x42
+'o',    # 0x43
+'o',    # 0x44
+'o',    # 0x45
+'[?]',    # 0x46
+'[?]',    # 0x47
+'O',    # 0x48
+'O',    # 0x49
+'O',    # 0x4a
+'O',    # 0x4b
+'O',    # 0x4c
+'O',    # 0x4d
+'[?]',    # 0x4e
+'[?]',    # 0x4f
+'u',    # 0x50
+'u',    # 0x51
+'u',    # 0x52
+'u',    # 0x53
+'u',    # 0x54
+'u',    # 0x55
+'u',    # 0x56
+'u',    # 0x57
+'[?]',    # 0x58
+'U',    # 0x59
+'[?]',    # 0x5a
+'U',    # 0x5b
+'[?]',    # 0x5c
+'U',    # 0x5d
+'[?]',    # 0x5e
+'U',    # 0x5f
+'o',    # 0x60
+'o',    # 0x61
+'o',    # 0x62
+'o',    # 0x63
+'o',    # 0x64
+'o',    # 0x65
+'o',    # 0x66
+'o',    # 0x67
+'O',    # 0x68
+'O',    # 0x69
+'O',    # 0x6a
+'O',    # 0x6b
+'O',    # 0x6c
+'O',    # 0x6d
+'O',    # 0x6e
+'O',    # 0x6f
+'a',    # 0x70
+'a',    # 0x71
+'e',    # 0x72
+'e',    # 0x73
+'e',    # 0x74
+'e',    # 0x75
+'i',    # 0x76
+'i',    # 0x77
+'o',    # 0x78
+'o',    # 0x79
+'u',    # 0x7a
+'u',    # 0x7b
+'o',    # 0x7c
+'o',    # 0x7d
+'[?]',    # 0x7e
+'[?]',    # 0x7f
+'a',    # 0x80
+'a',    # 0x81
+'a',    # 0x82
+'a',    # 0x83
+'a',    # 0x84
+'a',    # 0x85
+'a',    # 0x86
+'a',    # 0x87
+'A',    # 0x88
+'A',    # 0x89
+'A',    # 0x8a
+'A',    # 0x8b
+'A',    # 0x8c
+'A',    # 0x8d
+'A',    # 0x8e
+'A',    # 0x8f
+'e',    # 0x90
+'e',    # 0x91
+'e',    # 0x92
+'e',    # 0x93
+'e',    # 0x94
+'e',    # 0x95
+'e',    # 0x96
+'e',    # 0x97
+'E',    # 0x98
+'E',    # 0x99
+'E',    # 0x9a
+'E',    # 0x9b
+'E',    # 0x9c
+'E',    # 0x9d
+'E',    # 0x9e
+'E',    # 0x9f
+'o',    # 0xa0
+'o',    # 0xa1
+'o',    # 0xa2
+'o',    # 0xa3
+'o',    # 0xa4
+'o',    # 0xa5
+'o',    # 0xa6
+'o',    # 0xa7
+'O',    # 0xa8
+'O',    # 0xa9
+'O',    # 0xaa
+'O',    # 0xab
+'O',    # 0xac
+'O',    # 0xad
+'O',    # 0xae
+'O',    # 0xaf
+'a',    # 0xb0
+'a',    # 0xb1
+'a',    # 0xb2
+'a',    # 0xb3
+'a',    # 0xb4
+'[?]',    # 0xb5
+'a',    # 0xb6
+'a',    # 0xb7
+'A',    # 0xb8
+'A',    # 0xb9
+'A',    # 0xba
+'A',    # 0xbb
+'A',    # 0xbc
+'\'',    # 0xbd
+'i',    # 0xbe
+'\'',    # 0xbf
+'~',    # 0xc0
+'"~',    # 0xc1
+'e',    # 0xc2
+'e',    # 0xc3
+'e',    # 0xc4
+'[?]',    # 0xc5
+'e',    # 0xc6
+'e',    # 0xc7
+'E',    # 0xc8
+'E',    # 0xc9
+'E',    # 0xca
+'E',    # 0xcb
+'E',    # 0xcc
+'\'`',    # 0xcd
+'\'\'',    # 0xce
+'\'~',    # 0xcf
+'i',    # 0xd0
+'i',    # 0xd1
+'i',    # 0xd2
+'i',    # 0xd3
+'[?]',    # 0xd4
+'[?]',    # 0xd5
+'i',    # 0xd6
+'i',    # 0xd7
+'I',    # 0xd8
+'I',    # 0xd9
+'I',    # 0xda
+'I',    # 0xdb
+'[?]',    # 0xdc
+'`\'',    # 0xdd
+'`\'',    # 0xde
+'`~',    # 0xdf
+'u',    # 0xe0
+'u',    # 0xe1
+'u',    # 0xe2
+'u',    # 0xe3
+'R',    # 0xe4
+'R',    # 0xe5
+'u',    # 0xe6
+'u',    # 0xe7
+'U',    # 0xe8
+'U',    # 0xe9
+'U',    # 0xea
+'U',    # 0xeb
+'R',    # 0xec
+'"`',    # 0xed
+'"\'',    # 0xee
+'`',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'o',    # 0xf2
+'o',    # 0xf3
+'o',    # 0xf4
+'[?]',    # 0xf5
+'o',    # 0xf6
+'o',    # 0xf7
+'O',    # 0xf8
+'O',    # 0xf9
+'O',    # 0xfa
+'O',    # 0xfb
+'O',    # 0xfc
+'\'',    # 0xfd
+'`',    # 0xfe
+)

lib/unidecode/x020.py

@@ -0,0 +1,257 @@
+data = (
+' ',    # 0x00
+' ',    # 0x01
+' ',    # 0x02
+' ',    # 0x03
+' ',    # 0x04
+' ',    # 0x05
+' ',    # 0x06
+' ',    # 0x07
+' ',    # 0x08
+' ',    # 0x09
+' ',    # 0x0a
+' ',    # 0x0b
+'',    # 0x0c
+'',    # 0x0d
+'',    # 0x0e
+'',    # 0x0f
+'-',    # 0x10
+'-',    # 0x11
+'-',    # 0x12
+'-',    # 0x13
+'--',    # 0x14
+'--',    # 0x15
+'||',    # 0x16
+'_',    # 0x17
+'\'',    # 0x18
+'\'',    # 0x19
+',',    # 0x1a
+'\'',    # 0x1b
+'"',    # 0x1c
+'"',    # 0x1d
+',,',    # 0x1e
+'"',    # 0x1f
+'+',    # 0x20
+'++',    # 0x21
+'*',    # 0x22
+'*>',    # 0x23
+'.',    # 0x24
+'..',    # 0x25
+'...',    # 0x26
+'.',    # 0x27
+'\x0a',    # 0x28
+'\x0a\x0a',    # 0x29
+'',    # 0x2a
+'',    # 0x2b
+'',    # 0x2c
+'',    # 0x2d
+'',    # 0x2e
+' ',    # 0x2f
+'%0',    # 0x30
+'%00',    # 0x31
+'\'',    # 0x32
+'\'\'',    # 0x33
+'\'\'\'',    # 0x34
+'`',    # 0x35
+'``',    # 0x36
+'```',    # 0x37
+'^',    # 0x38
+'<',    # 0x39
+'>',    # 0x3a
+'*',    # 0x3b
+'!!',    # 0x3c
+'!?',    # 0x3d
+'-',    # 0x3e
+'_',    # 0x3f
+'-',    # 0x40
+'^',    # 0x41
+'***',    # 0x42
+'--',    # 0x43
+'/',    # 0x44
+'-[',    # 0x45
+']-',    # 0x46
+'??',    # 0x47
+'?!',    # 0x48
+'!?',    # 0x49
+'7',    # 0x4a
+'PP',    # 0x4b
+'(]',    # 0x4c
+'[)',    # 0x4d
+'*',    # 0x4e
+'[?]',    # 0x4f
+'[?]',    # 0x50
+'[?]',    # 0x51
+'%',    # 0x52
+'~',    # 0x53
+'[?]',    # 0x54
+'[?]',    # 0x55
+'[?]',    # 0x56
+"''''",    # 0x57
+'[?]',    # 0x58
+'[?]',    # 0x59
+'[?]',    # 0x5a
+'[?]',    # 0x5b
+'[?]',    # 0x5c
+'[?]',    # 0x5d
+'[?]',    # 0x5e
+'[?]',    # 0x5f
+'',    # 0x60
+'[?]',    # 0x61
+'[?]',    # 0x62
+'[?]',    # 0x63
+'[?]',    # 0x64
+'[?]',    # 0x65
+'[?]',    # 0x66
+'[?]',    # 0x67
+'[?]',    # 0x68
+'[?]',    # 0x69
+'',    # 0x6a
+'',    # 0x6b
+'',    # 0x6c
+'',    # 0x6d
+'',    # 0x6e
+'',    # 0x6f
+'0',    # 0x70
+'',    # 0x71
+'',    # 0x72
+'',    # 0x73
+'4',    # 0x74
+'5',    # 0x75
+'6',    # 0x76
+'7',    # 0x77
+'8',    # 0x78
+'9',    # 0x79
+'+',    # 0x7a
+'-',    # 0x7b
+'=',    # 0x7c
+'(',    # 0x7d
+')',    # 0x7e
+'n',    # 0x7f
+'0',    # 0x80
+'1',    # 0x81
+'2',    # 0x82
+'3',    # 0x83
+'4',    # 0x84
+'5',    # 0x85
+'6',    # 0x86
+'7',    # 0x87
+'8',    # 0x88
+'9',    # 0x89
+'+',    # 0x8a
+'-',    # 0x8b
+'=',    # 0x8c
+'(',    # 0x8d
+')',    # 0x8e
+'[?]',    # 0x8f
+'[?]',    # 0x90
+'[?]',    # 0x91
+'[?]',    # 0x92
+'[?]',    # 0x93
+'[?]',    # 0x94
+'[?]',    # 0x95
+'[?]',    # 0x96
+'[?]',    # 0x97
+'[?]',    # 0x98
+'[?]',    # 0x99
+'[?]',    # 0x9a
+'[?]',    # 0x9b
+'[?]',    # 0x9c
+'[?]',    # 0x9d
+'[?]',    # 0x9e
+'[?]',    # 0x9f
+'ECU',    # 0xa0
+'CL',    # 0xa1
+'Cr',    # 0xa2
+'FF',    # 0xa3
+'L',    # 0xa4
+'mil',    # 0xa5
+'N',    # 0xa6
+'Pts',    # 0xa7
+'Rs',    # 0xa8
+'W',    # 0xa9
+'NS',    # 0xaa
+'D',    # 0xab
+'EU',    # 0xac
+'K',    # 0xad
+'T',    # 0xae
+'Dr',    # 0xaf
+'[?]',    # 0xb0
+'[?]',    # 0xb1
+'[?]',    # 0xb2
+'[?]',    # 0xb3
+'[?]',    # 0xb4
+'[?]',    # 0xb5
+'[?]',    # 0xb6
+'[?]',    # 0xb7
+'[?]',    # 0xb8
+'[?]',    # 0xb9
+'[?]',    # 0xba
+'[?]',    # 0xbb
+'[?]',    # 0xbc
+'[?]',    # 0xbd
+'[?]',    # 0xbe
+'[?]',    # 0xbf
+'[?]',    # 0xc0
+'[?]',    # 0xc1
+'[?]',    # 0xc2
+'[?]',    # 0xc3
+'[?]',    # 0xc4
+'[?]',    # 0xc5
+'[?]',    # 0xc6
+'[?]',    # 0xc7
+'[?]',    # 0xc8
+'[?]',    # 0xc9
+'[?]',    # 0xca
+'[?]',    # 0xcb
+'[?]',    # 0xcc
+'[?]',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'',    # 0xd0
+'',    # 0xd1
+'',    # 0xd2
+'',    # 0xd3
+'',    # 0xd4
+'',    # 0xd5
+'',    # 0xd6
+'',    # 0xd7
+'',    # 0xd8
+'',    # 0xd9
+'',    # 0xda
+'',    # 0xdb
+'',    # 0xdc
+'',    # 0xdd
+'',    # 0xde
+'',    # 0xdf
+'',    # 0xe0
+'',    # 0xe1
+'',    # 0xe2
+'',    # 0xe3
+'[?]',    # 0xe4
+'',    # 0xe5
+'[?]',    # 0xe6
+'[?]',    # 0xe7
+'[?]',    # 0xe8
+'[?]',    # 0xe9
+'[?]',    # 0xea
+'[?]',    # 0xeb
+'[?]',    # 0xec
+'[?]',    # 0xed
+'[?]',    # 0xee
+'[?]',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'[?]',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x021.py

@@ -0,0 +1,257 @@
+data = (
+'',    # 0x00
+'',    # 0x01
+'',    # 0x02
+'',    # 0x03
+'',    # 0x04
+'',    # 0x05
+'',    # 0x06
+'',    # 0x07
+'',    # 0x08
+'',    # 0x09
+'',    # 0x0a
+'',    # 0x0b
+'',    # 0x0c
+'',    # 0x0d
+'',    # 0x0e
+'',    # 0x0f
+'',    # 0x10
+'',    # 0x11
+'',    # 0x12
+'',    # 0x13
+'',    # 0x14
+'',    # 0x15
+'',    # 0x16
+'',    # 0x17
+'',    # 0x18
+'',    # 0x19
+'',    # 0x1a
+'',    # 0x1b
+'',    # 0x1c
+'',    # 0x1d
+'',    # 0x1e
+'',    # 0x1f
+'(sm)',    # 0x20
+'TEL',    # 0x21
+'(tm)',    # 0x22
+'',    # 0x23
+'',    # 0x24
+'',    # 0x25
+'',    # 0x26
+'',    # 0x27
+'',    # 0x28
+'',    # 0x29
+'K',    # 0x2a
+'A',    # 0x2b
+'',    # 0x2c
+'',    # 0x2d
+'',    # 0x2e
+'',    # 0x2f
+'',    # 0x30
+'',    # 0x31
+'F',    # 0x32
+'',    # 0x33
+'',    # 0x34
+'',    # 0x35
+'',    # 0x36
+'',    # 0x37
+'',    # 0x38
+'',    # 0x39
+'',    # 0x3a
+'FAX',    # 0x3b
+'[?]',    # 0x3c
+'[?]',    # 0x3d
+'[?]',    # 0x3e
+'[?]',    # 0x3f
+'[?]',    # 0x40
+'[?]',    # 0x41
+'[?]',    # 0x42
+'[?]',    # 0x43
+'[?]',    # 0x44
+'[?]',    # 0x45
+'[?]',    # 0x46
+'[?]',    # 0x47
+'[?]',    # 0x48
+'[?]',    # 0x49
+'[?]',    # 0x4a
+'[?]',    # 0x4b
+'[?]',    # 0x4c
+'[?]',    # 0x4d
+'F',    # 0x4e
+'[?]',    # 0x4f
+'[?]',    # 0x50
+'[?]',    # 0x51
+'[?]',    # 0x52
+' 1/3 ',    # 0x53
+' 2/3 ',    # 0x54
+' 1/5 ',    # 0x55
+' 2/5 ',    # 0x56
+' 3/5 ',    # 0x57
+' 4/5 ',    # 0x58
+' 1/6 ',    # 0x59
+' 5/6 ',    # 0x5a
+' 1/8 ',    # 0x5b
+' 3/8 ',    # 0x5c
+' 5/8 ',    # 0x5d
+' 7/8 ',    # 0x5e
+' 1/',    # 0x5f
+'I',    # 0x60
+'II',    # 0x61
+'III',    # 0x62
+'IV',    # 0x63
+'V',    # 0x64
+'VI',    # 0x65
+'VII',    # 0x66
+'VIII',    # 0x67
+'IX',    # 0x68
+'X',    # 0x69
+'XI',    # 0x6a
+'XII',    # 0x6b
+'L',    # 0x6c
+'C',    # 0x6d
+'D',    # 0x6e
+'M',    # 0x6f
+'i',    # 0x70
+'ii',    # 0x71
+'iii',    # 0x72
+'iv',    # 0x73
+'v',    # 0x74
+'vi',    # 0x75
+'vii',    # 0x76
+'viii',    # 0x77
+'ix',    # 0x78
+'x',    # 0x79
+'xi',    # 0x7a
+'xii',    # 0x7b
+'l',    # 0x7c
+'c',    # 0x7d
+'d',    # 0x7e
+'m',    # 0x7f
+'(D',    # 0x80
+'D)',    # 0x81
+'((|))',    # 0x82
+')',    # 0x83
+'[?]',    # 0x84
+'[?]',    # 0x85
+'[?]',    # 0x86
+'[?]',    # 0x87
+'[?]',    # 0x88
+'[?]',    # 0x89
+'[?]',    # 0x8a
+'[?]',    # 0x8b
+'[?]',    # 0x8c
+'[?]',    # 0x8d
+'[?]',    # 0x8e
+'[?]',    # 0x8f
+'-',    # 0x90
+'|',    # 0x91
+'-',    # 0x92
+'|',    # 0x93
+'-',    # 0x94
+'|',    # 0x95
+'\\',    # 0x96
+'/',    # 0x97
+'\\',    # 0x98
+'/',    # 0x99
+'-',    # 0x9a
+'-',    # 0x9b
+'~',    # 0x9c
+'~',    # 0x9d
+'-',    # 0x9e
+'|',    # 0x9f
+'-',    # 0xa0
+'|',    # 0xa1
+'-',    # 0xa2
+'-',    # 0xa3
+'-',    # 0xa4
+'|',    # 0xa5
+'-',    # 0xa6
+'|',    # 0xa7
+'|',    # 0xa8
+'-',    # 0xa9
+'-',    # 0xaa
+'-',    # 0xab
+'-',    # 0xac
+'-',    # 0xad
+'-',    # 0xae
+'|',    # 0xaf
+'|',    # 0xb0
+'|',    # 0xb1
+'|',    # 0xb2
+'|',    # 0xb3
+'|',    # 0xb4
+'|',    # 0xb5
+'^',    # 0xb6
+'V',    # 0xb7
+'\\',    # 0xb8
+'=',    # 0xb9
+'V',    # 0xba
+'^',    # 0xbb
+'-',    # 0xbc
+'-',    # 0xbd
+'|',    # 0xbe
+'|',    # 0xbf
+'-',    # 0xc0
+'-',    # 0xc1
+'|',    # 0xc2
+'|',    # 0xc3
+'=',    # 0xc4
+'|',    # 0xc5
+'=',    # 0xc6
+'=',    # 0xc7
+'|',    # 0xc8
+'=',    # 0xc9
+'|',    # 0xca
+'=',    # 0xcb
+'=',    # 0xcc
+'=',    # 0xcd
+'=',    # 0xce
+'=',    # 0xcf
+'=',    # 0xd0
+'|',    # 0xd1
+'=',    # 0xd2
+'|',    # 0xd3
+'=',    # 0xd4
+'|',    # 0xd5
+'\\',    # 0xd6
+'/',    # 0xd7
+'\\',    # 0xd8
+'/',    # 0xd9
+'=',    # 0xda
+'=',    # 0xdb
+'~',    # 0xdc
+'~',    # 0xdd
+'|',    # 0xde
+'|',    # 0xdf
+'-',    # 0xe0
+'|',    # 0xe1
+'-',    # 0xe2
+'|',    # 0xe3
+'-',    # 0xe4
+'-',    # 0xe5
+'-',    # 0xe6
+'|',    # 0xe7
+'-',    # 0xe8
+'|',    # 0xe9
+'|',    # 0xea
+'|',    # 0xeb
+'|',    # 0xec
+'|',    # 0xed
+'|',    # 0xee
+'|',    # 0xef
+'-',    # 0xf0
+'\\',    # 0xf1
+'\\',    # 0xf2
+'|',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x022.py

@@ -0,0 +1,257 @@
+data = (
+'[?]',    # 0x00
+'[?]',    # 0x01
+'[?]',    # 0x02
+'[?]',    # 0x03
+'[?]',    # 0x04
+'[?]',    # 0x05
+'[?]',    # 0x06
+'[?]',    # 0x07
+'[?]',    # 0x08
+'[?]',    # 0x09
+'[?]',    # 0x0a
+'[?]',    # 0x0b
+'[?]',    # 0x0c
+'[?]',    # 0x0d
+'[?]',    # 0x0e
+'[?]',    # 0x0f
+'[?]',    # 0x10
+'[?]',    # 0x11
+'-',    # 0x12
+'[?]',    # 0x13
+'[?]',    # 0x14
+'/',    # 0x15
+'\\',    # 0x16
+'*',    # 0x17
+'[?]',    # 0x18
+'[?]',    # 0x19
+'[?]',    # 0x1a
+'[?]',    # 0x1b
+'[?]',    # 0x1c
+'[?]',    # 0x1d
+'[?]',    # 0x1e
+'[?]',    # 0x1f
+'[?]',    # 0x20
+'[?]',    # 0x21
+'[?]',    # 0x22
+'|',    # 0x23
+'[?]',    # 0x24
+'[?]',    # 0x25
+'[?]',    # 0x26
+'[?]',    # 0x27
+'[?]',    # 0x28
+'[?]',    # 0x29
+'[?]',    # 0x2a
+'[?]',    # 0x2b
+'[?]',    # 0x2c
+'[?]',    # 0x2d
+'[?]',    # 0x2e
+'[?]',    # 0x2f
+'[?]',    # 0x30
+'[?]',    # 0x31
+'[?]',    # 0x32
+'[?]',    # 0x33
+'[?]',    # 0x34
+'[?]',    # 0x35
+':',    # 0x36
+'[?]',    # 0x37
+'[?]',    # 0x38
+'[?]',    # 0x39
+'[?]',    # 0x3a
+'[?]',    # 0x3b
+'~',    # 0x3c
+'[?]',    # 0x3d
+'[?]',    # 0x3e
+'[?]',    # 0x3f
+'[?]',    # 0x40
+'[?]',    # 0x41
+'[?]',    # 0x42
+'[?]',    # 0x43
+'[?]',    # 0x44
+'[?]',    # 0x45
+'[?]',    # 0x46
+'[?]',    # 0x47
+'[?]',    # 0x48
+'[?]',    # 0x49
+'[?]',    # 0x4a
+'[?]',    # 0x4b
+'[?]',    # 0x4c
+'[?]',    # 0x4d
+'[?]',    # 0x4e
+'[?]',    # 0x4f
+'[?]',    # 0x50
+'[?]',    # 0x51
+'[?]',    # 0x52
+'[?]',    # 0x53
+'[?]',    # 0x54
+'[?]',    # 0x55
+'[?]',    # 0x56
+'[?]',    # 0x57
+'[?]',    # 0x58
+'[?]',    # 0x59
+'[?]',    # 0x5a
+'[?]',    # 0x5b
+'[?]',    # 0x5c
+'[?]',    # 0x5d
+'[?]',    # 0x5e
+'[?]',    # 0x5f
+'[?]',    # 0x60
+'[?]',    # 0x61
+'[?]',    # 0x62
+'[?]',    # 0x63
+'<=',    # 0x64
+'>=',    # 0x65
+'<=',    # 0x66
+'>=',    # 0x67
+'[?]',    # 0x68
+'[?]',    # 0x69
+'[?]',    # 0x6a
+'[?]',    # 0x6b
+'[?]',    # 0x6c
+'[?]',    # 0x6d
+'[?]',    # 0x6e
+'[?]',    # 0x6f
+'[?]',    # 0x70
+'[?]',    # 0x71
+'[?]',    # 0x72
+'[?]',    # 0x73
+'[?]',    # 0x74
+'[?]',    # 0x75
+'[?]',    # 0x76
+'[?]',    # 0x77
+'[?]',    # 0x78
+'[?]',    # 0x79
+'[?]',    # 0x7a
+'[?]',    # 0x7b
+'[?]',    # 0x7c
+'[?]',    # 0x7d
+'[?]',    # 0x7e
+'[?]',    # 0x7f
+'[?]',    # 0x80
+'[?]',    # 0x81
+'[?]',    # 0x82
+'[?]',    # 0x83
+'[?]',    # 0x84
+'[?]',    # 0x85
+'[?]',    # 0x86
+'[?]',    # 0x87
+'[?]',    # 0x88
+'[?]',    # 0x89
+'[?]',    # 0x8a
+'[?]',    # 0x8b
+'[?]',    # 0x8c
+'[?]',    # 0x8d
+'[?]',    # 0x8e
+'[?]',    # 0x8f
+'[?]',    # 0x90
+'[?]',    # 0x91
+'[?]',    # 0x92
+'[?]',    # 0x93
+'[?]',    # 0x94
+'[?]',    # 0x95
+'[?]',    # 0x96
+'[?]',    # 0x97
+'[?]',    # 0x98
+'[?]',    # 0x99
+'[?]',    # 0x9a
+'[?]',    # 0x9b
+'[?]',    # 0x9c
+'[?]',    # 0x9d
+'[?]',    # 0x9e
+'[?]',    # 0x9f
+'[?]',    # 0xa0
+'[?]',    # 0xa1
+'[?]',    # 0xa2
+'[?]',    # 0xa3
+'[?]',    # 0xa4
+'[?]',    # 0xa5
+'[?]',    # 0xa6
+'[?]',    # 0xa7
+'[?]',    # 0xa8
+'[?]',    # 0xa9
+'[?]',    # 0xaa
+'[?]',    # 0xab
+'[?]',    # 0xac
+'[?]',    # 0xad
+'[?]',    # 0xae
+'[?]',    # 0xaf
+'[?]',    # 0xb0
+'[?]',    # 0xb1
+'[?]',    # 0xb2
+'[?]',    # 0xb3
+'[?]',    # 0xb4
+'[?]',    # 0xb5
+'[?]',    # 0xb6
+'[?]',    # 0xb7
+'[?]',    # 0xb8
+'[?]',    # 0xb9
+'[?]',    # 0xba
+'[?]',    # 0xbb
+'[?]',    # 0xbc
+'[?]',    # 0xbd
+'[?]',    # 0xbe
+'[?]',    # 0xbf
+'[?]',    # 0xc0
+'[?]',    # 0xc1
+'[?]',    # 0xc2
+'[?]',    # 0xc3
+'[?]',    # 0xc4
+'[?]',    # 0xc5
+'[?]',    # 0xc6
+'[?]',    # 0xc7
+'[?]',    # 0xc8
+'[?]',    # 0xc9
+'[?]',    # 0xca
+'[?]',    # 0xcb
+'[?]',    # 0xcc
+'[?]',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'[?]',    # 0xd0
+'[?]',    # 0xd1
+'[?]',    # 0xd2
+'[?]',    # 0xd3
+'[?]',    # 0xd4
+'[?]',    # 0xd5
+'[?]',    # 0xd6
+'[?]',    # 0xd7
+'[?]',    # 0xd8
+'[?]',    # 0xd9
+'[?]',    # 0xda
+'[?]',    # 0xdb
+'[?]',    # 0xdc
+'[?]',    # 0xdd
+'[?]',    # 0xde
+'[?]',    # 0xdf
+'[?]',    # 0xe0
+'[?]',    # 0xe1
+'[?]',    # 0xe2
+'[?]',    # 0xe3
+'[?]',    # 0xe4
+'[?]',    # 0xe5
+'[?]',    # 0xe6
+'[?]',    # 0xe7
+'[?]',    # 0xe8
+'[?]',    # 0xe9
+'[?]',    # 0xea
+'[?]',    # 0xeb
+'[?]',    # 0xec
+'[?]',    # 0xed
+'[?]',    # 0xee
+'[?]',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'[?]',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x023.py

@@ -0,0 +1,257 @@
+data = (
+'[?]',    # 0x00
+'[?]',    # 0x01
+'[?]',    # 0x02
+'^',    # 0x03
+'[?]',    # 0x04
+'[?]',    # 0x05
+'[?]',    # 0x06
+'[?]',    # 0x07
+'[?]',    # 0x08
+'[?]',    # 0x09
+'[?]',    # 0x0a
+'[?]',    # 0x0b
+'[?]',    # 0x0c
+'[?]',    # 0x0d
+'[?]',    # 0x0e
+'[?]',    # 0x0f
+'[?]',    # 0x10
+'[?]',    # 0x11
+'[?]',    # 0x12
+'[?]',    # 0x13
+'[?]',    # 0x14
+'[?]',    # 0x15
+'[?]',    # 0x16
+'[?]',    # 0x17
+'[?]',    # 0x18
+'[?]',    # 0x19
+'[?]',    # 0x1a
+'[?]',    # 0x1b
+'[?]',    # 0x1c
+'[?]',    # 0x1d
+'[?]',    # 0x1e
+'[?]',    # 0x1f
+'[?]',    # 0x20
+'[?]',    # 0x21
+'[?]',    # 0x22
+'[?]',    # 0x23
+'[?]',    # 0x24
+'[?]',    # 0x25
+'[?]',    # 0x26
+'[?]',    # 0x27
+'[?]',    # 0x28
+'<',    # 0x29
+'> ',    # 0x2a
+'[?]',    # 0x2b
+'[?]',    # 0x2c
+'[?]',    # 0x2d
+'[?]',    # 0x2e
+'[?]',    # 0x2f
+'[?]',    # 0x30
+'[?]',    # 0x31
+'[?]',    # 0x32
+'[?]',    # 0x33
+'[?]',    # 0x34
+'[?]',    # 0x35
+'[?]',    # 0x36
+'[?]',    # 0x37
+'[?]',    # 0x38
+'[?]',    # 0x39
+'[?]',    # 0x3a
+'[?]',    # 0x3b
+'[?]',    # 0x3c
+'[?]',    # 0x3d
+'[?]',    # 0x3e
+'[?]',    # 0x3f
+'[?]',    # 0x40
+'[?]',    # 0x41
+'[?]',    # 0x42
+'[?]',    # 0x43
+'[?]',    # 0x44
+'[?]',    # 0x45
+'[?]',    # 0x46
+'[?]',    # 0x47
+'[?]',    # 0x48
+'[?]',    # 0x49
+'[?]',    # 0x4a
+'[?]',    # 0x4b
+'[?]',    # 0x4c
+'[?]',    # 0x4d
+'[?]',    # 0x4e
+'[?]',    # 0x4f
+'[?]',    # 0x50
+'[?]',    # 0x51
+'[?]',    # 0x52
+'[?]',    # 0x53
+'[?]',    # 0x54
+'[?]',    # 0x55
+'[?]',    # 0x56
+'[?]',    # 0x57
+'[?]',    # 0x58
+'[?]',    # 0x59
+'[?]',    # 0x5a
+'[?]',    # 0x5b
+'[?]',    # 0x5c
+'[?]',    # 0x5d
+'[?]',    # 0x5e
+'[?]',    # 0x5f
+'[?]',    # 0x60
+'[?]',    # 0x61
+'[?]',    # 0x62
+'[?]',    # 0x63
+'[?]',    # 0x64
+'[?]',    # 0x65
+'[?]',    # 0x66
+'[?]',    # 0x67
+'[?]',    # 0x68
+'[?]',    # 0x69
+'[?]',    # 0x6a
+'[?]',    # 0x6b
+'[?]',    # 0x6c
+'[?]',    # 0x6d
+'[?]',    # 0x6e
+'[?]',    # 0x6f
+'[?]',    # 0x70
+'[?]',    # 0x71
+'[?]',    # 0x72
+'[?]',    # 0x73
+'[?]',    # 0x74
+'[?]',    # 0x75
+'[?]',    # 0x76
+'[?]',    # 0x77
+'[?]',    # 0x78
+'[?]',    # 0x79
+'[?]',    # 0x7a
+'[?]',    # 0x7b
+'[?]',    # 0x7c
+'[?]',    # 0x7d
+'[?]',    # 0x7e
+'[?]',    # 0x7f
+'[?]',    # 0x80
+'[?]',    # 0x81
+'[?]',    # 0x82
+'[?]',    # 0x83
+'[?]',    # 0x84
+'[?]',    # 0x85
+'[?]',    # 0x86
+'[?]',    # 0x87
+'[?]',    # 0x88
+'[?]',    # 0x89
+'[?]',    # 0x8a
+'[?]',    # 0x8b
+'[?]',    # 0x8c
+'[?]',    # 0x8d
+'[?]',    # 0x8e
+'[?]',    # 0x8f
+'[?]',    # 0x90
+'[?]',    # 0x91
+'[?]',    # 0x92
+'[?]',    # 0x93
+'[?]',    # 0x94
+'[?]',    # 0x95
+'[?]',    # 0x96
+'[?]',    # 0x97
+'[?]',    # 0x98
+'[?]',    # 0x99
+'[?]',    # 0x9a
+'[?]',    # 0x9b
+'[?]',    # 0x9c
+'[?]',    # 0x9d
+'[?]',    # 0x9e
+'[?]',    # 0x9f
+'[?]',    # 0xa0
+'[?]',    # 0xa1
+'[?]',    # 0xa2
+'[?]',    # 0xa3
+'[?]',    # 0xa4
+'[?]',    # 0xa5
+'[?]',    # 0xa6
+'[?]',    # 0xa7
+'[?]',    # 0xa8
+'[?]',    # 0xa9
+'[?]',    # 0xaa
+'[?]',    # 0xab
+'[?]',    # 0xac
+'[?]',    # 0xad
+'[?]',    # 0xae
+'[?]',    # 0xaf
+'[?]',    # 0xb0
+'[?]',    # 0xb1
+'[?]',    # 0xb2
+'[?]',    # 0xb3
+'[?]',    # 0xb4
+'[?]',    # 0xb5
+'[?]',    # 0xb6
+'[?]',    # 0xb7
+'[?]',    # 0xb8
+'[?]',    # 0xb9
+'[?]',    # 0xba
+'[?]',    # 0xbb
+'[?]',    # 0xbc
+'[?]',    # 0xbd
+'[?]',    # 0xbe
+'[?]',    # 0xbf
+'[?]',    # 0xc0
+'[?]',    # 0xc1
+'[?]',    # 0xc2
+'[?]',    # 0xc3
+'[?]',    # 0xc4
+'[?]',    # 0xc5
+'[?]',    # 0xc6
+'[?]',    # 0xc7
+'[?]',    # 0xc8
+'[?]',    # 0xc9
+'[?]',    # 0xca
+'[?]',    # 0xcb
+'[?]',    # 0xcc
+'[?]',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'[?]',    # 0xd0
+'[?]',    # 0xd1
+'[?]',    # 0xd2
+'[?]',    # 0xd3
+'[?]',    # 0xd4
+'[?]',    # 0xd5
+'[?]',    # 0xd6
+'[?]',    # 0xd7
+'[?]',    # 0xd8
+'[?]',    # 0xd9
+'[?]',    # 0xda
+'[?]',    # 0xdb
+'[?]',    # 0xdc
+'[?]',    # 0xdd
+'[?]',    # 0xde
+'[?]',    # 0xdf
+'[?]',    # 0xe0
+'[?]',    # 0xe1
+'[?]',    # 0xe2
+'[?]',    # 0xe3
+'[?]',    # 0xe4
+'[?]',    # 0xe5
+'[?]',    # 0xe6
+'[?]',    # 0xe7
+'[?]',    # 0xe8
+'[?]',    # 0xe9
+'[?]',    # 0xea
+'[?]',    # 0xeb
+'[?]',    # 0xec
+'[?]',    # 0xed
+'[?]',    # 0xee
+'[?]',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'[?]',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x024.py

@@ -0,0 +1,257 @@
+data = (
+'',    # 0x00
+'',    # 0x01
+'',    # 0x02
+'',    # 0x03
+'',    # 0x04
+'',    # 0x05
+'',    # 0x06
+'',    # 0x07
+'',    # 0x08
+'',    # 0x09
+'',    # 0x0a
+'',    # 0x0b
+'',    # 0x0c
+'',    # 0x0d
+'',    # 0x0e
+'',    # 0x0f
+'',    # 0x10
+'',    # 0x11
+'',    # 0x12
+'',    # 0x13
+'',    # 0x14
+'',    # 0x15
+'',    # 0x16
+'',    # 0x17
+'',    # 0x18
+'',    # 0x19
+'',    # 0x1a
+'',    # 0x1b
+'',    # 0x1c
+'',    # 0x1d
+'',    # 0x1e
+'',    # 0x1f
+'',    # 0x20
+'',    # 0x21
+'',    # 0x22
+'',    # 0x23
+'',    # 0x24
+'',    # 0x25
+'',    # 0x26
+'[?]',    # 0x27
+'[?]',    # 0x28
+'[?]',    # 0x29
+'[?]',    # 0x2a
+'[?]',    # 0x2b
+'[?]',    # 0x2c
+'[?]',    # 0x2d
+'[?]',    # 0x2e
+'[?]',    # 0x2f
+'[?]',    # 0x30
+'[?]',    # 0x31
+'[?]',    # 0x32
+'[?]',    # 0x33
+'[?]',    # 0x34
+'[?]',    # 0x35
+'[?]',    # 0x36
+'[?]',    # 0x37
+'[?]',    # 0x38
+'[?]',    # 0x39
+'[?]',    # 0x3a
+'[?]',    # 0x3b
+'[?]',    # 0x3c
+'[?]',    # 0x3d
+'[?]',    # 0x3e
+'[?]',    # 0x3f
+'',    # 0x40
+'',    # 0x41
+'',    # 0x42
+'',    # 0x43
+'',    # 0x44
+'',    # 0x45
+'',    # 0x46
+'',    # 0x47
+'',    # 0x48
+'',    # 0x49
+'',    # 0x4a
+'[?]',    # 0x4b
+'[?]',    # 0x4c
+'[?]',    # 0x4d
+'[?]',    # 0x4e
+'[?]',    # 0x4f
+'[?]',    # 0x50
+'[?]',    # 0x51
+'[?]',    # 0x52
+'[?]',    # 0x53
+'[?]',    # 0x54
+'[?]',    # 0x55
+'[?]',    # 0x56
+'[?]',    # 0x57
+'[?]',    # 0x58
+'[?]',    # 0x59
+'[?]',    # 0x5a
+'[?]',    # 0x5b
+'[?]',    # 0x5c
+'[?]',    # 0x5d
+'[?]',    # 0x5e
+'[?]',    # 0x5f
+'1',    # 0x60
+'2',    # 0x61
+'3',    # 0x62
+'4',    # 0x63
+'5',    # 0x64
+'6',    # 0x65
+'7',    # 0x66
+'8',    # 0x67
+'9',    # 0x68
+'10',    # 0x69
+'11',    # 0x6a
+'12',    # 0x6b
+'13',    # 0x6c
+'14',    # 0x6d
+'15',    # 0x6e
+'16',    # 0x6f
+'17',    # 0x70
+'18',    # 0x71
+'19',    # 0x72
+'20',    # 0x73
+'(1)',    # 0x74
+'(2)',    # 0x75
+'(3)',    # 0x76
+'(4)',    # 0x77
+'(5)',    # 0x78
+'(6)',    # 0x79
+'(7)',    # 0x7a
+'(8)',    # 0x7b
+'(9)',    # 0x7c
+'(10)',    # 0x7d
+'(11)',    # 0x7e
+'(12)',    # 0x7f
+'(13)',    # 0x80
+'(14)',    # 0x81
+'(15)',    # 0x82
+'(16)',    # 0x83
+'(17)',    # 0x84
+'(18)',    # 0x85
+'(19)',    # 0x86
+'(20)',    # 0x87
+'1.',    # 0x88
+'2.',    # 0x89
+'3.',    # 0x8a
+'4.',    # 0x8b
+'5.',    # 0x8c
+'6.',    # 0x8d
+'7.',    # 0x8e
+'8.',    # 0x8f
+'9.',    # 0x90
+'10.',    # 0x91
+'11.',    # 0x92
+'12.',    # 0x93
+'13.',    # 0x94
+'14.',    # 0x95
+'15.',    # 0x96
+'16.',    # 0x97
+'17.',    # 0x98
+'18.',    # 0x99
+'19.',    # 0x9a
+'20.',    # 0x9b
+'(a)',    # 0x9c
+'(b)',    # 0x9d
+'(c)',    # 0x9e
+'(d)',    # 0x9f
+'(e)',    # 0xa0
+'(f)',    # 0xa1
+'(g)',    # 0xa2
+'(h)',    # 0xa3
+'(i)',    # 0xa4
+'(j)',    # 0xa5
+'(k)',    # 0xa6
+'(l)',    # 0xa7
+'(m)',    # 0xa8
+'(n)',    # 0xa9
+'(o)',    # 0xaa
+'(p)',    # 0xab
+'(q)',    # 0xac
+'(r)',    # 0xad
+'(s)',    # 0xae
+'(t)',    # 0xaf
+'(u)',    # 0xb0
+'(v)',    # 0xb1
+'(w)',    # 0xb2
+'(x)',    # 0xb3
+'(y)',    # 0xb4
+'(z)',    # 0xb5
+'a',    # 0xb6
+'b',    # 0xb7
+'c',    # 0xb8
+'d',    # 0xb9
+'e',    # 0xba
+'f',    # 0xbb
+'g',    # 0xbc
+'h',    # 0xbd
+'i',    # 0xbe
+'j',    # 0xbf
+'k',    # 0xc0
+'l',    # 0xc1
+'m',    # 0xc2
+'n',    # 0xc3
+'o',    # 0xc4
+'p',    # 0xc5
+'q',    # 0xc6
+'r',    # 0xc7
+'s',    # 0xc8
+'t',    # 0xc9
+'u',    # 0xca
+'v',    # 0xcb
+'w',    # 0xcc
+'x',    # 0xcd
+'y',    # 0xce
+'z',    # 0xcf
+'a',    # 0xd0
+'b',    # 0xd1
+'c',    # 0xd2
+'d',    # 0xd3
+'e',    # 0xd4
+'f',    # 0xd5
+'g',    # 0xd6
+'h',    # 0xd7
+'i',    # 0xd8
+'j',    # 0xd9
+'k',    # 0xda
+'l',    # 0xdb
+'m',    # 0xdc
+'n',    # 0xdd
+'o',    # 0xde
+'p',    # 0xdf
+'q',    # 0xe0
+'r',    # 0xe1
+'s',    # 0xe2
+'t',    # 0xe3
+'u',    # 0xe4
+'v',    # 0xe5
+'w',    # 0xe6
+'x',    # 0xe7
+'y',    # 0xe8
+'z',    # 0xe9
+'0',    # 0xea
+'[?]',    # 0xeb
+'[?]',    # 0xec
+'[?]',    # 0xed
+'[?]',    # 0xee
+'[?]',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'[?]',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x025.py

@@ -0,0 +1,257 @@
+data = (
+'-',    # 0x00
+'-',    # 0x01
+'|',    # 0x02
+'|',    # 0x03
+'-',    # 0x04
+'-',    # 0x05
+'|',    # 0x06
+'|',    # 0x07
+'-',    # 0x08
+'-',    # 0x09
+'|',    # 0x0a
+'|',    # 0x0b
+'+',    # 0x0c
+'+',    # 0x0d
+'+',    # 0x0e
+'+',    # 0x0f
+'+',    # 0x10
+'+',    # 0x11
+'+',    # 0x12
+'+',    # 0x13
+'+',    # 0x14
+'+',    # 0x15
+'+',    # 0x16
+'+',    # 0x17
+'+',    # 0x18
+'+',    # 0x19
+'+',    # 0x1a
+'+',    # 0x1b
+'+',    # 0x1c
+'+',    # 0x1d
+'+',    # 0x1e
+'+',    # 0x1f
+'+',    # 0x20
+'+',    # 0x21
+'+',    # 0x22
+'+',    # 0x23
+'+',    # 0x24
+'+',    # 0x25
+'+',    # 0x26
+'+',    # 0x27
+'+',    # 0x28
+'+',    # 0x29
+'+',    # 0x2a
+'+',    # 0x2b
+'+',    # 0x2c
+'+',    # 0x2d
+'+',    # 0x2e
+'+',    # 0x2f
+'+',    # 0x30
+'+',    # 0x31
+'+',    # 0x32
+'+',    # 0x33
+'+',    # 0x34
+'+',    # 0x35
+'+',    # 0x36
+'+',    # 0x37
+'+',    # 0x38
+'+',    # 0x39
+'+',    # 0x3a
+'+',    # 0x3b
+'+',    # 0x3c
+'+',    # 0x3d
+'+',    # 0x3e
+'+',    # 0x3f
+'+',    # 0x40
+'+',    # 0x41
+'+',    # 0x42
+'+',    # 0x43
+'+',    # 0x44
+'+',    # 0x45
+'+',    # 0x46
+'+',    # 0x47
+'+',    # 0x48
+'+',    # 0x49
+'+',    # 0x4a
+'+',    # 0x4b
+'-',    # 0x4c
+'-',    # 0x4d
+'|',    # 0x4e
+'|',    # 0x4f
+'-',    # 0x50
+'|',    # 0x51
+'+',    # 0x52
+'+',    # 0x53
+'+',    # 0x54
+'+',    # 0x55
+'+',    # 0x56
+'+',    # 0x57
+'+',    # 0x58
+'+',    # 0x59
+'+',    # 0x5a
+'+',    # 0x5b
+'+',    # 0x5c
+'+',    # 0x5d
+'+',    # 0x5e
+'+',    # 0x5f
+'+',    # 0x60
+'+',    # 0x61
+'+',    # 0x62
+'+',    # 0x63
+'+',    # 0x64
+'+',    # 0x65
+'+',    # 0x66
+'+',    # 0x67
+'+',    # 0x68
+'+',    # 0x69
+'+',    # 0x6a
+'+',    # 0x6b
+'+',    # 0x6c
+'+',    # 0x6d
+'+',    # 0x6e
+'+',    # 0x6f
+'+',    # 0x70
+'/',    # 0x71
+'\\',    # 0x72
+'X',    # 0x73
+'-',    # 0x74
+'|',    # 0x75
+'-',    # 0x76
+'|',    # 0x77
+'-',    # 0x78
+'|',    # 0x79
+'-',    # 0x7a
+'|',    # 0x7b
+'-',    # 0x7c
+'|',    # 0x7d
+'-',    # 0x7e
+'|',    # 0x7f
+'#',    # 0x80
+'#',    # 0x81
+'#',    # 0x82
+'#',    # 0x83
+'#',    # 0x84
+'#',    # 0x85
+'#',    # 0x86
+'#',    # 0x87
+'#',    # 0x88
+'#',    # 0x89
+'#',    # 0x8a
+'#',    # 0x8b
+'#',    # 0x8c
+'#',    # 0x8d
+'#',    # 0x8e
+'#',    # 0x8f
+'#',    # 0x90
+'#',    # 0x91
+'#',    # 0x92
+'#',    # 0x93
+'-',    # 0x94
+'|',    # 0x95
+'[?]',    # 0x96
+'[?]',    # 0x97
+'[?]',    # 0x98
+'[?]',    # 0x99
+'[?]',    # 0x9a
+'[?]',    # 0x9b
+'[?]',    # 0x9c
+'[?]',    # 0x9d
+'[?]',    # 0x9e
+'[?]',    # 0x9f
+'#',    # 0xa0
+'#',    # 0xa1
+'#',    # 0xa2
+'#',    # 0xa3
+'#',    # 0xa4
+'#',    # 0xa5
+'#',    # 0xa6
+'#',    # 0xa7
+'#',    # 0xa8
+'#',    # 0xa9
+'#',    # 0xaa
+'#',    # 0xab
+'#',    # 0xac
+'#',    # 0xad
+'#',    # 0xae
+'#',    # 0xaf
+'#',    # 0xb0
+'#',    # 0xb1
+'^',    # 0xb2
+'^',    # 0xb3
+'^',    # 0xb4
+'^',    # 0xb5
+'>',    # 0xb6
+'>',    # 0xb7
+'>',    # 0xb8
+'>',    # 0xb9
+'>',    # 0xba
+'>',    # 0xbb
+'V',    # 0xbc
+'V',    # 0xbd
+'V',    # 0xbe
+'V',    # 0xbf
+'<',    # 0xc0
+'<',    # 0xc1
+'<',    # 0xc2
+'<',    # 0xc3
+'<',    # 0xc4
+'<',    # 0xc5
+'*',    # 0xc6
+'*',    # 0xc7
+'*',    # 0xc8
+'*',    # 0xc9
+'*',    # 0xca
+'*',    # 0xcb
+'*',    # 0xcc
+'*',    # 0xcd
+'*',    # 0xce
+'*',    # 0xcf
+'*',    # 0xd0
+'*',    # 0xd1
+'*',    # 0xd2
+'*',    # 0xd3
+'*',    # 0xd4
+'*',    # 0xd5
+'*',    # 0xd6
+'*',    # 0xd7
+'*',    # 0xd8
+'*',    # 0xd9
+'*',    # 0xda
+'*',    # 0xdb
+'*',    # 0xdc
+'*',    # 0xdd
+'*',    # 0xde
+'*',    # 0xdf
+'*',    # 0xe0
+'*',    # 0xe1
+'*',    # 0xe2
+'*',    # 0xe3
+'*',    # 0xe4
+'*',    # 0xe5
+'*',    # 0xe6
+'#',    # 0xe7
+'#',    # 0xe8
+'#',    # 0xe9
+'#',    # 0xea
+'#',    # 0xeb
+'^',    # 0xec
+'^',    # 0xed
+'^',    # 0xee
+'O',    # 0xef
+'#',    # 0xf0
+'#',    # 0xf1
+'#',    # 0xf2
+'#',    # 0xf3
+'#',    # 0xf4
+'#',    # 0xf5
+'#',    # 0xf6
+'#',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x026.py

@@ -0,0 +1,257 @@
+data = (
+'',    # 0x00
+'',    # 0x01
+'',    # 0x02
+'',    # 0x03
+'',    # 0x04
+'',    # 0x05
+'',    # 0x06
+'',    # 0x07
+'',    # 0x08
+'',    # 0x09
+'',    # 0x0a
+'',    # 0x0b
+'',    # 0x0c
+'',    # 0x0d
+'',    # 0x0e
+'',    # 0x0f
+'',    # 0x10
+'',    # 0x11
+'',    # 0x12
+'',    # 0x13
+'[?]',    # 0x14
+'[?]',    # 0x15
+'[?]',    # 0x16
+'[?]',    # 0x17
+'[?]',    # 0x18
+'',    # 0x19
+'',    # 0x1a
+'',    # 0x1b
+'',    # 0x1c
+'',    # 0x1d
+'',    # 0x1e
+'',    # 0x1f
+'',    # 0x20
+'',    # 0x21
+'',    # 0x22
+'',    # 0x23
+'',    # 0x24
+'',    # 0x25
+'',    # 0x26
+'',    # 0x27
+'',    # 0x28
+'',    # 0x29
+'',    # 0x2a
+'',    # 0x2b
+'',    # 0x2c
+'',    # 0x2d
+'',    # 0x2e
+'',    # 0x2f
+'',    # 0x30
+'',    # 0x31
+'',    # 0x32
+'',    # 0x33
+'',    # 0x34
+'',    # 0x35
+'',    # 0x36
+'',    # 0x37
+'',    # 0x38
+'',    # 0x39
+'',    # 0x3a
+'',    # 0x3b
+'',    # 0x3c
+'',    # 0x3d
+'',    # 0x3e
+'',    # 0x3f
+'',    # 0x40
+'',    # 0x41
+'',    # 0x42
+'',    # 0x43
+'',    # 0x44
+'',    # 0x45
+'',    # 0x46
+'',    # 0x47
+'',    # 0x48
+'',    # 0x49
+'',    # 0x4a
+'',    # 0x4b
+'',    # 0x4c
+'',    # 0x4d
+'',    # 0x4e
+'',    # 0x4f
+'',    # 0x50
+'',    # 0x51
+'',    # 0x52
+'',    # 0x53
+'',    # 0x54
+'',    # 0x55
+'',    # 0x56
+'',    # 0x57
+'',    # 0x58
+'',    # 0x59
+'',    # 0x5a
+'',    # 0x5b
+'',    # 0x5c
+'',    # 0x5d
+'',    # 0x5e
+'',    # 0x5f
+'',    # 0x60
+'',    # 0x61
+'',    # 0x62
+'',    # 0x63
+'',    # 0x64
+'',    # 0x65
+'',    # 0x66
+'',    # 0x67
+'',    # 0x68
+'',    # 0x69
+'',    # 0x6a
+'',    # 0x6b
+'',    # 0x6c
+'',    # 0x6d
+'',    # 0x6e
+'#',    # 0x6f
+'',    # 0x70
+'',    # 0x71
+'[?]',    # 0x72
+'[?]',    # 0x73
+'[?]',    # 0x74
+'[?]',    # 0x75
+'[?]',    # 0x76
+'[?]',    # 0x77
+'[?]',    # 0x78
+'[?]',    # 0x79
+'[?]',    # 0x7a
+'[?]',    # 0x7b
+'[?]',    # 0x7c
+'[?]',    # 0x7d
+'[?]',    # 0x7e
+'[?]',    # 0x7f
+'[?]',    # 0x80
+'[?]',    # 0x81
+'[?]',    # 0x82
+'[?]',    # 0x83
+'[?]',    # 0x84
+'[?]',    # 0x85
+'[?]',    # 0x86
+'[?]',    # 0x87
+'[?]',    # 0x88
+'[?]',    # 0x89
+'[?]',    # 0x8a
+'[?]',    # 0x8b
+'[?]',    # 0x8c
+'[?]',    # 0x8d
+'[?]',    # 0x8e
+'[?]',    # 0x8f
+'[?]',    # 0x90
+'[?]',    # 0x91
+'[?]',    # 0x92
+'[?]',    # 0x93
+'[?]',    # 0x94
+'[?]',    # 0x95
+'[?]',    # 0x96
+'[?]',    # 0x97
+'[?]',    # 0x98
+'[?]',    # 0x99
+'[?]',    # 0x9a
+'[?]',    # 0x9b
+'[?]',    # 0x9c
+'[?]',    # 0x9d
+'[?]',    # 0x9e
+'[?]',    # 0x9f
+'[?]',    # 0xa0
+'[?]',    # 0xa1
+'[?]',    # 0xa2
+'[?]',    # 0xa3
+'[?]',    # 0xa4
+'[?]',    # 0xa5
+'[?]',    # 0xa6
+'[?]',    # 0xa7
+'[?]',    # 0xa8
+'[?]',    # 0xa9
+'[?]',    # 0xaa
+'[?]',    # 0xab
+'[?]',    # 0xac
+'[?]',    # 0xad
+'[?]',    # 0xae
+'[?]',    # 0xaf
+'[?]',    # 0xb0
+'[?]',    # 0xb1
+'[?]',    # 0xb2
+'[?]',    # 0xb3
+'[?]',    # 0xb4
+'[?]',    # 0xb5
+'[?]',    # 0xb6
+'[?]',    # 0xb7
+'[?]',    # 0xb8
+'[?]',    # 0xb9
+'[?]',    # 0xba
+'[?]',    # 0xbb
+'[?]',    # 0xbc
+'[?]',    # 0xbd
+'[?]',    # 0xbe
+'[?]',    # 0xbf
+'[?]',    # 0xc0
+'[?]',    # 0xc1
+'[?]',    # 0xc2
+'[?]',    # 0xc3
+'[?]',    # 0xc4
+'[?]',    # 0xc5
+'[?]',    # 0xc6
+'[?]',    # 0xc7
+'[?]',    # 0xc8
+'[?]',    # 0xc9
+'[?]',    # 0xca
+'[?]',    # 0xcb
+'[?]',    # 0xcc
+'[?]',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'[?]',    # 0xd0
+'[?]',    # 0xd1
+'[?]',    # 0xd2
+'[?]',    # 0xd3
+'[?]',    # 0xd4
+'[?]',    # 0xd5
+'[?]',    # 0xd6
+'[?]',    # 0xd7
+'[?]',    # 0xd8
+'[?]',    # 0xd9
+'[?]',    # 0xda
+'[?]',    # 0xdb
+'[?]',    # 0xdc
+'[?]',    # 0xdd
+'[?]',    # 0xde
+'[?]',    # 0xdf
+'[?]',    # 0xe0
+'[?]',    # 0xe1
+'[?]',    # 0xe2
+'[?]',    # 0xe3
+'[?]',    # 0xe4
+'[?]',    # 0xe5
+'[?]',    # 0xe6
+'[?]',    # 0xe7
+'[?]',    # 0xe8
+'[?]',    # 0xe9
+'[?]',    # 0xea
+'[?]',    # 0xeb
+'[?]',    # 0xec
+'[?]',    # 0xed
+'[?]',    # 0xee
+'[?]',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'[?]',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x027.py

@@ -0,0 +1,257 @@
+data = (
+'[?]',    # 0x00
+'',    # 0x01
+'',    # 0x02
+'',    # 0x03
+'',    # 0x04
+'',    # 0x05
+'',    # 0x06
+'',    # 0x07
+'',    # 0x08
+'',    # 0x09
+'',    # 0x0a
+'',    # 0x0b
+'',    # 0x0c
+'',    # 0x0d
+'',    # 0x0e
+'',    # 0x0f
+'',    # 0x10
+'',    # 0x11
+'',    # 0x12
+'',    # 0x13
+'',    # 0x14
+'',    # 0x15
+'',    # 0x16
+'',    # 0x17
+'',    # 0x18
+'',    # 0x19
+'',    # 0x1a
+'',    # 0x1b
+'',    # 0x1c
+'',    # 0x1d
+'',    # 0x1e
+'',    # 0x1f
+'',    # 0x20
+'',    # 0x21
+'',    # 0x22
+'',    # 0x23
+'',    # 0x24
+'',    # 0x25
+'',    # 0x26
+'',    # 0x27
+'',    # 0x28
+'',    # 0x29
+'',    # 0x2a
+'',    # 0x2b
+'',    # 0x2c
+'',    # 0x2d
+'',    # 0x2e
+'',    # 0x2f
+'',    # 0x30
+'*',    # 0x31
+'',    # 0x32
+'',    # 0x33
+'',    # 0x34
+'',    # 0x35
+'',    # 0x36
+'',    # 0x37
+'',    # 0x38
+'',    # 0x39
+'',    # 0x3a
+'',    # 0x3b
+'',    # 0x3c
+'',    # 0x3d
+'',    # 0x3e
+'',    # 0x3f
+'',    # 0x40
+'',    # 0x41
+'',    # 0x42
+'',    # 0x43
+'',    # 0x44
+'',    # 0x45
+'',    # 0x46
+'',    # 0x47
+'',    # 0x48
+'',    # 0x49
+'',    # 0x4a
+'',    # 0x4b
+'',    # 0x4c
+'',    # 0x4d
+'',    # 0x4e
+'',    # 0x4f
+'',    # 0x50
+'',    # 0x51
+'',    # 0x52
+'',    # 0x53
+'',    # 0x54
+'',    # 0x55
+'',    # 0x56
+'',    # 0x57
+'|',    # 0x58
+'',    # 0x59
+'',    # 0x5a
+'',    # 0x5b
+'',    # 0x5c
+'',    # 0x5d
+'',    # 0x5e
+'[?]',    # 0x5f
+'[?]',    # 0x60
+'',    # 0x61
+'!',    # 0x62
+'',    # 0x63
+'',    # 0x64
+'',    # 0x65
+'',    # 0x66
+'',    # 0x67
+'',    # 0x68
+'',    # 0x69
+'',    # 0x6a
+'',    # 0x6b
+'',    # 0x6c
+'',    # 0x6d
+'',    # 0x6e
+'',    # 0x6f
+'',    # 0x70
+'',    # 0x71
+'',    # 0x72
+'',    # 0x73
+'',    # 0x74
+'',    # 0x75
+'',    # 0x76
+'',    # 0x77
+'',    # 0x78
+'',    # 0x79
+'',    # 0x7a
+'',    # 0x7b
+'',    # 0x7c
+'',    # 0x7d
+'',    # 0x7e
+'',    # 0x7f
+'',    # 0x80
+'',    # 0x81
+'',    # 0x82
+'',    # 0x83
+'',    # 0x84
+'',    # 0x85
+'',    # 0x86
+'',    # 0x87
+'',    # 0x88
+'',    # 0x89
+'',    # 0x8a
+'',    # 0x8b
+'',    # 0x8c
+'',    # 0x8d
+'',    # 0x8e
+'',    # 0x8f
+'',    # 0x90
+'',    # 0x91
+'',    # 0x92
+'',    # 0x93
+'',    # 0x94
+'',    # 0x95
+'',    # 0x96
+'',    # 0x97
+'',    # 0x98
+'',    # 0x99
+'',    # 0x9a
+'',    # 0x9b
+'',    # 0x9c
+'',    # 0x9d
+'',    # 0x9e
+'',    # 0x9f
+'',    # 0xa0
+'',    # 0xa1
+'',    # 0xa2
+'',    # 0xa3
+'',    # 0xa4
+'',    # 0xa5
+'',    # 0xa6
+'',    # 0xa7
+'',    # 0xa8
+'',    # 0xa9
+'',    # 0xaa
+'',    # 0xab
+'',    # 0xac
+'',    # 0xad
+'',    # 0xae
+'',    # 0xaf
+'[?]',    # 0xb0
+'',    # 0xb1
+'',    # 0xb2
+'',    # 0xb3
+'',    # 0xb4
+'',    # 0xb5
+'',    # 0xb6
+'',    # 0xb7
+'',    # 0xb8
+'',    # 0xb9
+'',    # 0xba
+'',    # 0xbb
+'',    # 0xbc
+'',    # 0xbd
+'',    # 0xbe
+'[?]',    # 0xbf
+'[?]',    # 0xc0
+'[?]',    # 0xc1
+'[?]',    # 0xc2
+'[?]',    # 0xc3
+'[?]',    # 0xc4
+'[?]',    # 0xc5
+'[?]',    # 0xc6
+'[?]',    # 0xc7
+'[?]',    # 0xc8
+'[?]',    # 0xc9
+'[?]',    # 0xca
+'[?]',    # 0xcb
+'[?]',    # 0xcc
+'[?]',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'[?]',    # 0xd0
+'[?]',    # 0xd1
+'[?]',    # 0xd2
+'[?]',    # 0xd3
+'[?]',    # 0xd4
+'[?]',    # 0xd5
+'[?]',    # 0xd6
+'[?]',    # 0xd7
+'[?]',    # 0xd8
+'[?]',    # 0xd9
+'[?]',    # 0xda
+'[?]',    # 0xdb
+'[?]',    # 0xdc
+'[?]',    # 0xdd
+'[?]',    # 0xde
+'[?]',    # 0xdf
+'[?]',    # 0xe0
+'[?]',    # 0xe1
+'[?]',    # 0xe2
+'[?]',    # 0xe3
+'[?]',    # 0xe4
+'[?]',    # 0xe5
+'[',    # 0xe6
+'[?]',    # 0xe7
+'<',    # 0xe8
+'> ',    # 0xe9
+'[?]',    # 0xea
+'[?]',    # 0xeb
+'[?]',    # 0xec
+'[?]',    # 0xed
+'[?]',    # 0xee
+'[?]',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'[?]',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x028.py

@@ -0,0 +1,258 @@
+data = (
+' ',    # 0x00
+'a',    # 0x01
+'1',    # 0x02
+'b',    # 0x03
+'\'',    # 0x04
+'k',    # 0x05
+'2',    # 0x06
+'l',    # 0x07
+'@',    # 0x08
+'c',    # 0x09
+'i',    # 0x0a
+'f',    # 0x0b
+'/',    # 0x0c
+'m',    # 0x0d
+'s',    # 0x0e
+'p',    # 0x0f
+'"',    # 0x10
+'e',    # 0x11
+'3',    # 0x12
+'h',    # 0x13
+'9',    # 0x14
+'o',    # 0x15
+'6',    # 0x16
+'r',    # 0x17
+'^',    # 0x18
+'d',    # 0x19
+'j',    # 0x1a
+'g',    # 0x1b
+'>',    # 0x1c
+'n',    # 0x1d
+'t',    # 0x1e
+'q',    # 0x1f
+',',    # 0x20
+'*',    # 0x21
+'5',    # 0x22
+'<',    # 0x23
+'-',    # 0x24
+'u',    # 0x25
+'8',    # 0x26
+'v',    # 0x27
+'.',    # 0x28
+'%',    # 0x29
+'[',    # 0x2a
+'$',    # 0x2b
+'+',    # 0x2c
+'x',    # 0x2d
+'!',    # 0x2e
+'&',    # 0x2f
+';',    # 0x30
+':',    # 0x31
+'4',    # 0x32
+'\\',    # 0x33
+'0',    # 0x34
+'z',    # 0x35
+'7',    # 0x36
+'(',    # 0x37
+'_',    # 0x38
+'?',    # 0x39
+'w',    # 0x3a
+']',    # 0x3b
+'#',    # 0x3c
+'y',    # 0x3d
+')',    # 0x3e
+'=',    # 0x3f
+'[d7]',    # 0x40
+'[d17]',    # 0x41
+'[d27]',    # 0x42
+'[d127]',    # 0x43
+'[d37]',    # 0x44
+'[d137]',    # 0x45
+'[d237]',    # 0x46
+'[d1237]',    # 0x47
+'[d47]',    # 0x48
+'[d147]',    # 0x49
+'[d247]',    # 0x4a
+'[d1247]',    # 0x4b
+'[d347]',    # 0x4c
+'[d1347]',    # 0x4d
+'[d2347]',    # 0x4e
+'[d12347]',    # 0x4f
+'[d57]',    # 0x50
+'[d157]',    # 0x51
+'[d257]',    # 0x52
+'[d1257]',    # 0x53
+'[d357]',    # 0x54
+'[d1357]',    # 0x55
+'[d2357]',    # 0x56
+'[d12357]',    # 0x57
+'[d457]',    # 0x58
+'[d1457]',    # 0x59
+'[d2457]',    # 0x5a
+'[d12457]',    # 0x5b
+'[d3457]',    # 0x5c
+'[d13457]',    # 0x5d
+'[d23457]',    # 0x5e
+'[d123457]',    # 0x5f
+'[d67]',    # 0x60
+'[d167]',    # 0x61
+'[d267]',    # 0x62
+'[d1267]',    # 0x63
+'[d367]',    # 0x64
+'[d1367]',    # 0x65
+'[d2367]',    # 0x66
+'[d12367]',    # 0x67
+'[d467]',    # 0x68
+'[d1467]',    # 0x69
+'[d2467]',    # 0x6a
+'[d12467]',    # 0x6b
+'[d3467]',    # 0x6c
+'[d13467]',    # 0x6d
+'[d23467]',    # 0x6e
+'[d123467]',    # 0x6f
+'[d567]',    # 0x70
+'[d1567]',    # 0x71
+'[d2567]',    # 0x72
+'[d12567]',    # 0x73
+'[d3567]',    # 0x74
+'[d13567]',    # 0x75
+'[d23567]',    # 0x76
+'[d123567]',    # 0x77
+'[d4567]',    # 0x78
+'[d14567]',    # 0x79
+'[d24567]',    # 0x7a
+'[d124567]',    # 0x7b
+'[d34567]',    # 0x7c
+'[d134567]',    # 0x7d
+'[d234567]',    # 0x7e
+'[d1234567]',    # 0x7f
+'[d8]',    # 0x80
+'[d18]',    # 0x81
+'[d28]',    # 0x82
+'[d128]',    # 0x83
+'[d38]',    # 0x84
+'[d138]',    # 0x85
+'[d238]',    # 0x86
+'[d1238]',    # 0x87
+'[d48]',    # 0x88
+'[d148]',    # 0x89
+'[d248]',    # 0x8a
+'[d1248]',    # 0x8b
+'[d348]',    # 0x8c
+'[d1348]',    # 0x8d
+'[d2348]',    # 0x8e
+'[d12348]',    # 0x8f
+'[d58]',    # 0x90
+'[d158]',    # 0x91
+'[d258]',    # 0x92
+'[d1258]',    # 0x93
+'[d358]',    # 0x94
+'[d1358]',    # 0x95
+'[d2358]',    # 0x96
+'[d12358]',    # 0x97
+'[d458]',    # 0x98
+'[d1458]',    # 0x99
+'[d2458]',    # 0x9a
+'[d12458]',    # 0x9b
+'[d3458]',    # 0x9c
+'[d13458]',    # 0x9d
+'[d23458]',    # 0x9e
+'[d123458]',    # 0x9f
+'[d68]',    # 0xa0
+'[d168]',    # 0xa1
+'[d268]',    # 0xa2
+'[d1268]',    # 0xa3
+'[d368]',    # 0xa4
+'[d1368]',    # 0xa5
+'[d2368]',    # 0xa6
+'[d12368]',    # 0xa7
+'[d468]',    # 0xa8
+'[d1468]',    # 0xa9
+'[d2468]',    # 0xaa
+'[d12468]',    # 0xab
+'[d3468]',    # 0xac
+'[d13468]',    # 0xad
+'[d23468]',    # 0xae
+'[d123468]',    # 0xaf
+'[d568]',    # 0xb0
+'[d1568]',    # 0xb1
+'[d2568]',    # 0xb2
+'[d12568]',    # 0xb3
+'[d3568]',    # 0xb4
+'[d13568]',    # 0xb5
+'[d23568]',    # 0xb6
+'[d123568]',    # 0xb7
+'[d4568]',    # 0xb8
+'[d14568]',    # 0xb9
+'[d24568]',    # 0xba
+'[d124568]',    # 0xbb
+'[d34568]',    # 0xbc
+'[d134568]',    # 0xbd
+'[d234568]',    # 0xbe
+'[d1234568]',    # 0xbf
+'[d78]',    # 0xc0
+'[d178]',    # 0xc1
+'[d278]',    # 0xc2
+'[d1278]',    # 0xc3
+'[d378]',    # 0xc4
+'[d1378]',    # 0xc5
+'[d2378]',    # 0xc6
+'[d12378]',    # 0xc7
+'[d478]',    # 0xc8
+'[d1478]',    # 0xc9
+'[d2478]',    # 0xca
+'[d12478]',    # 0xcb
+'[d3478]',    # 0xcc
+'[d13478]',    # 0xcd
+'[d23478]',    # 0xce
+'[d123478]',    # 0xcf
+'[d578]',    # 0xd0
+'[d1578]',    # 0xd1
+'[d2578]',    # 0xd2
+'[d12578]',    # 0xd3
+'[d3578]',    # 0xd4
+'[d13578]',    # 0xd5
+'[d23578]',    # 0xd6
+'[d123578]',    # 0xd7
+'[d4578]',    # 0xd8
+'[d14578]',    # 0xd9
+'[d24578]',    # 0xda
+'[d124578]',    # 0xdb
+'[d34578]',    # 0xdc
+'[d134578]',    # 0xdd
+'[d234578]',    # 0xde
+'[d1234578]',    # 0xdf
+'[d678]',    # 0xe0
+'[d1678]',    # 0xe1
+'[d2678]',    # 0xe2
+'[d12678]',    # 0xe3
+'[d3678]',    # 0xe4
+'[d13678]',    # 0xe5
+'[d23678]',    # 0xe6
+'[d123678]',    # 0xe7
+'[d4678]',    # 0xe8
+'[d14678]',    # 0xe9
+'[d24678]',    # 0xea
+'[d124678]',    # 0xeb
+'[d34678]',    # 0xec
+'[d134678]',    # 0xed
+'[d234678]',    # 0xee
+'[d1234678]',    # 0xef
+'[d5678]',    # 0xf0
+'[d15678]',    # 0xf1
+'[d25678]',    # 0xf2
+'[d125678]',    # 0xf3
+'[d35678]',    # 0xf4
+'[d135678]',    # 0xf5
+'[d235678]',    # 0xf6
+'[d1235678]',    # 0xf7
+'[d45678]',    # 0xf8
+'[d145678]',    # 0xf9
+'[d245678]',    # 0xfa
+'[d1245678]',    # 0xfb
+'[d345678]',    # 0xfc
+'[d1345678]',    # 0xfd
+'[d2345678]',    # 0xfe
+'[d12345678]',    # 0xff
+)

lib/unidecode/x029.py

@@ -0,0 +1,257 @@
+data = (
+'',    # 0x00
+'',    # 0x01
+'',    # 0x02
+'',    # 0x03
+'',    # 0x04
+'',    # 0x05
+'',    # 0x06
+'',    # 0x07
+'',    # 0x08
+'',    # 0x09
+'',    # 0x0a
+'',    # 0x0b
+'',    # 0x0c
+'',    # 0x0d
+'',    # 0x0e
+'',    # 0x0f
+'',    # 0x10
+'',    # 0x11
+'',    # 0x12
+'',    # 0x13
+'',    # 0x14
+'',    # 0x15
+'',    # 0x16
+'',    # 0x17
+'',    # 0x18
+'',    # 0x19
+'',    # 0x1a
+'',    # 0x1b
+'',    # 0x1c
+'',    # 0x1d
+'',    # 0x1e
+'',    # 0x1f
+'',    # 0x20
+'',    # 0x21
+'',    # 0x22
+'',    # 0x23
+'',    # 0x24
+'',    # 0x25
+'',    # 0x26
+'',    # 0x27
+'',    # 0x28
+'',    # 0x29
+'',    # 0x2a
+'',    # 0x2b
+'',    # 0x2c
+'',    # 0x2d
+'',    # 0x2e
+'',    # 0x2f
+'',    # 0x30
+'',    # 0x31
+'',    # 0x32
+'',    # 0x33
+'',    # 0x34
+'',    # 0x35
+'',    # 0x36
+'',    # 0x37
+'',    # 0x38
+'',    # 0x39
+'',    # 0x3a
+'',    # 0x3b
+'',    # 0x3c
+'',    # 0x3d
+'',    # 0x3e
+'',    # 0x3f
+'',    # 0x40
+'',    # 0x41
+'',    # 0x42
+'',    # 0x43
+'',    # 0x44
+'',    # 0x45
+'',    # 0x46
+'',    # 0x47
+'',    # 0x48
+'',    # 0x49
+'',    # 0x4a
+'',    # 0x4b
+'',    # 0x4c
+'',    # 0x4d
+'',    # 0x4e
+'',    # 0x4f
+'',    # 0x50
+'',    # 0x51
+'',    # 0x52
+'',    # 0x53
+'',    # 0x54
+'',    # 0x55
+'',    # 0x56
+'',    # 0x57
+'',    # 0x58
+'',    # 0x59
+'',    # 0x5a
+'',    # 0x5b
+'',    # 0x5c
+'',    # 0x5d
+'',    # 0x5e
+'',    # 0x5f
+'',    # 0x60
+'',    # 0x61
+'',    # 0x62
+'',    # 0x63
+'',    # 0x64
+'',    # 0x65
+'',    # 0x66
+'',    # 0x67
+'',    # 0x68
+'',    # 0x69
+'',    # 0x6a
+'',    # 0x6b
+'',    # 0x6c
+'',    # 0x6d
+'',    # 0x6e
+'',    # 0x6f
+'',    # 0x70
+'',    # 0x71
+'',    # 0x72
+'',    # 0x73
+'',    # 0x74
+'',    # 0x75
+'',    # 0x76
+'',    # 0x77
+'',    # 0x78
+'',    # 0x79
+'',    # 0x7a
+'',    # 0x7b
+'',    # 0x7c
+'',    # 0x7d
+'',    # 0x7e
+'',    # 0x7f
+'',    # 0x80
+'',    # 0x81
+'',    # 0x82
+'{',    # 0x83
+'} ',    # 0x84
+'',    # 0x85
+'',    # 0x86
+'',    # 0x87
+'',    # 0x88
+'',    # 0x89
+'',    # 0x8a
+'',    # 0x8b
+'',    # 0x8c
+'',    # 0x8d
+'',    # 0x8e
+'',    # 0x8f
+'',    # 0x90
+'',    # 0x91
+'',    # 0x92
+'',    # 0x93
+'',    # 0x94
+'',    # 0x95
+'',    # 0x96
+'',    # 0x97
+'',    # 0x98
+'',    # 0x99
+'',    # 0x9a
+'',    # 0x9b
+'',    # 0x9c
+'',    # 0x9d
+'',    # 0x9e
+'',    # 0x9f
+'',    # 0xa0
+'',    # 0xa1
+'',    # 0xa2
+'',    # 0xa3
+'',    # 0xa4
+'',    # 0xa5
+'',    # 0xa6
+'',    # 0xa7
+'',    # 0xa8
+'',    # 0xa9
+'',    # 0xaa
+'',    # 0xab
+'',    # 0xac
+'',    # 0xad
+'',    # 0xae
+'',    # 0xaf
+'',    # 0xb0
+'',    # 0xb1
+'',    # 0xb2
+'',    # 0xb3
+'',    # 0xb4
+'',    # 0xb5
+'',    # 0xb6
+'',    # 0xb7
+'',    # 0xb8
+'',    # 0xb9
+'',    # 0xba
+'',    # 0xbb
+'',    # 0xbc
+'',    # 0xbd
+'',    # 0xbe
+'',    # 0xbf
+'',    # 0xc0
+'',    # 0xc1
+'',    # 0xc2
+'',    # 0xc3
+'',    # 0xc4
+'',    # 0xc5
+'',    # 0xc6
+'',    # 0xc7
+'',    # 0xc8
+'',    # 0xc9
+'',    # 0xca
+'',    # 0xcb
+'',    # 0xcc
+'',    # 0xcd
+'',    # 0xce
+'',    # 0xcf
+'',    # 0xd0
+'',    # 0xd1
+'',    # 0xd2
+'',    # 0xd3
+'',    # 0xd4
+'',    # 0xd5
+'',    # 0xd6
+'',    # 0xd7
+'',    # 0xd8
+'',    # 0xd9
+'',    # 0xda
+'',    # 0xdb
+'',    # 0xdc
+'',    # 0xdd
+'',    # 0xde
+'',    # 0xdf
+'',    # 0xe0
+'',    # 0xe1
+'',    # 0xe2
+'',    # 0xe3
+'',    # 0xe4
+'',    # 0xe5
+'',    # 0xe6
+'',    # 0xe7
+'',    # 0xe8
+'',    # 0xe9
+'',    # 0xea
+'',    # 0xeb
+'',    # 0xec
+'',    # 0xed
+'',    # 0xee
+'',    # 0xef
+'',    # 0xf0
+'',    # 0xf1
+'',    # 0xf2
+'',    # 0xf3
+'',    # 0xf4
+'',    # 0xf5
+'',    # 0xf6
+'',    # 0xf7
+'',    # 0xf8
+'',    # 0xf9
+'',    # 0xfa
+'',    # 0xfb
+'',    # 0xfc
+'',    # 0xfd
+'',    # 0xfe
+)

lib/unidecode/x02a.py

@@ -0,0 +1,257 @@
+data = (
+'',    # 0x00
+'',    # 0x01
+'',    # 0x02
+'',    # 0x03
+'',    # 0x04
+'',    # 0x05
+'',    # 0x06
+'',    # 0x07
+'',    # 0x08
+'',    # 0x09
+'',    # 0x0a
+'',    # 0x0b
+'',    # 0x0c
+'',    # 0x0d
+'',    # 0x0e
+'',    # 0x0f
+'',    # 0x10
+'',    # 0x11
+'',    # 0x12
+'',    # 0x13
+'',    # 0x14
+'',    # 0x15
+'',    # 0x16
+'',    # 0x17
+'',    # 0x18
+'',    # 0x19
+'',    # 0x1a
+'',    # 0x1b
+'',    # 0x1c
+'',    # 0x1d
+'',    # 0x1e
+'',    # 0x1f
+'',    # 0x20
+'',    # 0x21
+'',    # 0x22
+'',    # 0x23
+'',    # 0x24
+'',    # 0x25
+'',    # 0x26
+'',    # 0x27
+'',    # 0x28
+'',    # 0x29
+'',    # 0x2a
+'',    # 0x2b
+'',    # 0x2c
+'',    # 0x2d
+'',    # 0x2e
+'',    # 0x2f
+'',    # 0x30
+'',    # 0x31
+'',    # 0x32
+'',    # 0x33
+'',    # 0x34
+'',    # 0x35
+'',    # 0x36
+'',    # 0x37
+'',    # 0x38
+'',    # 0x39
+'',    # 0x3a
+'',    # 0x3b
+'',    # 0x3c
+'',    # 0x3d
+'',    # 0x3e
+'',    # 0x3f
+'',    # 0x40
+'',    # 0x41
+'',    # 0x42
+'',    # 0x43
+'',    # 0x44
+'',    # 0x45
+'',    # 0x46
+'',    # 0x47
+'',    # 0x48
+'',    # 0x49
+'',    # 0x4a
+'',    # 0x4b
+'',    # 0x4c
+'',    # 0x4d
+'',    # 0x4e
+'',    # 0x4f
+'',    # 0x50
+'',    # 0x51
+'',    # 0x52
+'',    # 0x53
+'',    # 0x54
+'',    # 0x55
+'',    # 0x56
+'',    # 0x57
+'',    # 0x58
+'',    # 0x59
+'',    # 0x5a
+'',    # 0x5b
+'',    # 0x5c
+'',    # 0x5d
+'',    # 0x5e
+'',    # 0x5f
+'',    # 0x60
+'',    # 0x61
+'',    # 0x62
+'',    # 0x63
+'',    # 0x64
+'',    # 0x65
+'',    # 0x66
+'',    # 0x67
+'',    # 0x68
+'',    # 0x69
+'',    # 0x6a
+'',    # 0x6b
+'',    # 0x6c
+'',    # 0x6d
+'',    # 0x6e
+'',    # 0x6f
+'',    # 0x70
+'',    # 0x71
+'',    # 0x72
+'',    # 0x73
+'::=',    # 0x74
+'==',    # 0x75
+'===',    # 0x76
+'',    # 0x77
+'',    # 0x78
+'',    # 0x79
+'',    # 0x7a
+'',    # 0x7b
+'',    # 0x7c
+'',    # 0x7d
+'',    # 0x7e
+'',    # 0x7f
+'',    # 0x80
+'',    # 0x81
+'',    # 0x82
+'',    # 0x83
+'',    # 0x84
+'',    # 0x85
+'',    # 0x86
+'',    # 0x87
+'',    # 0x88
+'',    # 0x89
+'',    # 0x8a
+'',    # 0x8b
+'',    # 0x8c
+'',    # 0x8d
+'',    # 0x8e
+'',    # 0x8f
+'',    # 0x90
+'',    # 0x91
+'',    # 0x92
+'',    # 0x93
+'',    # 0x94
+'',    # 0x95
+'',    # 0x96
+'',    # 0x97
+'',    # 0x98
+'',    # 0x99
+'',    # 0x9a
+'',    # 0x9b
+'',    # 0x9c
+'',    # 0x9d
+'',    # 0x9e
+'',    # 0x9f
+'',    # 0xa0
+'',    # 0xa1
+'',    # 0xa2
+'',    # 0xa3
+'',    # 0xa4
+'',    # 0xa5
+'',    # 0xa6
+'',    # 0xa7
+'',    # 0xa8
+'',    # 0xa9
+'',    # 0xaa
+'',    # 0xab
+'',    # 0xac
+'',    # 0xad
+'',    # 0xae
+'',    # 0xaf
+'',    # 0xb0
+'',    # 0xb1
+'',    # 0xb2
+'',    # 0xb3
+'',    # 0xb4
+'',    # 0xb5
+'',    # 0xb6
+'',    # 0xb7
+'',    # 0xb8
+'',    # 0xb9
+'',    # 0xba
+'',    # 0xbb
+'',    # 0xbc
+'',    # 0xbd
+'',    # 0xbe
+'',    # 0xbf
+'',    # 0xc0
+'',    # 0xc1
+'',    # 0xc2
+'',    # 0xc3
+'',    # 0xc4
+'',    # 0xc5
+'',    # 0xc6
+'',    # 0xc7
+'',    # 0xc8
+'',    # 0xc9
+'',    # 0xca
+'',    # 0xcb
+'',    # 0xcc
+'',    # 0xcd
+'',    # 0xce
+'',    # 0xcf
+'',    # 0xd0
+'',    # 0xd1
+'',    # 0xd2
+'',    # 0xd3
+'',    # 0xd4
+'',    # 0xd5
+'',    # 0xd6
+'',    # 0xd7
+'',    # 0xd8
+'',    # 0xd9
+'',    # 0xda
+'',    # 0xdb
+'',    # 0xdc
+'',    # 0xdd
+'',    # 0xde
+'',    # 0xdf
+'',    # 0xe0
+'',    # 0xe1
+'',    # 0xe2
+'',    # 0xe3
+'',    # 0xe4
+'',    # 0xe5
+'',    # 0xe6
+'',    # 0xe7
+'',    # 0xe8
+'',    # 0xe9
+'',    # 0xea
+'',    # 0xeb
+'',    # 0xec
+'',    # 0xed
+'',    # 0xee
+'',    # 0xef
+'',    # 0xf0
+'',    # 0xf1
+'',    # 0xf2
+'',    # 0xf3
+'',    # 0xf4
+'',    # 0xf5
+'',    # 0xf6
+'',    # 0xf7
+'',    # 0xf8
+'',    # 0xf9
+'',    # 0xfa
+'',    # 0xfb
+'',    # 0xfc
+'',    # 0xfd
+'',    # 0xfe
+)

lib/unidecode/x02c.py

@@ -0,0 +1,257 @@
+data = (
+'',    # 0x00
+'',    # 0x01
+'',    # 0x02
+'',    # 0x03
+'',    # 0x04
+'',    # 0x05
+'',    # 0x06
+'',    # 0x07
+'',    # 0x08
+'',    # 0x09
+'',    # 0x0a
+'',    # 0x0b
+'',    # 0x0c
+'',    # 0x0d
+'',    # 0x0e
+'',    # 0x0f
+'',    # 0x10
+'',    # 0x11
+'',    # 0x12
+'',    # 0x13
+'',    # 0x14
+'',    # 0x15
+'',    # 0x16
+'',    # 0x17
+'',    # 0x18
+'',    # 0x19
+'',    # 0x1a
+'',    # 0x1b
+'',    # 0x1c
+'',    # 0x1d
+'',    # 0x1e
+'',    # 0x1f
+'',    # 0x20
+'',    # 0x21
+'',    # 0x22
+'',    # 0x23
+'',    # 0x24
+'',    # 0x25
+'',    # 0x26
+'',    # 0x27
+'',    # 0x28
+'',    # 0x29
+'',    # 0x2a
+'',    # 0x2b
+'',    # 0x2c
+'',    # 0x2d
+'',    # 0x2e
+'',    # 0x2f
+'',    # 0x30
+'',    # 0x31
+'',    # 0x32
+'',    # 0x33
+'',    # 0x34
+'',    # 0x35
+'',    # 0x36
+'',    # 0x37
+'',    # 0x38
+'',    # 0x39
+'',    # 0x3a
+'',    # 0x3b
+'',    # 0x3c
+'',    # 0x3d
+'',    # 0x3e
+'',    # 0x3f
+'',    # 0x40
+'',    # 0x41
+'',    # 0x42
+'',    # 0x43
+'',    # 0x44
+'',    # 0x45
+'',    # 0x46
+'',    # 0x47
+'',    # 0x48
+'',    # 0x49
+'',    # 0x4a
+'',    # 0x4b
+'',    # 0x4c
+'',    # 0x4d
+'',    # 0x4e
+'',    # 0x4f
+'',    # 0x50
+'',    # 0x51
+'',    # 0x52
+'',    # 0x53
+'',    # 0x54
+'',    # 0x55
+'',    # 0x56
+'',    # 0x57
+'',    # 0x58
+'',    # 0x59
+'',    # 0x5a
+'',    # 0x5b
+'',    # 0x5c
+'',    # 0x5d
+'',    # 0x5e
+'',    # 0x5f
+'L',    # 0x60
+'l',    # 0x61
+'L',    # 0x62
+'P',    # 0x63
+'R',    # 0x64
+'a',    # 0x65
+'t',    # 0x66
+'H',    # 0x67
+'h',    # 0x68
+'K',    # 0x69
+'k',    # 0x6a
+'Z',    # 0x6b
+'z',    # 0x6c
+'',    # 0x6d
+'M',    # 0x6e
+'A',    # 0x6f
+'',    # 0x70
+'',    # 0x71
+'',    # 0x72
+'',    # 0x73
+'',    # 0x74
+'',    # 0x75
+'',    # 0x76
+'',    # 0x77
+'',    # 0x78
+'',    # 0x79
+'',    # 0x7a
+'',    # 0x7b
+'',    # 0x7c
+'',    # 0x7d
+'',    # 0x7e
+'',    # 0x7f
+'',    # 0x80
+'',    # 0x81
+'',    # 0x82
+'',    # 0x83
+'',    # 0x84
+'',    # 0x85
+'',    # 0x86
+'',    # 0x87
+'',    # 0x88
+'',    # 0x89
+'',    # 0x8a
+'',    # 0x8b
+'',    # 0x8c
+'',    # 0x8d
+'',    # 0x8e
+'',    # 0x8f
+'',    # 0x90
+'',    # 0x91
+'',    # 0x92
+'',    # 0x93
+'',    # 0x94
+'',    # 0x95
+'',    # 0x96
+'',    # 0x97
+'',    # 0x98
+'',    # 0x99
+'',    # 0x9a
+'',    # 0x9b
+'',    # 0x9c
+'',    # 0x9d
+'',    # 0x9e
+'',    # 0x9f
+'',    # 0xa0
+'',    # 0xa1
+'',    # 0xa2
+'',    # 0xa3
+'',    # 0xa4
+'',    # 0xa5
+'',    # 0xa6
+'',    # 0xa7
+'',    # 0xa8
+'',    # 0xa9
+'',    # 0xaa
+'',    # 0xab
+'',    # 0xac
+'',    # 0xad
+'',    # 0xae
+'',    # 0xaf
+'',    # 0xb0
+'',    # 0xb1
+'',    # 0xb2
+'',    # 0xb3
+'',    # 0xb4
+'',    # 0xb5
+'',    # 0xb6
+'',    # 0xb7
+'',    # 0xb8
+'',    # 0xb9
+'',    # 0xba
+'',    # 0xbb
+'',    # 0xbc
+'',    # 0xbd
+'',    # 0xbe
+'',    # 0xbf
+'',    # 0xc0
+'',    # 0xc1
+'',    # 0xc2
+'',    # 0xc3
+'',    # 0xc4
+'',    # 0xc5
+'',    # 0xc6
+'',    # 0xc7
+'',    # 0xc8
+'',    # 0xc9
+'',    # 0xca
+'',    # 0xcb
+'',    # 0xcc
+'',    # 0xcd
+'',    # 0xce
+'',    # 0xcf
+'',    # 0xd0
+'',    # 0xd1
+'',    # 0xd2
+'',    # 0xd3
+'',    # 0xd4
+'',    # 0xd5
+'',    # 0xd6
+'',    # 0xd7
+'',    # 0xd8
+'',    # 0xd9
+'',    # 0xda
+'',    # 0xdb
+'',    # 0xdc
+'',    # 0xdd
+'',    # 0xde
+'',    # 0xdf
+'',    # 0xe0
+'',    # 0xe1
+'',    # 0xe2
+'',    # 0xe3
+'',    # 0xe4
+'',    # 0xe5
+'',    # 0xe6
+'',    # 0xe7
+'',    # 0xe8
+'',    # 0xe9
+'',    # 0xea
+'',    # 0xeb
+'',    # 0xec
+'',    # 0xed
+'',    # 0xee
+'',    # 0xef
+'',    # 0xf0
+'',    # 0xf1
+'',    # 0xf2
+'',    # 0xf3
+'',    # 0xf4
+'',    # 0xf5
+'',    # 0xf6
+'',    # 0xf7
+'',    # 0xf8
+'',    # 0xf9
+'',    # 0xfa
+'',    # 0xfb
+'',    # 0xfc
+'',    # 0xfd
+'',    # 0xfe
+)

lib/unidecode/x02e.py

@@ -0,0 +1,257 @@
+data = (
+'[?]',    # 0x00
+'[?]',    # 0x01
+'[?]',    # 0x02
+'[?]',    # 0x03
+'[?]',    # 0x04
+'[?]',    # 0x05
+'[?]',    # 0x06
+'[?]',    # 0x07
+'[?]',    # 0x08
+'[?]',    # 0x09
+'[?]',    # 0x0a
+'[?]',    # 0x0b
+'[?]',    # 0x0c
+'[?]',    # 0x0d
+'[?]',    # 0x0e
+'[?]',    # 0x0f
+'[?]',    # 0x10
+'[?]',    # 0x11
+'[?]',    # 0x12
+'[?]',    # 0x13
+'[?]',    # 0x14
+'[?]',    # 0x15
+'[?]',    # 0x16
+'[?]',    # 0x17
+'[?]',    # 0x18
+'[?]',    # 0x19
+'[?]',    # 0x1a
+'[?]',    # 0x1b
+'[?]',    # 0x1c
+'[?]',    # 0x1d
+'[?]',    # 0x1e
+'[?]',    # 0x1f
+'[?]',    # 0x20
+'[?]',    # 0x21
+'[?]',    # 0x22
+'[?]',    # 0x23
+'[?]',    # 0x24
+'[?]',    # 0x25
+'[?]',    # 0x26
+'[?]',    # 0x27
+'[?]',    # 0x28
+'[?]',    # 0x29
+'[?]',    # 0x2a
+'[?]',    # 0x2b
+'[?]',    # 0x2c
+'[?]',    # 0x2d
+'[?]',    # 0x2e
+'[?]',    # 0x2f
+'[?]',    # 0x30
+'[?]',    # 0x31
+'[?]',    # 0x32
+'[?]',    # 0x33
+'[?]',    # 0x34
+'[?]',    # 0x35
+'[?]',    # 0x36
+'[?]',    # 0x37
+'[?]',    # 0x38
+'[?]',    # 0x39
+'[?]',    # 0x3a
+'[?]',    # 0x3b
+'[?]',    # 0x3c
+'[?]',    # 0x3d
+'[?]',    # 0x3e
+'[?]',    # 0x3f
+'[?]',    # 0x40
+'[?]',    # 0x41
+'[?]',    # 0x42
+'[?]',    # 0x43
+'[?]',    # 0x44
+'[?]',    # 0x45
+'[?]',    # 0x46
+'[?]',    # 0x47
+'[?]',    # 0x48
+'[?]',    # 0x49
+'[?]',    # 0x4a
+'[?]',    # 0x4b
+'[?]',    # 0x4c
+'[?]',    # 0x4d
+'[?]',    # 0x4e
+'[?]',    # 0x4f
+'[?]',    # 0x50
+'[?]',    # 0x51
+'[?]',    # 0x52
+'[?]',    # 0x53
+'[?]',    # 0x54
+'[?]',    # 0x55
+'[?]',    # 0x56
+'[?]',    # 0x57
+'[?]',    # 0x58
+'[?]',    # 0x59
+'[?]',    # 0x5a
+'[?]',    # 0x5b
+'[?]',    # 0x5c
+'[?]',    # 0x5d
+'[?]',    # 0x5e
+'[?]',    # 0x5f
+'[?]',    # 0x60
+'[?]',    # 0x61
+'[?]',    # 0x62
+'[?]',    # 0x63
+'[?]',    # 0x64
+'[?]',    # 0x65
+'[?]',    # 0x66
+'[?]',    # 0x67
+'[?]',    # 0x68
+'[?]',    # 0x69
+'[?]',    # 0x6a
+'[?]',    # 0x6b
+'[?]',    # 0x6c
+'[?]',    # 0x6d
+'[?]',    # 0x6e
+'[?]',    # 0x6f
+'[?]',    # 0x70
+'[?]',    # 0x71
+'[?]',    # 0x72
+'[?]',    # 0x73
+'[?]',    # 0x74
+'[?]',    # 0x75
+'[?]',    # 0x76
+'[?]',    # 0x77
+'[?]',    # 0x78
+'[?]',    # 0x79
+'[?]',    # 0x7a
+'[?]',    # 0x7b
+'[?]',    # 0x7c
+'[?]',    # 0x7d
+'[?]',    # 0x7e
+'[?]',    # 0x7f
+'[?] ',    # 0x80
+'[?] ',    # 0x81
+'[?] ',    # 0x82
+'[?] ',    # 0x83
+'[?] ',    # 0x84
+'[?] ',    # 0x85
+'[?] ',    # 0x86
+'[?] ',    # 0x87
+'[?] ',    # 0x88
+'[?] ',    # 0x89
+'[?] ',    # 0x8a
+'[?] ',    # 0x8b
+'[?] ',    # 0x8c
+'[?] ',    # 0x8d
+'[?] ',    # 0x8e
+'[?] ',    # 0x8f
+'[?] ',    # 0x90
+'[?] ',    # 0x91
+'[?] ',    # 0x92
+'[?] ',    # 0x93
+'[?] ',    # 0x94
+'[?] ',    # 0x95
+'[?] ',    # 0x96
+'[?] ',    # 0x97
+'[?] ',    # 0x98
+'[?] ',    # 0x99
+'[?]',    # 0x9a
+'[?] ',    # 0x9b
+'[?] ',    # 0x9c
+'[?] ',    # 0x9d
+'[?] ',    # 0x9e
+'[?] ',    # 0x9f
+'[?] ',    # 0xa0
+'[?] ',    # 0xa1
+'[?] ',    # 0xa2
+'[?] ',    # 0xa3
+'[?] ',    # 0xa4
+'[?] ',    # 0xa5
+'[?] ',    # 0xa6
+'[?] ',    # 0xa7
+'[?] ',    # 0xa8
+'[?] ',    # 0xa9
+'[?] ',    # 0xaa
+'[?] ',    # 0xab
+'[?] ',    # 0xac
+'[?] ',    # 0xad
+'[?] ',    # 0xae
+'[?] ',    # 0xaf
+'[?] ',    # 0xb0
+'[?] ',    # 0xb1
+'[?] ',    # 0xb2
+'[?] ',    # 0xb3
+'[?] ',    # 0xb4
+'[?] ',    # 0xb5
+'[?] ',    # 0xb6
+'[?] ',    # 0xb7
+'[?] ',    # 0xb8
+'[?] ',    # 0xb9
+'[?] ',    # 0xba
+'[?] ',    # 0xbb
+'[?] ',    # 0xbc
+'[?] ',    # 0xbd
+'[?] ',    # 0xbe
+'[?] ',    # 0xbf
+'[?] ',    # 0xc0
+'[?] ',    # 0xc1
+'[?] ',    # 0xc2
+'[?] ',    # 0xc3
+'[?] ',    # 0xc4
+'[?] ',    # 0xc5
+'[?] ',    # 0xc6
+'[?] ',    # 0xc7
+'[?] ',    # 0xc8
+'[?] ',    # 0xc9
+'[?] ',    # 0xca
+'[?] ',    # 0xcb
+'[?] ',    # 0xcc
+'[?] ',    # 0xcd
+'[?] ',    # 0xce
+'[?] ',    # 0xcf
+'[?] ',    # 0xd0
+'[?] ',    # 0xd1
+'[?] ',    # 0xd2
+'[?] ',    # 0xd3
+'[?] ',    # 0xd4
+'[?] ',    # 0xd5
+'[?] ',    # 0xd6
+'[?] ',    # 0xd7
+'[?] ',    # 0xd8
+'[?] ',    # 0xd9
+'[?] ',    # 0xda
+'[?] ',    # 0xdb
+'[?] ',    # 0xdc
+'[?] ',    # 0xdd
+'[?] ',    # 0xde
+'[?] ',    # 0xdf
+'[?] ',    # 0xe0
+'[?] ',    # 0xe1
+'[?] ',    # 0xe2
+'[?] ',    # 0xe3
+'[?] ',    # 0xe4
+'[?] ',    # 0xe5
+'[?] ',    # 0xe6
+'[?] ',    # 0xe7
+'[?] ',    # 0xe8
+'[?] ',    # 0xe9
+'[?] ',    # 0xea
+'[?] ',    # 0xeb
+'[?] ',    # 0xec
+'[?] ',    # 0xed
+'[?] ',    # 0xee
+'[?] ',    # 0xef
+'[?] ',    # 0xf0
+'[?] ',    # 0xf1
+'[?] ',    # 0xf2
+'[?] ',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x02f.py

@@ -0,0 +1,257 @@
+data = (
+'[?] ',    # 0x00
+'[?] ',    # 0x01
+'[?] ',    # 0x02
+'[?] ',    # 0x03
+'[?] ',    # 0x04
+'[?] ',    # 0x05
+'[?] ',    # 0x06
+'[?] ',    # 0x07
+'[?] ',    # 0x08
+'[?] ',    # 0x09
+'[?] ',    # 0x0a
+'[?] ',    # 0x0b
+'[?] ',    # 0x0c
+'[?] ',    # 0x0d
+'[?] ',    # 0x0e
+'[?] ',    # 0x0f
+'[?] ',    # 0x10
+'[?] ',    # 0x11
+'[?] ',    # 0x12
+'[?] ',    # 0x13
+'[?] ',    # 0x14
+'[?] ',    # 0x15
+'[?] ',    # 0x16
+'[?] ',    # 0x17
+'[?] ',    # 0x18
+'[?] ',    # 0x19
+'[?] ',    # 0x1a
+'[?] ',    # 0x1b
+'[?] ',    # 0x1c
+'[?] ',    # 0x1d
+'[?] ',    # 0x1e
+'[?] ',    # 0x1f
+'[?] ',    # 0x20
+'[?] ',    # 0x21
+'[?] ',    # 0x22
+'[?] ',    # 0x23
+'[?] ',    # 0x24
+'[?] ',    # 0x25
+'[?] ',    # 0x26
+'[?] ',    # 0x27
+'[?] ',    # 0x28
+'[?] ',    # 0x29
+'[?] ',    # 0x2a
+'[?] ',    # 0x2b
+'[?] ',    # 0x2c
+'[?] ',    # 0x2d
+'[?] ',    # 0x2e
+'[?] ',    # 0x2f
+'[?] ',    # 0x30
+'[?] ',    # 0x31
+'[?] ',    # 0x32
+'[?] ',    # 0x33
+'[?] ',    # 0x34
+'[?] ',    # 0x35
+'[?] ',    # 0x36
+'[?] ',    # 0x37
+'[?] ',    # 0x38
+'[?] ',    # 0x39
+'[?] ',    # 0x3a
+'[?] ',    # 0x3b
+'[?] ',    # 0x3c
+'[?] ',    # 0x3d
+'[?] ',    # 0x3e
+'[?] ',    # 0x3f
+'[?] ',    # 0x40
+'[?] ',    # 0x41
+'[?] ',    # 0x42
+'[?] ',    # 0x43
+'[?] ',    # 0x44
+'[?] ',    # 0x45
+'[?] ',    # 0x46
+'[?] ',    # 0x47
+'[?] ',    # 0x48
+'[?] ',    # 0x49
+'[?] ',    # 0x4a
+'[?] ',    # 0x4b
+'[?] ',    # 0x4c
+'[?] ',    # 0x4d
+'[?] ',    # 0x4e
+'[?] ',    # 0x4f
+'[?] ',    # 0x50
+'[?] ',    # 0x51
+'[?] ',    # 0x52
+'[?] ',    # 0x53
+'[?] ',    # 0x54
+'[?] ',    # 0x55
+'[?] ',    # 0x56
+'[?] ',    # 0x57
+'[?] ',    # 0x58
+'[?] ',    # 0x59
+'[?] ',    # 0x5a
+'[?] ',    # 0x5b
+'[?] ',    # 0x5c
+'[?] ',    # 0x5d
+'[?] ',    # 0x5e
+'[?] ',    # 0x5f
+'[?] ',    # 0x60
+'[?] ',    # 0x61
+'[?] ',    # 0x62
+'[?] ',    # 0x63
+'[?] ',    # 0x64
+'[?] ',    # 0x65
+'[?] ',    # 0x66
+'[?] ',    # 0x67
+'[?] ',    # 0x68
+'[?] ',    # 0x69
+'[?] ',    # 0x6a
+'[?] ',    # 0x6b
+'[?] ',    # 0x6c
+'[?] ',    # 0x6d
+'[?] ',    # 0x6e
+'[?] ',    # 0x6f
+'[?] ',    # 0x70
+'[?] ',    # 0x71
+'[?] ',    # 0x72
+'[?] ',    # 0x73
+'[?] ',    # 0x74
+'[?] ',    # 0x75
+'[?] ',    # 0x76
+'[?] ',    # 0x77
+'[?] ',    # 0x78
+'[?] ',    # 0x79
+'[?] ',    # 0x7a
+'[?] ',    # 0x7b
+'[?] ',    # 0x7c
+'[?] ',    # 0x7d
+'[?] ',    # 0x7e
+'[?] ',    # 0x7f
+'[?] ',    # 0x80
+'[?] ',    # 0x81
+'[?] ',    # 0x82
+'[?] ',    # 0x83
+'[?] ',    # 0x84
+'[?] ',    # 0x85
+'[?] ',    # 0x86
+'[?] ',    # 0x87
+'[?] ',    # 0x88
+'[?] ',    # 0x89
+'[?] ',    # 0x8a
+'[?] ',    # 0x8b
+'[?] ',    # 0x8c
+'[?] ',    # 0x8d
+'[?] ',    # 0x8e
+'[?] ',    # 0x8f
+'[?] ',    # 0x90
+'[?] ',    # 0x91
+'[?] ',    # 0x92
+'[?] ',    # 0x93
+'[?] ',    # 0x94
+'[?] ',    # 0x95
+'[?] ',    # 0x96
+'[?] ',    # 0x97
+'[?] ',    # 0x98
+'[?] ',    # 0x99
+'[?] ',    # 0x9a
+'[?] ',    # 0x9b
+'[?] ',    # 0x9c
+'[?] ',    # 0x9d
+'[?] ',    # 0x9e
+'[?] ',    # 0x9f
+'[?] ',    # 0xa0
+'[?] ',    # 0xa1
+'[?] ',    # 0xa2
+'[?] ',    # 0xa3
+'[?] ',    # 0xa4
+'[?] ',    # 0xa5
+'[?] ',    # 0xa6
+'[?] ',    # 0xa7
+'[?] ',    # 0xa8
+'[?] ',    # 0xa9
+'[?] ',    # 0xaa
+'[?] ',    # 0xab
+'[?] ',    # 0xac
+'[?] ',    # 0xad
+'[?] ',    # 0xae
+'[?] ',    # 0xaf
+'[?] ',    # 0xb0
+'[?] ',    # 0xb1
+'[?] ',    # 0xb2
+'[?] ',    # 0xb3
+'[?] ',    # 0xb4
+'[?] ',    # 0xb5
+'[?] ',    # 0xb6
+'[?] ',    # 0xb7
+'[?] ',    # 0xb8
+'[?] ',    # 0xb9
+'[?] ',    # 0xba
+'[?] ',    # 0xbb
+'[?] ',    # 0xbc
+'[?] ',    # 0xbd
+'[?] ',    # 0xbe
+'[?] ',    # 0xbf
+'[?] ',    # 0xc0
+'[?] ',    # 0xc1
+'[?] ',    # 0xc2
+'[?] ',    # 0xc3
+'[?] ',    # 0xc4
+'[?] ',    # 0xc5
+'[?] ',    # 0xc6
+'[?] ',    # 0xc7
+'[?] ',    # 0xc8
+'[?] ',    # 0xc9
+'[?] ',    # 0xca
+'[?] ',    # 0xcb
+'[?] ',    # 0xcc
+'[?] ',    # 0xcd
+'[?] ',    # 0xce
+'[?] ',    # 0xcf
+'[?] ',    # 0xd0
+'[?] ',    # 0xd1
+'[?] ',    # 0xd2
+'[?] ',    # 0xd3
+'[?] ',    # 0xd4
+'[?] ',    # 0xd5
+'[?]',    # 0xd6
+'[?]',    # 0xd7
+'[?]',    # 0xd8
+'[?]',    # 0xd9
+'[?]',    # 0xda
+'[?]',    # 0xdb
+'[?]',    # 0xdc
+'[?]',    # 0xdd
+'[?]',    # 0xde
+'[?]',    # 0xdf
+'[?]',    # 0xe0
+'[?]',    # 0xe1
+'[?]',    # 0xe2
+'[?]',    # 0xe3
+'[?]',    # 0xe4
+'[?]',    # 0xe5
+'[?]',    # 0xe6
+'[?]',    # 0xe7
+'[?]',    # 0xe8
+'[?]',    # 0xe9
+'[?]',    # 0xea
+'[?]',    # 0xeb
+'[?]',    # 0xec
+'[?]',    # 0xed
+'[?]',    # 0xee
+'[?]',    # 0xef
+'[?] ',    # 0xf0
+'[?] ',    # 0xf1
+'[?] ',    # 0xf2
+'[?] ',    # 0xf3
+'[?] ',    # 0xf4
+'[?] ',    # 0xf5
+'[?] ',    # 0xf6
+'[?] ',    # 0xf7
+'[?] ',    # 0xf8
+'[?] ',    # 0xf9
+'[?] ',    # 0xfa
+'[?] ',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x030.py

@@ -0,0 +1,257 @@
+data = (
+' ',    # 0x00
+', ',    # 0x01
+'. ',    # 0x02
+'"',    # 0x03
+'[JIS]',    # 0x04
+'"',    # 0x05
+'/',    # 0x06
+'0',    # 0x07
+'<',    # 0x08
+'> ',    # 0x09
+'<<',    # 0x0a
+'>> ',    # 0x0b
+'[',    # 0x0c
+'] ',    # 0x0d
+'{',    # 0x0e
+'} ',    # 0x0f
+'[(',    # 0x10
+')] ',    # 0x11
+'@',    # 0x12
+'X ',    # 0x13
+'[',    # 0x14
+'] ',    # 0x15
+'[[',    # 0x16
+']] ',    # 0x17
+'((',    # 0x18
+')) ',    # 0x19
+'[[',    # 0x1a
+']] ',    # 0x1b
+'~ ',    # 0x1c
+'``',    # 0x1d
+'\'\'',    # 0x1e
+',,',    # 0x1f
+'@',    # 0x20
+'1',    # 0x21
+'2',    # 0x22
+'3',    # 0x23
+'4',    # 0x24
+'5',    # 0x25
+'6',    # 0x26
+'7',    # 0x27
+'8',    # 0x28
+'9',    # 0x29
+'',    # 0x2a
+'',    # 0x2b
+'',    # 0x2c
+'',    # 0x2d
+'',    # 0x2e
+'',    # 0x2f
+'~',    # 0x30
+'+',    # 0x31
+'+',    # 0x32
+'+',    # 0x33
+'+',    # 0x34
+'',    # 0x35
+'@',    # 0x36
+' // ',    # 0x37
+'+10+',    # 0x38
+'+20+',    # 0x39
+'+30+',    # 0x3a
+'[?]',    # 0x3b
+'[?]',    # 0x3c
+'[?]',    # 0x3d
+'',    # 0x3e
+'',    # 0x3f
+'[?]',    # 0x40
+'a',    # 0x41
+'a',    # 0x42
+'i',    # 0x43
+'i',    # 0x44
+'u',    # 0x45
+'u',    # 0x46
+'e',    # 0x47
+'e',    # 0x48
+'o',    # 0x49
+'o',    # 0x4a
+'ka',    # 0x4b
+'ga',    # 0x4c
+'ki',    # 0x4d
+'gi',    # 0x4e
+'ku',    # 0x4f
+'gu',    # 0x50
+'ke',    # 0x51
+'ge',    # 0x52
+'ko',    # 0x53
+'go',    # 0x54
+'sa',    # 0x55
+'za',    # 0x56
+'shi',   # 0x57
+'zi',    # 0x58
+'su',    # 0x59
+'zu',    # 0x5a
+'se',    # 0x5b
+'ze',    # 0x5c
+'so',    # 0x5d
+'zo',    # 0x5e
+'ta',    # 0x5f
+'da',    # 0x60
+'chi',   # 0x61
+'di',    # 0x62
+'tsu',   # 0x63
+'tsu',   # 0x64
+'du',    # 0x65
+'te',    # 0x66
+'de',    # 0x67
+'to',    # 0x68
+'do',    # 0x69
+'na',    # 0x6a
+'ni',    # 0x6b
+'nu',    # 0x6c
+'ne',    # 0x6d
+'no',    # 0x6e
+'ha',    # 0x6f
+'ba',    # 0x70
+'pa',    # 0x71
+'hi',    # 0x72
+'bi',    # 0x73
+'pi',    # 0x74
+'hu',    # 0x75
+'bu',    # 0x76
+'pu',    # 0x77
+'he',    # 0x78
+'be',    # 0x79
+'pe',    # 0x7a
+'ho',    # 0x7b
+'bo',    # 0x7c
+'po',    # 0x7d
+'ma',    # 0x7e
+'mi',    # 0x7f
+'mu',    # 0x80
+'me',    # 0x81
+'mo',    # 0x82
+'ya',    # 0x83
+'ya',    # 0x84
+'yu',    # 0x85
+'yu',    # 0x86
+'yo',    # 0x87
+'yo',    # 0x88
+'ra',    # 0x89
+'ri',    # 0x8a
+'ru',    # 0x8b
+'re',    # 0x8c
+'ro',    # 0x8d
+'wa',    # 0x8e
+'wa',    # 0x8f
+'wi',    # 0x90
+'we',    # 0x91
+'wo',    # 0x92
+'n',    # 0x93
+'vu',    # 0x94
+'[?]',    # 0x95
+'[?]',    # 0x96
+'[?]',    # 0x97
+'[?]',    # 0x98
+'',    # 0x99
+'',    # 0x9a
+'',    # 0x9b
+'',    # 0x9c
+'"',    # 0x9d
+'"',    # 0x9e
+'[?]',    # 0x9f
+'[?]',    # 0xa0
+'a',    # 0xa1
+'a',    # 0xa2
+'i',    # 0xa3
+'i',    # 0xa4
+'u',    # 0xa5
+'u',    # 0xa6
+'e',    # 0xa7
+'e',    # 0xa8
+'o',    # 0xa9
+'o',    # 0xaa
+'ka',    # 0xab
+'ga',    # 0xac
+'ki',    # 0xad
+'gi',    # 0xae
+'ku',    # 0xaf
+'gu',    # 0xb0
+'ke',    # 0xb1
+'ge',    # 0xb2
+'ko',    # 0xb3
+'go',    # 0xb4
+'sa',    # 0xb5
+'za',    # 0xb6
+'shi',   # 0xb7
+'zi',    # 0xb8
+'su',    # 0xb9
+'zu',    # 0xba
+'se',    # 0xbb
+'ze',    # 0xbc
+'so',    # 0xbd
+'zo',    # 0xbe
+'ta',    # 0xbf
+'da',    # 0xc0
+'chi',   # 0xc1
+'di',    # 0xc2
+'tsu',   # 0xc3
+'tsu',   # 0xc4
+'du',    # 0xc5
+'te',    # 0xc6
+'de',    # 0xc7
+'to',    # 0xc8
+'do',    # 0xc9
+'na',    # 0xca
+'ni',    # 0xcb
+'nu',    # 0xcc
+'ne',    # 0xcd
+'no',    # 0xce
+'ha',    # 0xcf
+'ba',    # 0xd0
+'pa',    # 0xd1
+'hi',    # 0xd2
+'bi',    # 0xd3
+'pi',    # 0xd4
+'hu',    # 0xd5
+'bu',    # 0xd6
+'pu',    # 0xd7
+'he',    # 0xd8
+'be',    # 0xd9
+'pe',    # 0xda
+'ho',    # 0xdb
+'bo',    # 0xdc
+'po',    # 0xdd
+'ma',    # 0xde
+'mi',    # 0xdf
+'mu',    # 0xe0
+'me',    # 0xe1
+'mo',    # 0xe2
+'ya',    # 0xe3
+'ya',    # 0xe4
+'yu',    # 0xe5
+'yu',    # 0xe6
+'yo',    # 0xe7
+'yo',    # 0xe8
+'ra',    # 0xe9
+'ri',    # 0xea
+'ru',    # 0xeb
+'re',    # 0xec
+'ro',    # 0xed
+'wa',    # 0xee
+'wa',    # 0xef
+'wi',    # 0xf0
+'we',    # 0xf1
+'wo',    # 0xf2
+'n',    # 0xf3
+'vu',    # 0xf4
+'ka',    # 0xf5
+'ke',    # 0xf6
+'va',    # 0xf7
+'vi',    # 0xf8
+'ve',    # 0xf9
+'vo',    # 0xfa
+'',    # 0xfb
+'',    # 0xfc
+'"',    # 0xfd
+'"',    # 0xfe
+)

lib/unidecode/x031.py

@@ -0,0 +1,257 @@
+data = (
+'[?]',    # 0x00
+'[?]',    # 0x01
+'[?]',    # 0x02
+'[?]',    # 0x03
+'[?]',    # 0x04
+'B',    # 0x05
+'P',    # 0x06
+'M',    # 0x07
+'F',    # 0x08
+'D',    # 0x09
+'T',    # 0x0a
+'N',    # 0x0b
+'L',    # 0x0c
+'G',    # 0x0d
+'K',    # 0x0e
+'H',    # 0x0f
+'J',    # 0x10
+'Q',    # 0x11
+'X',    # 0x12
+'ZH',    # 0x13
+'CH',    # 0x14
+'SH',    # 0x15
+'R',    # 0x16
+'Z',    # 0x17
+'C',    # 0x18
+'S',    # 0x19
+'A',    # 0x1a
+'O',    # 0x1b
+'E',    # 0x1c
+'EH',    # 0x1d
+'AI',    # 0x1e
+'EI',    # 0x1f
+'AU',    # 0x20
+'OU',    # 0x21
+'AN',    # 0x22
+'EN',    # 0x23
+'ANG',    # 0x24
+'ENG',    # 0x25
+'ER',    # 0x26
+'I',    # 0x27
+'U',    # 0x28
+'IU',    # 0x29
+'V',    # 0x2a
+'NG',    # 0x2b
+'GN',    # 0x2c
+'[?]',    # 0x2d
+'[?]',    # 0x2e
+'[?]',    # 0x2f
+'[?]',    # 0x30
+'g',    # 0x31
+'gg',    # 0x32
+'gs',    # 0x33
+'n',    # 0x34
+'nj',    # 0x35
+'nh',    # 0x36
+'d',    # 0x37
+'dd',    # 0x38
+'r',    # 0x39
+'lg',    # 0x3a
+'lm',    # 0x3b
+'lb',    # 0x3c
+'ls',    # 0x3d
+'lt',    # 0x3e
+'lp',    # 0x3f
+'rh',    # 0x40
+'m',    # 0x41
+'b',    # 0x42
+'bb',    # 0x43
+'bs',    # 0x44
+'s',    # 0x45
+'ss',    # 0x46
+'',    # 0x47
+'j',    # 0x48
+'jj',    # 0x49
+'c',    # 0x4a
+'k',    # 0x4b
+'t',    # 0x4c
+'p',    # 0x4d
+'h',    # 0x4e
+'a',    # 0x4f
+'ae',    # 0x50
+'ya',    # 0x51
+'yae',    # 0x52
+'eo',    # 0x53
+'e',    # 0x54
+'yeo',    # 0x55
+'ye',    # 0x56
+'o',    # 0x57
+'wa',    # 0x58
+'wae',    # 0x59
+'oe',    # 0x5a
+'yo',    # 0x5b
+'u',    # 0x5c
+'weo',    # 0x5d
+'we',    # 0x5e
+'wi',    # 0x5f
+'yu',    # 0x60
+'eu',    # 0x61
+'yi',    # 0x62
+'i',    # 0x63
+'',    # 0x64
+'nn',    # 0x65
+'nd',    # 0x66
+'ns',    # 0x67
+'nZ',    # 0x68
+'lgs',    # 0x69
+'ld',    # 0x6a
+'lbs',    # 0x6b
+'lZ',    # 0x6c
+'lQ',    # 0x6d
+'mb',    # 0x6e
+'ms',    # 0x6f
+'mZ',    # 0x70
+'mN',    # 0x71
+'bg',    # 0x72
+'',    # 0x73
+'bsg',    # 0x74
+'bst',    # 0x75
+'bj',    # 0x76
+'bt',    # 0x77
+'bN',    # 0x78
+'bbN',    # 0x79
+'sg',    # 0x7a
+'sn',    # 0x7b
+'sd',    # 0x7c
+'sb',    # 0x7d
+'sj',    # 0x7e
+'Z',    # 0x7f
+'',    # 0x80
+'N',    # 0x81
+'Ns',    # 0x82
+'NZ',    # 0x83
+'pN',    # 0x84
+'hh',    # 0x85
+'Q',    # 0x86
+'yo-ya',    # 0x87
+'yo-yae',    # 0x88
+'yo-i',    # 0x89
+'yu-yeo',    # 0x8a
+'yu-ye',    # 0x8b
+'yu-i',    # 0x8c
+'U',    # 0x8d
+'U-i',    # 0x8e
+'[?]',    # 0x8f
+'',    # 0x90
+'',    # 0x91
+'',    # 0x92
+'',    # 0x93
+'',    # 0x94
+'',    # 0x95
+'',    # 0x96
+'',    # 0x97
+'',    # 0x98
+'',    # 0x99
+'',    # 0x9a
+'',    # 0x9b
+'',    # 0x9c
+'',    # 0x9d
+'',    # 0x9e
+'',    # 0x9f
+'BU',    # 0xa0
+'ZI',    # 0xa1
+'JI',    # 0xa2
+'GU',    # 0xa3
+'EE',    # 0xa4
+'ENN',    # 0xa5
+'OO',    # 0xa6
+'ONN',    # 0xa7
+'IR',    # 0xa8
+'ANN',    # 0xa9
+'INN',    # 0xaa
+'UNN',    # 0xab
+'IM',    # 0xac
+'NGG',    # 0xad
+'AINN',    # 0xae
+'AUNN',    # 0xaf
+'AM',    # 0xb0
+'OM',    # 0xb1
+'ONG',    # 0xb2
+'INNN',    # 0xb3
+'P',    # 0xb4
+'T',    # 0xb5
+'K',    # 0xb6
+'H',    # 0xb7
+'[?]',    # 0xb8
+'[?]',    # 0xb9
+'[?]',    # 0xba
+'[?]',    # 0xbb
+'[?]',    # 0xbc
+'[?]',    # 0xbd
+'[?]',    # 0xbe
+'[?]',    # 0xbf
+'[?]',    # 0xc0
+'[?]',    # 0xc1
+'[?]',    # 0xc2
+'[?]',    # 0xc3
+'[?]',    # 0xc4
+'[?]',    # 0xc5
+'[?]',    # 0xc6
+'[?]',    # 0xc7
+'[?]',    # 0xc8
+'[?]',    # 0xc9
+'[?]',    # 0xca
+'[?]',    # 0xcb
+'[?]',    # 0xcc
+'[?]',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'[?]',    # 0xd0
+'[?]',    # 0xd1
+'[?]',    # 0xd2
+'[?]',    # 0xd3
+'[?]',    # 0xd4
+'[?]',    # 0xd5
+'[?]',    # 0xd6
+'[?]',    # 0xd7
+'[?]',    # 0xd8
+'[?]',    # 0xd9
+'[?]',    # 0xda
+'[?]',    # 0xdb
+'[?]',    # 0xdc
+'[?]',    # 0xdd
+'[?]',    # 0xde
+'[?]',    # 0xdf
+'[?]',    # 0xe0
+'[?]',    # 0xe1
+'[?]',    # 0xe2
+'[?]',    # 0xe3
+'[?]',    # 0xe4
+'[?]',    # 0xe5
+'[?]',    # 0xe6
+'[?]',    # 0xe7
+'[?]',    # 0xe8
+'[?]',    # 0xe9
+'[?]',    # 0xea
+'[?]',    # 0xeb
+'[?]',    # 0xec
+'[?]',    # 0xed
+'[?]',    # 0xee
+'[?]',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'[?]',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x032.py

@@ -0,0 +1,257 @@
+data = (
+'(g)',    # 0x00
+'(n)',    # 0x01
+'(d)',    # 0x02
+'(r)',    # 0x03
+'(m)',    # 0x04
+'(b)',    # 0x05
+'(s)',    # 0x06
+'()',    # 0x07
+'(j)',    # 0x08
+'(c)',    # 0x09
+'(k)',    # 0x0a
+'(t)',    # 0x0b
+'(p)',    # 0x0c
+'(h)',    # 0x0d
+'(ga)',    # 0x0e
+'(na)',    # 0x0f
+'(da)',    # 0x10
+'(ra)',    # 0x11
+'(ma)',    # 0x12
+'(ba)',    # 0x13
+'(sa)',    # 0x14
+'(a)',    # 0x15
+'(ja)',    # 0x16
+'(ca)',    # 0x17
+'(ka)',    # 0x18
+'(ta)',    # 0x19
+'(pa)',    # 0x1a
+'(ha)',    # 0x1b
+'(ju)',    # 0x1c
+'[?]',    # 0x1d
+'[?]',    # 0x1e
+'[?]',    # 0x1f
+'(1) ',    # 0x20
+'(2) ',    # 0x21
+'(3) ',    # 0x22
+'(4) ',    # 0x23
+'(5) ',    # 0x24
+'(6) ',    # 0x25
+'(7) ',    # 0x26
+'(8) ',    # 0x27
+'(9) ',    # 0x28
+'(10) ',    # 0x29
+'(Yue) ',    # 0x2a
+'(Huo) ',    # 0x2b
+'(Shui) ',    # 0x2c
+'(Mu) ',    # 0x2d
+'(Jin) ',    # 0x2e
+'(Tu) ',    # 0x2f
+'(Ri) ',    # 0x30
+'(Zhu) ',    # 0x31
+'(You) ',    # 0x32
+'(She) ',    # 0x33
+'(Ming) ',    # 0x34
+'(Te) ',    # 0x35
+'(Cai) ',    # 0x36
+'(Zhu) ',    # 0x37
+'(Lao) ',    # 0x38
+'(Dai) ',    # 0x39
+'(Hu) ',    # 0x3a
+'(Xue) ',    # 0x3b
+'(Jian) ',    # 0x3c
+'(Qi) ',    # 0x3d
+'(Zi) ',    # 0x3e
+'(Xie) ',    # 0x3f
+'(Ji) ',    # 0x40
+'(Xiu) ',    # 0x41
+'<<',    # 0x42
+'>>',    # 0x43
+'[?]',    # 0x44
+'[?]',    # 0x45
+'[?]',    # 0x46
+'[?]',    # 0x47
+'[?]',    # 0x48
+'[?]',    # 0x49
+'[?]',    # 0x4a
+'[?]',    # 0x4b
+'[?]',    # 0x4c
+'[?]',    # 0x4d
+'[?]',    # 0x4e
+'[?]',    # 0x4f
+'[?]',    # 0x50
+'21',    # 0x51
+'22',    # 0x52
+'23',    # 0x53
+'24',    # 0x54
+'25',    # 0x55
+'26',    # 0x56
+'27',    # 0x57
+'28',    # 0x58
+'29',    # 0x59
+'30',    # 0x5a
+'31',    # 0x5b
+'32',    # 0x5c
+'33',    # 0x5d
+'34',    # 0x5e
+'35',    # 0x5f
+'(g)',    # 0x60
+'(n)',    # 0x61
+'(d)',    # 0x62
+'(r)',    # 0x63
+'(m)',    # 0x64
+'(b)',    # 0x65
+'(s)',    # 0x66
+'()',    # 0x67
+'(j)',    # 0x68
+'(c)',    # 0x69
+'(k)',    # 0x6a
+'(t)',    # 0x6b
+'(p)',    # 0x6c
+'(h)',    # 0x6d
+'(ga)',    # 0x6e
+'(na)',    # 0x6f
+'(da)',    # 0x70
+'(ra)',    # 0x71
+'(ma)',    # 0x72
+'(ba)',    # 0x73
+'(sa)',    # 0x74
+'(a)',    # 0x75
+'(ja)',    # 0x76
+'(ca)',    # 0x77
+'(ka)',    # 0x78
+'(ta)',    # 0x79
+'(pa)',    # 0x7a
+'(ha)',    # 0x7b
+'[?]',    # 0x7c
+'[?]',    # 0x7d
+'[?]',    # 0x7e
+'KIS ',    # 0x7f
+'(1) ',    # 0x80
+'(2) ',    # 0x81
+'(3) ',    # 0x82
+'(4) ',    # 0x83
+'(5) ',    # 0x84
+'(6) ',    # 0x85
+'(7) ',    # 0x86
+'(8) ',    # 0x87
+'(9) ',    # 0x88
+'(10) ',    # 0x89
+'(Yue) ',    # 0x8a
+'(Huo) ',    # 0x8b
+'(Shui) ',    # 0x8c
+'(Mu) ',    # 0x8d
+'(Jin) ',    # 0x8e
+'(Tu) ',    # 0x8f
+'(Ri) ',    # 0x90
+'(Zhu) ',    # 0x91
+'(You) ',    # 0x92
+'(She) ',    # 0x93
+'(Ming) ',    # 0x94
+'(Te) ',    # 0x95
+'(Cai) ',    # 0x96
+'(Zhu) ',    # 0x97
+'(Lao) ',    # 0x98
+'(Mi) ',    # 0x99
+'(Nan) ',    # 0x9a
+'(Nu) ',    # 0x9b
+'(Shi) ',    # 0x9c
+'(You) ',    # 0x9d
+'(Yin) ',    # 0x9e
+'(Zhu) ',    # 0x9f
+'(Xiang) ',    # 0xa0
+'(Xiu) ',    # 0xa1
+'(Xie) ',    # 0xa2
+'(Zheng) ',    # 0xa3
+'(Shang) ',    # 0xa4
+'(Zhong) ',    # 0xa5
+'(Xia) ',    # 0xa6
+'(Zuo) ',    # 0xa7
+'(You) ',    # 0xa8
+'(Yi) ',    # 0xa9
+'(Zong) ',    # 0xaa
+'(Xue) ',    # 0xab
+'(Jian) ',    # 0xac
+'(Qi) ',    # 0xad
+'(Zi) ',    # 0xae
+'(Xie) ',    # 0xaf
+'(Ye) ',    # 0xb0
+'36',    # 0xb1
+'37',    # 0xb2
+'38',    # 0xb3
+'39',    # 0xb4
+'40',    # 0xb5
+'41',    # 0xb6
+'42',    # 0xb7
+'43',    # 0xb8
+'44',    # 0xb9
+'45',    # 0xba
+'46',    # 0xbb
+'47',    # 0xbc
+'48',    # 0xbd
+'49',    # 0xbe
+'50',    # 0xbf
+'1M',    # 0xc0
+'2M',    # 0xc1
+'3M',    # 0xc2
+'4M',    # 0xc3
+'5M',    # 0xc4
+'6M',    # 0xc5
+'7M',    # 0xc6
+'8M',    # 0xc7
+'9M',    # 0xc8
+'10M',    # 0xc9
+'11M',    # 0xca
+'12M',    # 0xcb
+'[?]',    # 0xcc
+'[?]',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'a',    # 0xd0
+'i',    # 0xd1
+'u',    # 0xd2
+'u',    # 0xd3
+'o',    # 0xd4
+'ka',    # 0xd5
+'ki',    # 0xd6
+'ku',    # 0xd7
+'ke',    # 0xd8
+'ko',    # 0xd9
+'sa',    # 0xda
+'si',    # 0xdb
+'su',    # 0xdc
+'se',    # 0xdd
+'so',    # 0xde
+'ta',    # 0xdf
+'ti',    # 0xe0
+'tu',    # 0xe1
+'te',    # 0xe2
+'to',    # 0xe3
+'na',    # 0xe4
+'ni',    # 0xe5
+'nu',    # 0xe6
+'ne',    # 0xe7
+'no',    # 0xe8
+'ha',    # 0xe9
+'hi',    # 0xea
+'hu',    # 0xeb
+'he',    # 0xec
+'ho',    # 0xed
+'ma',    # 0xee
+'mi',    # 0xef
+'mu',    # 0xf0
+'me',    # 0xf1
+'mo',    # 0xf2
+'ya',    # 0xf3
+'yu',    # 0xf4
+'yo',    # 0xf5
+'ra',    # 0xf6
+'ri',    # 0xf7
+'ru',    # 0xf8
+'re',    # 0xf9
+'ro',    # 0xfa
+'wa',    # 0xfb
+'wi',    # 0xfc
+'we',    # 0xfd
+'wo',    # 0xfe
+)

lib/unidecode/x033.py

@@ -0,0 +1,257 @@
+data = (
+'apartment',    # 0x00
+'alpha',    # 0x01
+'ampere',    # 0x02
+'are',    # 0x03
+'inning',    # 0x04
+'inch',    # 0x05
+'won',    # 0x06
+'escudo',    # 0x07
+'acre',    # 0x08
+'ounce',    # 0x09
+'ohm',    # 0x0a
+'kai-ri',    # 0x0b
+'carat',    # 0x0c
+'calorie',    # 0x0d
+'gallon',    # 0x0e
+'gamma',    # 0x0f
+'giga',    # 0x10
+'guinea',    # 0x11
+'curie',    # 0x12
+'guilder',    # 0x13
+'kilo',    # 0x14
+'kilogram',    # 0x15
+'kilometer',    # 0x16
+'kilowatt',    # 0x17
+'gram',    # 0x18
+'gram ton',    # 0x19
+'cruzeiro',    # 0x1a
+'krone',    # 0x1b
+'case',    # 0x1c
+'koruna',    # 0x1d
+'co-op',    # 0x1e
+'cycle',    # 0x1f
+'centime',    # 0x20
+'shilling',    # 0x21
+'centi',    # 0x22
+'cent',    # 0x23
+'dozen',    # 0x24
+'desi',    # 0x25
+'dollar',    # 0x26
+'ton',    # 0x27
+'nano',    # 0x28
+'knot',    # 0x29
+'heights',    # 0x2a
+'percent',    # 0x2b
+'parts',    # 0x2c
+'barrel',    # 0x2d
+'piaster',    # 0x2e
+'picul',    # 0x2f
+'pico',    # 0x30
+'building',    # 0x31
+'farad',    # 0x32
+'feet',    # 0x33
+'bushel',    # 0x34
+'franc',    # 0x35
+'hectare',    # 0x36
+'peso',    # 0x37
+'pfennig',    # 0x38
+'hertz',    # 0x39
+'pence',    # 0x3a
+'page',    # 0x3b
+'beta',    # 0x3c
+'point',    # 0x3d
+'volt',    # 0x3e
+'hon',    # 0x3f
+'pound',    # 0x40
+'hall',    # 0x41
+'horn',    # 0x42
+'micro',    # 0x43
+'mile',    # 0x44
+'mach',    # 0x45
+'mark',    # 0x46
+'mansion',    # 0x47
+'micron',    # 0x48
+'milli',    # 0x49
+'millibar',    # 0x4a
+'mega',    # 0x4b
+'megaton',    # 0x4c
+'meter',    # 0x4d
+'yard',    # 0x4e
+'yard',    # 0x4f
+'yuan',    # 0x50
+'liter',    # 0x51
+'lira',    # 0x52
+'rupee',    # 0x53
+'ruble',    # 0x54
+'rem',    # 0x55
+'roentgen',    # 0x56
+'watt',    # 0x57
+'0h',    # 0x58
+'1h',    # 0x59
+'2h',    # 0x5a
+'3h',    # 0x5b
+'4h',    # 0x5c
+'5h',    # 0x5d
+'6h',    # 0x5e
+'7h',    # 0x5f
+'8h',    # 0x60
+'9h',    # 0x61
+'10h',    # 0x62
+'11h',    # 0x63
+'12h',    # 0x64
+'13h',    # 0x65
+'14h',    # 0x66
+'15h',    # 0x67
+'16h',    # 0x68
+'17h',    # 0x69
+'18h',    # 0x6a
+'19h',    # 0x6b
+'20h',    # 0x6c
+'21h',    # 0x6d
+'22h',    # 0x6e
+'23h',    # 0x6f
+'24h',    # 0x70
+'HPA',    # 0x71
+'da',    # 0x72
+'AU',    # 0x73
+'bar',    # 0x74
+'oV',    # 0x75
+'pc',    # 0x76
+'[?]',    # 0x77
+'[?]',    # 0x78
+'[?]',    # 0x79
+'[?]',    # 0x7a
+'Heisei',    # 0x7b
+'Syouwa',    # 0x7c
+'Taisyou',    # 0x7d
+'Meiji',    # 0x7e
+'Inc.',    # 0x7f
+'pA',    # 0x80
+'nA',    # 0x81
+'microamp',    # 0x82
+'mA',    # 0x83
+'kA',    # 0x84
+'kB',    # 0x85
+'MB',    # 0x86
+'GB',    # 0x87
+'cal',    # 0x88
+'kcal',    # 0x89
+'pF',    # 0x8a
+'nF',    # 0x8b
+'microFarad',    # 0x8c
+'microgram',    # 0x8d
+'mg',    # 0x8e
+'kg',    # 0x8f
+'Hz',    # 0x90
+'kHz',    # 0x91
+'MHz',    # 0x92
+'GHz',    # 0x93
+'THz',    # 0x94
+'microliter',    # 0x95
+'ml',    # 0x96
+'dl',    # 0x97
+'kl',    # 0x98
+'fm',    # 0x99
+'nm',    # 0x9a
+'micrometer',    # 0x9b
+'mm',    # 0x9c
+'cm',    # 0x9d
+'km',    # 0x9e
+'mm^2',    # 0x9f
+'cm^2',    # 0xa0
+'m^2',    # 0xa1
+'km^2',    # 0xa2
+'mm^4',    # 0xa3
+'cm^3',    # 0xa4
+'m^3',    # 0xa5
+'km^3',    # 0xa6
+'m/s',    # 0xa7
+'m/s^2',    # 0xa8
+'Pa',    # 0xa9
+'kPa',    # 0xaa
+'MPa',    # 0xab
+'GPa',    # 0xac
+'rad',    # 0xad
+'rad/s',    # 0xae
+'rad/s^2',    # 0xaf
+'ps',    # 0xb0
+'ns',    # 0xb1
+'microsecond',    # 0xb2
+'ms',    # 0xb3
+'pV',    # 0xb4
+'nV',    # 0xb5
+'microvolt',    # 0xb6
+'mV',    # 0xb7
+'kV',    # 0xb8
+'MV',    # 0xb9
+'pW',    # 0xba
+'nW',    # 0xbb
+'microwatt',    # 0xbc
+'mW',    # 0xbd
+'kW',    # 0xbe
+'MW',    # 0xbf
+'kOhm',    # 0xc0
+'MOhm',    # 0xc1
+'a.m.',    # 0xc2
+'Bq',    # 0xc3
+'cc',    # 0xc4
+'cd',    # 0xc5
+'C/kg',    # 0xc6
+'Co.',    # 0xc7
+'dB',    # 0xc8
+'Gy',    # 0xc9
+'ha',    # 0xca
+'HP',    # 0xcb
+'in',    # 0xcc
+'K.K.',    # 0xcd
+'KM',    # 0xce
+'kt',    # 0xcf
+'lm',    # 0xd0
+'ln',    # 0xd1
+'log',    # 0xd2
+'lx',    # 0xd3
+'mb',    # 0xd4
+'mil',    # 0xd5
+'mol',    # 0xd6
+'pH',    # 0xd7
+'p.m.',    # 0xd8
+'PPM',    # 0xd9
+'PR',    # 0xda
+'sr',    # 0xdb
+'Sv',    # 0xdc
+'Wb',    # 0xdd
+'[?]',    # 0xde
+'[?]',    # 0xdf
+'1d',    # 0xe0
+'2d',    # 0xe1
+'3d',    # 0xe2
+'4d',    # 0xe3
+'5d',    # 0xe4
+'6d',    # 0xe5
+'7d',    # 0xe6
+'8d',    # 0xe7
+'9d',    # 0xe8
+'10d',    # 0xe9
+'11d',    # 0xea
+'12d',    # 0xeb
+'13d',    # 0xec
+'14d',    # 0xed
+'15d',    # 0xee
+'16d',    # 0xef
+'17d',    # 0xf0
+'18d',    # 0xf1
+'19d',    # 0xf2
+'20d',    # 0xf3
+'21d',    # 0xf4
+'22d',    # 0xf5
+'23d',    # 0xf6
+'24d',    # 0xf7
+'25d',    # 0xf8
+'26d',    # 0xf9
+'27d',    # 0xfa
+'28d',    # 0xfb
+'29d',    # 0xfc
+'30d',    # 0xfd
+'31d',    # 0xfe
+)

lib/unidecode/x04d.py

@@ -0,0 +1,257 @@
+data = (
+'[?] ',    # 0x00
+'[?] ',    # 0x01
+'[?] ',    # 0x02
+'[?] ',    # 0x03
+'[?] ',    # 0x04
+'[?] ',    # 0x05
+'[?] ',    # 0x06
+'[?] ',    # 0x07
+'[?] ',    # 0x08
+'[?] ',    # 0x09
+'[?] ',    # 0x0a
+'[?] ',    # 0x0b
+'[?] ',    # 0x0c
+'[?] ',    # 0x0d
+'[?] ',    # 0x0e
+'[?] ',    # 0x0f
+'[?] ',    # 0x10
+'[?] ',    # 0x11
+'[?] ',    # 0x12
+'[?] ',    # 0x13
+'[?] ',    # 0x14
+'[?] ',    # 0x15
+'[?] ',    # 0x16
+'[?] ',    # 0x17
+'[?] ',    # 0x18
+'[?] ',    # 0x19
+'[?] ',    # 0x1a
+'[?] ',    # 0x1b
+'[?] ',    # 0x1c
+'[?] ',    # 0x1d
+'[?] ',    # 0x1e
+'[?] ',    # 0x1f
+'[?] ',    # 0x20
+'[?] ',    # 0x21
+'[?] ',    # 0x22
+'[?] ',    # 0x23
+'[?] ',    # 0x24
+'[?] ',    # 0x25
+'[?] ',    # 0x26
+'[?] ',    # 0x27
+'[?] ',    # 0x28
+'[?] ',    # 0x29
+'[?] ',    # 0x2a
+'[?] ',    # 0x2b
+'[?] ',    # 0x2c
+'[?] ',    # 0x2d
+'[?] ',    # 0x2e
+'[?] ',    # 0x2f
+'[?] ',    # 0x30
+'[?] ',    # 0x31
+'[?] ',    # 0x32
+'[?] ',    # 0x33
+'[?] ',    # 0x34
+'[?] ',    # 0x35
+'[?] ',    # 0x36
+'[?] ',    # 0x37
+'[?] ',    # 0x38
+'[?] ',    # 0x39
+'[?] ',    # 0x3a
+'[?] ',    # 0x3b
+'[?] ',    # 0x3c
+'[?] ',    # 0x3d
+'[?] ',    # 0x3e
+'[?] ',    # 0x3f
+'[?] ',    # 0x40
+'[?] ',    # 0x41
+'[?] ',    # 0x42
+'[?] ',    # 0x43
+'[?] ',    # 0x44
+'[?] ',    # 0x45
+'[?] ',    # 0x46
+'[?] ',    # 0x47
+'[?] ',    # 0x48
+'[?] ',    # 0x49
+'[?] ',    # 0x4a
+'[?] ',    # 0x4b
+'[?] ',    # 0x4c
+'[?] ',    # 0x4d
+'[?] ',    # 0x4e
+'[?] ',    # 0x4f
+'[?] ',    # 0x50
+'[?] ',    # 0x51
+'[?] ',    # 0x52
+'[?] ',    # 0x53
+'[?] ',    # 0x54
+'[?] ',    # 0x55
+'[?] ',    # 0x56
+'[?] ',    # 0x57
+'[?] ',    # 0x58
+'[?] ',    # 0x59
+'[?] ',    # 0x5a
+'[?] ',    # 0x5b
+'[?] ',    # 0x5c
+'[?] ',    # 0x5d
+'[?] ',    # 0x5e
+'[?] ',    # 0x5f
+'[?] ',    # 0x60
+'[?] ',    # 0x61
+'[?] ',    # 0x62
+'[?] ',    # 0x63
+'[?] ',    # 0x64
+'[?] ',    # 0x65
+'[?] ',    # 0x66
+'[?] ',    # 0x67
+'[?] ',    # 0x68
+'[?] ',    # 0x69
+'[?] ',    # 0x6a
+'[?] ',    # 0x6b
+'[?] ',    # 0x6c
+'[?] ',    # 0x6d
+'[?] ',    # 0x6e
+'[?] ',    # 0x6f
+'[?] ',    # 0x70
+'[?] ',    # 0x71
+'[?] ',    # 0x72
+'[?] ',    # 0x73
+'[?] ',    # 0x74
+'[?] ',    # 0x75
+'[?] ',    # 0x76
+'[?] ',    # 0x77
+'[?] ',    # 0x78
+'[?] ',    # 0x79
+'[?] ',    # 0x7a
+'[?] ',    # 0x7b
+'[?] ',    # 0x7c
+'[?] ',    # 0x7d
+'[?] ',    # 0x7e
+'[?] ',    # 0x7f
+'[?] ',    # 0x80
+'[?] ',    # 0x81
+'[?] ',    # 0x82
+'[?] ',    # 0x83
+'[?] ',    # 0x84
+'[?] ',    # 0x85
+'[?] ',    # 0x86
+'[?] ',    # 0x87
+'[?] ',    # 0x88
+'[?] ',    # 0x89
+'[?] ',    # 0x8a
+'[?] ',    # 0x8b
+'[?] ',    # 0x8c
+'[?] ',    # 0x8d
+'[?] ',    # 0x8e
+'[?] ',    # 0x8f
+'[?] ',    # 0x90
+'[?] ',    # 0x91
+'[?] ',    # 0x92
+'[?] ',    # 0x93
+'[?] ',    # 0x94
+'[?] ',    # 0x95
+'[?] ',    # 0x96
+'[?] ',    # 0x97
+'[?] ',    # 0x98
+'[?] ',    # 0x99
+'[?] ',    # 0x9a
+'[?] ',    # 0x9b
+'[?] ',    # 0x9c
+'[?] ',    # 0x9d
+'[?] ',    # 0x9e
+'[?] ',    # 0x9f
+'[?] ',    # 0xa0
+'[?] ',    # 0xa1
+'[?] ',    # 0xa2
+'[?] ',    # 0xa3
+'[?] ',    # 0xa4
+'[?] ',    # 0xa5
+'[?] ',    # 0xa6
+'[?] ',    # 0xa7
+'[?] ',    # 0xa8
+'[?] ',    # 0xa9
+'[?] ',    # 0xaa
+'[?] ',    # 0xab
+'[?] ',    # 0xac
+'[?] ',    # 0xad
+'[?] ',    # 0xae
+'[?] ',    # 0xaf
+'[?] ',    # 0xb0
+'[?] ',    # 0xb1
+'[?] ',    # 0xb2
+'[?] ',    # 0xb3
+'[?] ',    # 0xb4
+'[?] ',    # 0xb5
+'[?]',    # 0xb6
+'[?]',    # 0xb7
+'[?]',    # 0xb8
+'[?]',    # 0xb9
+'[?]',    # 0xba
+'[?]',    # 0xbb
+'[?]',    # 0xbc
+'[?]',    # 0xbd
+'[?]',    # 0xbe
+'[?]',    # 0xbf
+'[?]',    # 0xc0
+'[?]',    # 0xc1
+'[?]',    # 0xc2
+'[?]',    # 0xc3
+'[?]',    # 0xc4
+'[?]',    # 0xc5
+'[?]',    # 0xc6
+'[?]',    # 0xc7
+'[?]',    # 0xc8
+'[?]',    # 0xc9
+'[?]',    # 0xca
+'[?]',    # 0xcb
+'[?]',    # 0xcc
+'[?]',    # 0xcd
+'[?]',    # 0xce
+'[?]',    # 0xcf
+'[?]',    # 0xd0
+'[?]',    # 0xd1
+'[?]',    # 0xd2
+'[?]',    # 0xd3
+'[?]',    # 0xd4
+'[?]',    # 0xd5
+'[?]',    # 0xd6
+'[?]',    # 0xd7
+'[?]',    # 0xd8
+'[?]',    # 0xd9
+'[?]',    # 0xda
+'[?]',    # 0xdb
+'[?]',    # 0xdc
+'[?]',    # 0xdd
+'[?]',    # 0xde
+'[?]',    # 0xdf
+'[?]',    # 0xe0
+'[?]',    # 0xe1
+'[?]',    # 0xe2
+'[?]',    # 0xe3
+'[?]',    # 0xe4
+'[?]',    # 0xe5
+'[?]',    # 0xe6
+'[?]',    # 0xe7
+'[?]',    # 0xe8
+'[?]',    # 0xe9
+'[?]',    # 0xea
+'[?]',    # 0xeb
+'[?]',    # 0xec
+'[?]',    # 0xed
+'[?]',    # 0xee
+'[?]',    # 0xef
+'[?]',    # 0xf0
+'[?]',    # 0xf1
+'[?]',    # 0xf2
+'[?]',    # 0xf3
+'[?]',    # 0xf4
+'[?]',    # 0xf5
+'[?]',    # 0xf6
+'[?]',    # 0xf7
+'[?]',    # 0xf8
+'[?]',    # 0xf9
+'[?]',    # 0xfa
+'[?]',    # 0xfb
+'[?]',    # 0xfc
+'[?]',    # 0xfd
+'[?]',    # 0xfe
+)

lib/unidecode/x04e.py

@@ -0,0 +1,258 @@
+data = (
+'[?] ',    # 0x00
+'Ding ',    # 0x01
+'Kao ',    # 0x02
+'Qi ',    # 0x03
+'Shang ',    # 0x04
+'Xia ',    # 0x05
+'[?] ',    # 0x06
+'Mo ',    # 0x07
+'Zhang ',    # 0x08
+'San ',    # 0x09
+'Shang ',    # 0x0a
+'Xia ',    # 0x0b
+'Ji ',    # 0x0c
+'Bu ',    # 0x0d
+'Yu ',    # 0x0e
+'Mian ',    # 0x0f
+'Gai ',    # 0x10
+'Chou ',    # 0x11
+'Chou ',    # 0x12
+'Zhuan ',    # 0x13
+'Qie ',    # 0x14
+'Pi ',    # 0x15
+'Shi ',    # 0x16
+'Shi ',    # 0x17
+'Qiu ',    # 0x18
+'Bing ',    # 0x19
+'Ye ',    # 0x1a
+'Cong ',    # 0x1b
+'Dong ',    # 0x1c
+'Si ',    # 0x1d
+'Cheng ',    # 0x1e
+'Diu ',    # 0x1f
+'Qiu ',    # 0x20
+'Liang ',    # 0x21
+'Diu ',    # 0x22
+'You ',    # 0x23
+'Liang ',    # 0x24
+'Yan ',    # 0x25
+'Bing ',    # 0x26
+'Sang ',    # 0x27
+'Gun ',    # 0x28
+'Jiu ',    # 0x29
+'Ge ',    # 0x2a
+'Ya ',    # 0x2b
+'Qiang ',    # 0x2c
+'Zhong ',    # 0x2d
+'Ji ',    # 0x2e
+'Jie ',    # 0x2f
+'Feng ',    # 0x30
+'Guan ',    # 0x31
+'Chuan ',    # 0x32
+'Chan ',    # 0x33
+'Lin ',    # 0x34
+'Zhuo ',    # 0x35
+'Zhu ',    # 0x36
+'Ha ',    # 0x37
+'Wan ',    # 0x38
+'Dan ',    # 0x39
+'Wei ',    # 0x3a
+'Zhu ',    # 0x3b
+'Jing ',    # 0x3c
+'Li ',    # 0x3d
+'Ju ',    # 0x3e
+'Pie ',    # 0x3f
+'Fu ',    # 0x40
+'Yi ',    # 0x41
+'Yi ',    # 0x42
+'Nai ',    # 0x43
+'Shime ',    # 0x44
+'Jiu ',    # 0x45
+'Jiu ',    # 0x46
+'Zhe ',    # 0x47
+'Yao ',    # 0x48
+'Yi ',    # 0x49
+'[?] ',    # 0x4a
+'Zhi ',    # 0x4b
+'Wu ',    # 0x4c
+'Zha ',    # 0x4d
+'Hu ',    # 0x4e
+'Fa ',    # 0x4f
+'Le ',    # 0x50
+'Zhong ',    # 0x51
+'Ping ',    # 0x52
+'Pang ',    # 0x53
+'Qiao ',    # 0x54
+'Hu ',    # 0x55
+'Guai ',    # 0x56
+'Cheng ',    # 0x57
+'Cheng ',    # 0x58
+'Yi ',    # 0x59
+'Yin ',    # 0x5a
+'[?] ',    # 0x5b
+'Mie ',    # 0x5c
+'Jiu ',    # 0x5d
+'Qi ',    # 0x5e
+'Ye ',    # 0x5f
+'Xi ',    # 0x60
+'Xiang ',    # 0x61
+'Gai ',    # 0x62
+'Diu ',    # 0x63
+'Hal ',    # 0x64
+'[?] ',    # 0x65
+'Shu ',    # 0x66
+'Twul ',    # 0x67
+'Shi ',    # 0x68
+'Ji ',    # 0x69
+'Nang ',    # 0x6a
+'Jia ',    # 0x6b
+'Kel ',    # 0x6c
+'Shi ',    # 0x6d
+'[?] ',    # 0x6e
+'Ol ',    # 0x6f
+'Mai ',    # 0x70
+'Luan ',    # 0x71
+'Cal ',    # 0x72
+'Ru ',    # 0x73
+'Xue ',    # 0x74
+'Yan ',    # 0x75
+'Fu ',    # 0x76
+'Sha ',    # 0x77
+'Na ',    # 0x78
+'Gan ',    # 0x79
+'Sol ',    # 0x7a
+'El ',    # 0x7b
+'Cwul ',    # 0x7c
+'[?] ',    # 0x7d
+'Gan ',    # 0x7e
+'Chi ',    # 0x7f
+'Gui ',    # 0x80
+'Gan ',    # 0x81
+'Luan ',    # 0x82
+'Lin ',    # 0x83
+'Yi ',    # 0x84
+'Jue ',    # 0x85
+'Liao ',    # 0x86
+'Ma ',    # 0x87
+'Yu ',    # 0x88
+'Zheng ',    # 0x89
+'Shi ',    # 0x8a
+'Shi ',    # 0x8b
+'Er ',    # 0x8c
+'Chu ',    # 0x8d
+'Yu ',    # 0x8e
+'Yu ',    # 0x8f
+'Yu ',    # 0x90
+'Yun ',    # 0x91
+'Hu ',    # 0x92
+'Qi ',    # 0x93
+'Wu ',    # 0x94
+'Jing ',    # 0x95
+'Si ',    # 0x96
+'Sui ',    # 0x97
+'Gen ',    # 0x98
+'Gen ',    # 0x99
+'Ya ',    # 0x9a
+'Xie ',    # 0x9b
+'Ya ',    # 0x9c
+'Qi ',    # 0x9d
+'Ya ',    # 0x9e
+'Ji ',    # 0x9f
+'Tou ',    # 0xa0
+'Wang ',    # 0xa1
+'Kang ',    # 0xa2
+'Ta ',    # 0xa3
+'Jiao ',    # 0xa4
+'Hai ',    # 0xa5
+'Yi ',    # 0xa6
+'Chan ',    # 0xa7
+'Heng ',    # 0xa8
+'Mu ',    # 0xa9
+'[?] ',    # 0xaa
+'Xiang ',    # 0xab
+'Jing ',    # 0xac
+'Ting ',    # 0xad
+'Liang ',    # 0xae
+'Xiang ',    # 0xaf
+'Jing ',    # 0xb0
+'Ye ',    # 0xb1
+'Qin ',    # 0xb2
+'Bo ',    # 0xb3
+'You ',    # 0xb4
+'Xie ',    # 0xb5
+'Dan ',    # 0xb6
+'Lian ',    # 0xb7
+'Duo ',    # 0xb8
+'Wei ',    # 0xb9
+'Ren ',    # 0xba
+'Ren ',    # 0xbb
+'Ji ',    # 0xbc
+'La ',    # 0xbd
+'Wang ',    # 0xbe
+'Yi ',    # 0xbf
+'Shi ',    # 0xc0
+'Ren ',    # 0xc1
+'Le ',    # 0xc2
+'Ding ',    # 0xc3
+'Ze ',    # 0xc4
+'Jin ',    # 0xc5
+'Pu ',    # 0xc6
+'Chou ',    # 0xc7
+'Ba ',    # 0xc8
+'Zhang ',    # 0xc9
+'Jin ',    # 0xca
+'Jie ',    # 0xcb
+'Bing ',    # 0xcc
+'Reng ',    # 0xcd
+'Cong ',    # 0xce
+'Fo ',    # 0xcf
+'San ',    # 0xd0
+'Lun ',    # 0xd1
+'Sya ',    # 0xd2
+'Cang ',    # 0xd3
+'Zi ',    # 0xd4
+'Shi ',    # 0xd5
+'Ta ',    # 0xd6
+'Zhang ',    # 0xd7
+'Fu ',    # 0xd8
+'Xian ',    # 0xd9
+'Xian ',    # 0xda
+'Tuo ',    # 0xdb
+'Hong ',    # 0xdc
+'Tong ',    # 0xdd
+'Ren ',    # 0xde
+'Qian ',    # 0xdf
+'Gan ',    # 0xe0
+'Yi ',    # 0xe1
+'Di ',    # 0xe2
+'Dai ',    # 0xe3
+'Ling ',    # 0xe4
+'Yi ',    # 0xe5
+'Chao ',    # 0xe6
+'Chang ',    # 0xe7
+'Sa ',    # 0xe8
+'[?] ',    # 0xe9
+'Yi ',    # 0xea
+'Mu ',    # 0xeb
+'Men ',    # 0xec
+'Ren ',    # 0xed
+'Jia ',    # 0xee
+'Chao ',    # 0xef
+'Yang ',    # 0xf0
+'Qian ',    # 0xf1
+'Zhong ',    # 0xf2
+'Pi ',    # 0xf3
+'Wan ',    # 0xf4
+'Wu ',    # 0xf5
+'Jian ',    # 0xf6
+'Jie ',    # 0xf7
+'Yao ',    # 0xf8
+'Feng ',    # 0xf9
+'Cang ',    # 0xfa
+'Ren ',    # 0xfb
+'Wang ',    # 0xfc
+'Fen ',    # 0xfd
+'Di ',    # 0xfe
+'Fang ',    # 0xff
+)

lib/unidecode/x04f.py

@@ -0,0 +1,258 @@
+data = (
+'Zhong ',    # 0x00
+'Qi ',    # 0x01
+'Pei ',    # 0x02
+'Yu ',    # 0x03
+'Diao ',    # 0x04
+'Dun ',    # 0x05
+'Wen ',    # 0x06
+'Yi ',    # 0x07
+'Xin ',    # 0x08
+'Kang ',    # 0x09
+'Yi ',    # 0x0a
+'Ji ',    # 0x0b
+'Ai ',    # 0x0c
+'Wu ',    # 0x0d
+'Ji ',    # 0x0e
+'Fu ',    # 0x0f
+'Fa ',    # 0x10
+'Xiu ',    # 0x11
+'Jin ',    # 0x12
+'Bei ',    # 0x13
+'Dan ',    # 0x14
+'Fu ',    # 0x15
+'Tang ',    # 0x16
+'Zhong ',    # 0x17
+'You ',    # 0x18
+'Huo ',    # 0x19
+'Hui ',    # 0x1a
+'Yu ',    # 0x1b
+'Cui ',    # 0x1c
+'Chuan ',    # 0x1d
+'San ',    # 0x1e
+'Wei ',    # 0x1f
+'Chuan ',    # 0x20
+'Che ',    # 0x21
+'Ya ',    # 0x22
+'Xian ',    # 0x23
+'Shang ',    # 0x24
+'Chang ',    # 0x25
+'Lun ',    # 0x26
+'Cang ',    # 0x27
+'Xun ',    # 0x28
+'Xin ',    # 0x29
+'Wei ',    # 0x2a
+'Zhu ',    # 0x2b
+'[?] ',    # 0x2c
+'Xuan ',    # 0x2d
+'Nu ',    # 0x2e
+'Bo ',    # 0x2f
+'Gu ',    # 0x30
+'Ni ',    # 0x31
+'Ni ',    # 0x32
+'Xie ',    # 0x33
+'Ban ',    # 0x34
+'Xu ',    # 0x35
+'Ling ',    # 0x36
+'Zhou ',    # 0x37
+'Shen ',    # 0x38
+'Qu ',    # 0x39
+'Si ',    # 0x3a
+'Beng ',    # 0x3b
+'Si ',    # 0x3c
+'Jia ',    # 0x3d
+'Pi ',    # 0x3e
+'Yi ',    # 0x3f
+'Si ',    # 0x40
+'Ai ',    # 0x41
+'Zheng ',    # 0x42
+'Dian ',    # 0x43
+'Han ',    # 0x44
+'Mai ',    # 0x45
+'Dan ',    # 0x46
+'Zhu ',    # 0x47
+'Bu ',    # 0x48
+'Qu ',    # 0x49
+'Bi ',    # 0x4a
+'Shao ',    # 0x4b
+'Ci ',    # 0x4c
+'Wei ',    # 0x4d
+'Di ',    # 0x4e
+'Zhu ',    # 0x4f
+'Zuo ',    # 0x50
+'You ',    # 0x51
+'Yang ',    # 0x52
+'Ti ',    # 0x53
+'Zhan ',    # 0x54
+'He ',    # 0x55
+'Bi ',    # 0x56
+'Tuo ',    # 0x57
+'She ',    # 0x58
+'Yu ',    # 0x59
+'Yi ',    # 0x5a
+'Fo ',    # 0x5b
+'Zuo ',    # 0x5c
+'Kou ',    # 0x5d
+'Ning ',    # 0x5e
+'Tong ',    # 0x5f
+'Ni ',    # 0x60
+'Xuan ',    # 0x61
+'Qu ',    # 0x62
+'Yong ',    # 0x63
+'Wa ',    # 0x64
+'Qian ',    # 0x65
+'[?] ',    # 0x66
+'Ka ',    # 0x67
+'[?] ',    # 0x68
+'Pei ',    # 0x69
+'Huai ',    # 0x6a
+'He ',    # 0x6b
+'Lao ',    # 0x6c
+'Xiang ',    # 0x6d
+'Ge ',    # 0x6e
+'Yang ',    # 0x6f
+'Bai ',    # 0x70
+'Fa ',    # 0x71
+'Ming ',    # 0x72
+'Jia ',    # 0x73
+'Er ',    # 0x74
+'Bing ',    # 0x75
+'Ji ',    # 0x76
+'Hen ',    # 0x77
+'Huo ',    # 0x78
+'Gui ',    # 0x79
+'Quan ',    # 0x7a
+'Tiao ',    # 0x7b
+'Jiao ',    # 0x7c
+'Ci ',    # 0x7d
+'Yi ',    # 0x7e
+'Shi ',    # 0x7f
+'Xing ',    # 0x80
+'Shen ',    # 0x81
+'Tuo ',    # 0x82
+'Kan ',    # 0x83
+'Zhi ',    # 0x84
+'Gai ',    # 0x85
+'Lai ',    # 0x86
+'Yi ',    # 0x87
+'Chi ',    # 0x88
+'Kua ',    # 0x89
+'Guang ',    # 0x8a
+'Li ',    # 0x8b
+'Yin ',    # 0x8c
+'Shi ',    # 0x8d
+'Mi ',    # 0x8e
+'Zhu ',    # 0x8f
+'Xu ',    # 0x90
+'You ',    # 0x91
+'An ',    # 0x92
+'Lu ',    # 0x93
+'Mou ',    # 0x94
+'Er ',    # 0x95
+'Lun ',    # 0x96
+'Tong ',    # 0x97
+'Cha ',    # 0x98
+'Chi ',    # 0x99
+'Xun ',    # 0x9a
+'Gong ',    # 0x9b
+'Zhou ',    # 0x9c
+'Yi ',    # 0x9d
+'Ru ',    # 0x9e
+'Jian ',    # 0x9f
+'Xia ',    # 0xa0
+'Jia ',    # 0xa1
+'Zai ',    # 0xa2
+'Lu ',    # 0xa3
+'Ko ',    # 0xa4
+'Jiao ',    # 0xa5
+'Zhen ',    # 0xa6
+'Ce ',    # 0xa7
+'Qiao ',    # 0xa8
+'Kuai ',    # 0xa9
+'Chai ',    # 0xaa
+'Ning ',    # 0xab
+'Nong ',    # 0xac
+'Jin ',    # 0xad
+'Wu ',    # 0xae
+'Hou ',    # 0xaf
+'Jiong ',    # 0xb0
+'Cheng ',    # 0xb1
+'Zhen ',    # 0xb2
+'Zuo ',    # 0xb3
+'Chou ',    # 0xb4
+'Qin ',    # 0xb5
+'Lu ',    # 0xb6
+'Ju ',    # 0xb7
+'Shu ',    # 0xb8
+'Ting ',    # 0xb9
+'Shen ',    # 0xba
+'Tuo ',    # 0xbb
+'Bo ',    # 0xbc
+'Nan ',    # 0xbd
+'Hao ',    # 0xbe
+'Bian ',    # 0xbf
+'Tui ',    # 0xc0
+'Yu ',    # 0xc1
+'Xi ',    # 0xc2
+'Cu ',    # 0xc3
+'E ',    # 0xc4
+'Qiu ',    # 0xc5
+'Xu ',    # 0xc6
+'Kuang ',    # 0xc7
+'Ku ',    # 0xc8
+'Wu ',    # 0xc9
+'Jun ',    # 0xca
+'Yi ',    # 0xcb
+'Fu ',    # 0xcc
+'Lang ',    # 0xcd
+'Zu ',    # 0xce
+'Qiao ',    # 0xcf
+'Li ',    # 0xd0
+'Yong ',    # 0xd1
+'Hun ',    # 0xd2
+'Jing ',    # 0xd3
+'Xian ',    # 0xd4
+'San ',    # 0xd5
+'Pai ',    # 0xd6
+'Su ',    # 0xd7
+'Fu ',    # 0xd8
+'Xi ',    # 0xd9
+'Li ',    # 0xda
+'Fu ',    # 0xdb
+'Ping ',    # 0xdc
+'Bao ',    # 0xdd
+'Yu ',    # 0xde
+'Si ',    # 0xdf
+'Xia ',    # 0xe0
+'Xin ',    # 0xe1
+'Xiu ',    # 0xe2
+'Yu ',    # 0xe3
+'Ti ',    # 0xe4
+'Che ',    # 0xe5
+'Chou ',    # 0xe6
+'[?] ',    # 0xe7
+'Yan ',    # 0xe8
+'Lia ',    # 0xe9
+'Li ',    # 0xea
+'Lai ',    # 0xeb
+'[?] ',    # 0xec
+'Jian ',    # 0xed
+'Xiu ',    # 0xee
+'Fu ',    # 0xef
+'He ',    # 0xf0
+'Ju ',    # 0xf1
+'Xiao ',    # 0xf2
+'Pai ',    # 0xf3
+'Jian ',    # 0xf4
+'Biao ',    # 0xf5
+'Chu ',    # 0xf6
+'Fei ',    # 0xf7
+'Feng ',    # 0xf8
+'Ya ',    # 0xf9
+'An ',    # 0xfa
+'Bei ',    # 0xfb
+'Yu ',    # 0xfc
+'Xin ',    # 0xfd
+'Bi ',    # 0xfe
+'Jian ',    # 0xff
+)

lib/unidecode/x050.py

@@ -0,0 +1,258 @@
+data = (
+'Chang ',    # 0x00
+'Chi ',    # 0x01
+'Bing ',    # 0x02
+'Zan ',    # 0x03
+'Yao ',    # 0x04
+'Cui ',    # 0x05
+'Lia ',    # 0x06
+'Wan ',    # 0x07
+'Lai ',    # 0x08
+'Cang ',    # 0x09
+'Zong ',    # 0x0a
+'Ge ',    # 0x0b
+'Guan ',    # 0x0c
+'Bei ',    # 0x0d
+'Tian ',    # 0x0e
+'Shu ',    # 0x0f
+'Shu ',    # 0x10
+'Men ',    # 0x11
+'Dao ',    # 0x12
+'Tan ',    # 0x13
+'Jue ',    # 0x14
+'Chui ',    # 0x15
+'Xing ',    # 0x16
+'Peng ',    # 0x17
+'Tang ',    # 0x18
+'Hou ',    # 0x19
+'Yi ',    # 0x1a
+'Qi ',    # 0x1b
+'Ti ',    # 0x1c
+'Gan ',    # 0x1d
+'Jing ',    # 0x1e
+'Jie ',    # 0x1f
+'Sui ',    # 0x20
+'Chang ',    # 0x21
+'Jie ',    # 0x22
+'Fang ',    # 0x23
+'Zhi ',    # 0x24
+'Kong ',    # 0x25
+'Juan ',    # 0x26
+'Zong ',    # 0x27
+'Ju ',    # 0x28
+'Qian ',    # 0x29
+'Ni ',    # 0x2a
+'Lun ',    # 0x2b
+'Zhuo ',    # 0x2c
+'Wei ',    # 0x2d
+'Luo ',    # 0x2e
+'Song ',    # 0x2f
+'Leng ',    # 0x30
+'Hun ',    # 0x31
+'Dong ',    # 0x32
+'Zi ',    # 0x33
+'Ben ',    # 0x34
+'Wu ',    # 0x35
+'Ju ',    # 0x36
+'Nai ',    # 0x37
+'Cai ',    # 0x38
+'Jian ',    # 0x39
+'Zhai ',    # 0x3a
+'Ye ',    # 0x3b
+'Zhi ',    # 0x3c
+'Sha ',    # 0x3d
+'Qing ',    # 0x3e
+'[?] ',    # 0x3f
+'Ying ',    # 0x40
+'Cheng ',    # 0x41
+'Jian ',    # 0x42
+'Yan ',    # 0x43
+'Nuan ',    # 0x44
+'Zhong ',    # 0x45
+'Chun ',    # 0x46
+'Jia ',    # 0x47
+'Jie ',    # 0x48
+'Wei ',    # 0x49
+'Yu ',    # 0x4a
+'Bing ',    # 0x4b
+'Ruo ',    # 0x4c
+'Ti ',    # 0x4d
+'Wei ',    # 0x4e
+'Pian ',    # 0x4f
+'Yan ',    # 0x50
+'Feng ',    # 0x51
+'Tang ',    # 0x52
+'Wo ',    # 0x53
+'E ',    # 0x54
+'Xie ',    # 0x55
+'Che ',    # 0x56
+'Sheng ',    # 0x57
+'Kan ',    # 0x58
+'Di ',    # 0x59
+'Zuo ',    # 0x5a
+'Cha ',    # 0x5b
+'Ting ',    # 0x5c
+'Bei ',    # 0x5d
+'Ye ',    # 0x5e
+'Huang ',    # 0x5f
+'Yao ',    # 0x60
+'Zhan ',    # 0x61
+'Chou ',    # 0x62
+'Yan ',    # 0x63
+'You ',    # 0x64
+'Jian ',    # 0x65
+'Xu ',    # 0x66
+'Zha ',    # 0x67
+'Ci ',    # 0x68
+'Fu ',    # 0x69
+'Bi ',    # 0x6a
+'Zhi ',    # 0x6b
+'Zong ',    # 0x6c
+'Mian ',    # 0x6d
+'Ji ',    # 0x6e
+'Yi ',    # 0x6f
+'Xie ',    # 0x70
+'Xun ',    # 0x71
+'Si ',    # 0x72
+'Duan ',    # 0x73
+'Ce ',    # 0x74
+'Zhen ',    # 0x75
+'Ou ',    # 0x76
+'Tou ',    # 0x77
+'Tou ',    # 0x78
+'Bei ',    # 0x79
+'Za ',    # 0x7a
+'Lu ',    # 0x7b
+'Jie ',    # 0x7c
+'Wei ',    # 0x7d
+'Fen ',    # 0x7e
+'Chang ',    # 0x7f
+'Gui ',    # 0x80
+'Sou ',    # 0x81
+'Zhi ',    # 0x82
+'Su ',    # 0x83
+'Xia ',    # 0x84
+'Fu ',    # 0x85
+'Yuan ',    # 0x86
+'Rong ',    # 0x87
+'Li ',    # 0x88
+'Ru ',    # 0x89
+'Yun ',    # 0x8a
+'Gou ',    # 0x8b
+'Ma ',    # 0x8c
+'Bang ',    # 0x8d
+'Dian ',    # 0x8e
+'Tang ',    # 0x8f
+'Hao ',    # 0x90
+'Jie ',    # 0x91
+'Xi ',    # 0x92
+'Shan ',    # 0x93
+'Qian ',    # 0x94
+'Jue ',    # 0x95
+'Cang ',    # 0x96
+'Chu ',    # 0x97
+'San ',    # 0x98
+'Bei ',    # 0x99
+'Xiao ',    # 0x9a
+'Yong ',    # 0x9b
+'Yao ',    # 0x9c
+'Tan ',    # 0x9d
+'Suo ',    # 0x9e
+'Yang ',    # 0x9f
+'Fa ',    # 0xa0
+'Bing ',    # 0xa1
+'Jia ',    # 0xa2
+'Dai ',    # 0xa3
+'Zai ',    # 0xa4
+'Tang ',    # 0xa5
+'[?] ',    # 0xa6
+'Bin ',    # 0xa7
+'Chu ',    # 0xa8
+'Nuo ',    # 0xa9
+'Can ',    # 0xaa
+'Lei ',    # 0xab
+'Cui ',    # 0xac
+'Yong ',    # 0xad
+'Zao ',    # 0xae
+'Zong ',    # 0xaf
+'Peng ',    # 0xb0
+'Song ',    # 0xb1
+'Ao ',    # 0xb2
+'Chuan ',    # 0xb3
+'Yu ',    # 0xb4
+'Zhai ',    # 0xb5
+'Cou ',    # 0xb6
+'Shang ',    # 0xb7
+'Qiang ',    # 0xb8
+'Jing ',    # 0xb9
+'Chi ',    # 0xba
+'Sha ',    # 0xbb
+'Han ',    # 0xbc
+'Zhang ',    # 0xbd
+'Qing ',    # 0xbe
+'Yan ',    # 0xbf
+'Di ',    # 0xc0
+'Xi ',    # 0xc1
+'Lu ',    # 0xc2
+'Bei ',    # 0xc3
+'Piao ',    # 0xc4
+'Jin ',    # 0xc5
+'Lian ',    # 0xc6
+'Lu ',    # 0xc7
+'Man ',    # 0xc8
+'Qian ',    # 0xc9
+'Xian ',    # 0xca
+'Tan ',    # 0xcb
+'Ying ',    # 0xcc
+'Dong ',    # 0xcd
+'Zhuan ',    # 0xce
+'Xiang ',    # 0xcf
+'Shan ',    # 0xd0
+'Qiao ',    # 0xd1
+'Jiong ',    # 0xd2
+'Tui ',    # 0xd3
+'Zun ',    # 0xd4
+'Pu ',    # 0xd5
+'Xi ',    # 0xd6
+'Lao ',    # 0xd7
+'Chang ',    # 0xd8
+'Guang ',    # 0xd9
+'Liao ',    # 0xda
+'Qi ',    # 0xdb
+'Deng ',    # 0xdc
+'Chan ',    # 0xdd
+'Wei ',    # 0xde
+'Ji ',    # 0xdf
+'Fan ',    # 0xe0
+'Hui ',    # 0xe1
+'Chuan ',    # 0xe2
+'Jian ',    # 0xe3
+'Dan ',    # 0xe4
+'Jiao ',    # 0xe5
+'Jiu ',    # 0xe6
+'Seng ',    # 0xe7
+'Fen ',    # 0xe8
+'Xian ',    # 0xe9
+'Jue ',    # 0xea
+'E ',    # 0xeb
+'Jiao ',    # 0xec
+'Jian ',    # 0xed
+'Tong ',    # 0xee
+'Lin ',    # 0xef
+'Bo ',    # 0xf0
+'Gu ',    # 0xf1
+'[?] ',    # 0xf2
+'Su ',    # 0xf3
+'Xian ',    # 0xf4
+'Jiang ',    # 0xf5
+'Min ',    # 0xf6
+'Ye ',    # 0xf7
+'Jin ',    # 0xf8
+'Jia ',    # 0xf9
+'Qiao ',    # 0xfa
+'Pi ',    # 0xfb
+'Feng ',    # 0xfc
+'Zhou ',    # 0xfd
+'Ai ',    # 0xfe
+'Sai ',    # 0xff
+)

lib/unidecode/x051.py

@@ -0,0 +1,258 @@
+data = (
+'Yi ',    # 0x00
+'Jun ',    # 0x01
+'Nong ',    # 0x02
+'Chan ',    # 0x03
+'Yi ',    # 0x04
+'Dang ',    # 0x05
+'Jing ',    # 0x06
+'Xuan ',    # 0x07
+'Kuai ',    # 0x08
+'Jian ',    # 0x09
+'Chu ',    # 0x0a
+'Dan ',    # 0x0b
+'Jiao ',    # 0x0c
+'Sha ',    # 0x0d
+'Zai ',    # 0x0e
+'[?] ',    # 0x0f
+'Bin ',    # 0x10
+'An ',    # 0x11
+'Ru ',    # 0x12
+'Tai ',    # 0x13
+'Chou ',    # 0x14
+'Chai ',    # 0x15
+'Lan ',    # 0x16
+'Ni ',    # 0x17
+'Jin ',    # 0x18
+'Qian ',    # 0x19
+'Meng ',    # 0x1a
+'Wu ',    # 0x1b
+'Ning ',    # 0x1c
+'Qiong ',    # 0x1d
+'Ni ',    # 0x1e
+'Chang ',    # 0x1f
+'Lie ',    # 0x20
+'Lei ',    # 0x21
+'Lu ',    # 0x22
+'Kuang ',    # 0x23
+'Bao ',    # 0x24
+'Du ',    # 0x25
+'Biao ',    # 0x26
+'Zan ',    # 0x27
+'Zhi ',    # 0x28
+'Si ',    # 0x29
+'You ',    # 0x2a
+'Hao ',    # 0x2b
+'Chen ',    # 0x2c
+'Chen ',    # 0x2d
+'Li ',    # 0x2e
+'Teng ',    # 0x2f
+'Wei ',    # 0x30
+'Long ',    # 0x31
+'Chu ',    # 0x32
+'Chan ',    # 0x33
+'Rang ',    # 0x34
+'Shu ',    # 0x35
+'Hui ',    # 0x36
+'Li ',    # 0x37
+'Luo ',    # 0x38
+'Zan ',    # 0x39
+'Nuo ',    # 0x3a
+'Tang ',    # 0x3b
+'Yan ',    # 0x3c
+'Lei ',    # 0x3d
+'Nang ',    # 0x3e
+'Er ',    # 0x3f
+'Wu ',    # 0x40
+'Yun ',    # 0x41
+'Zan ',    # 0x42
+'Yuan ',    # 0x43
+'Xiong ',    # 0x44
+'Chong ',    # 0x45
+'Zhao ',    # 0x46
+'Xiong ',    # 0x47
+'Xian ',    # 0x48
+'Guang ',    # 0x49
+'Dui ',    # 0x4a
+'Ke ',    # 0x4b
+'Dui ',    # 0x4c
+'Mian ',    # 0x4d
+'Tu ',    # 0x4e
+'Chang ',    # 0x4f
+'Er ',    # 0x50
+'Dui ',    # 0x51
+'Er ',    # 0x52
+'Xin ',    # 0x53
+'Tu ',    # 0x54
+'Si ',    # 0x55
+'Yan ',    # 0x56
+'Yan ',    # 0x57
+'Shi ',    # 0x58
+'Shi ',    # 0x59
+'Dang ',    # 0x5a
+'Qian ',    # 0x5b
+'Dou ',    # 0x5c
+'Fen ',    # 0x5d
+'Mao ',    # 0x5e
+'Shen ',    # 0x5f
+'Dou ',    # 0x60
+'Bai ',    # 0x61
+'Jing ',    # 0x62
+'Li ',    # 0x63
+'Huang ',    # 0x64
+'Ru ',    # 0x65
+'Wang ',    # 0x66
+'Nei ',    # 0x67
+'Quan ',    # 0x68
+'Liang ',    # 0x69
+'Yu ',    # 0x6a
+'Ba ',    # 0x6b
+'Gong ',    # 0x6c
+'Liu ',    # 0x6d
+'Xi ',    # 0x6e
+'[?] ',    # 0x6f
+'Lan ',    # 0x70
+'Gong ',    # 0x71
+'Tian ',    # 0x72
+'Guan ',    # 0x73
+'Xing ',    # 0x74
+'Bing ',    # 0x75
+'Qi ',    # 0x76
+'Ju ',    # 0x77
+'Dian ',    # 0x78
+'Zi ',    # 0x79
+'Ppwun ',    # 0x7a
+'Yang ',    # 0x7b
+'Jian ',    # 0x7c
+'Shou ',    # 0x7d
+'Ji ',    # 0x7e
+'Yi ',    # 0x7f
+'Ji ',    # 0x80
+'Chan ',    # 0x81
+'Jiong ',    # 0x82
+'Mao ',    # 0x83
+'Ran ',    # 0x84
+'Nei ',    # 0x85
+'Yuan ',    # 0x86
+'Mao ',    # 0x87
+'Gang ',    # 0x88
+'Ran ',    # 0x89
+'Ce ',    # 0x8a
+'Jiong ',    # 0x8b
+'Ce ',    # 0x8c
+'Zai ',    # 0x8d
+'Gua ',    # 0x8e
+'Jiong ',    # 0x8f
+'Mao ',    # 0x90
+'Zhou ',    # 0x91
+'Mou ',    # 0x92
+'Gou ',    # 0x93
+'Xu ',    # 0x94
+'Mian ',    # 0x95
+'Mi ',    # 0x96
+'Rong ',    # 0x97
+'Yin ',    # 0x98
+'Xie ',    # 0x99
+'Kan ',    # 0x9a
+'Jun ',    # 0x9b
+'Nong ',    # 0x9c
+'Yi ',    # 0x9d
+'Mi ',    # 0x9e
+'Shi ',    # 0x9f
+'Guan ',    # 0xa0
+'Meng ',    # 0xa1
+'Zhong ',    # 0xa2
+'Ju ',    # 0xa3
+'Yuan ',    # 0xa4
+'Ming ',    # 0xa5
+'Kou ',    # 0xa6
+'Lam ',    # 0xa7
+'Fu ',    # 0xa8
+'Xie ',    # 0xa9
+'Mi ',    # 0xaa
+'Bing ',    # 0xab
+'Dong ',    # 0xac
+'Tai ',    # 0xad
+'Gang ',    # 0xae
+'Feng ',    # 0xaf
+'Bing ',    # 0xb0
+'Hu ',    # 0xb1
+'Chong ',    # 0xb2
+'Jue ',    # 0xb3
+'Hu ',    # 0xb4
+'Kuang ',    # 0xb5
+'Ye ',    # 0xb6
+'Leng ',    # 0xb7
+'Pan ',    # 0xb8
+'Fu ',    # 0xb9
+'Min ',    # 0xba
+'Dong ',    # 0xbb
+'Xian ',    # 0xbc
+'Lie ',    # 0xbd
+'Xia ',    # 0xbe
+'Jian ',    # 0xbf
+'Jing ',    # 0xc0
+'Shu ',    # 0xc1
+'Mei ',    # 0xc2
+'Tu ',    # 0xc3
+'Qi ',    # 0xc4
+'Gu ',    # 0xc5
+'Zhun ',    # 0xc6
+'Song ',    # 0xc7
+'Jing ',    # 0xc8
+'Liang ',    # 0xc9
+'Qing ',    # 0xca
+'Diao ',    # 0xcb
+'Ling ',    # 0xcc
+'Dong ',    # 0xcd
+'Gan ',    # 0xce
+'Jian ',    # 0xcf
+'Yin ',    # 0xd0
+'Cou ',    # 0xd1
+'Yi ',    # 0xd2
+'Li ',    # 0xd3
+'Cang ',    # 0xd4
+'Ming ',    # 0xd5
+'Zhuen ',    # 0xd6
+'Cui ',    # 0xd7
+'Si ',    # 0xd8
+'Duo ',    # 0xd9
+'Jin ',    # 0xda
+'Lin ',    # 0xdb
+'Lin ',    # 0xdc
+'Ning ',    # 0xdd
+'Xi ',    # 0xde
+'Du ',    # 0xdf
+'Ji ',    # 0xe0
+'Fan ',    # 0xe1
+'Fan ',    # 0xe2
+'Fan ',    # 0xe3
+'Feng ',    # 0xe4
+'Ju ',    # 0xe5
+'Chu ',    # 0xe6
+'Tako ',    # 0xe7
+'Feng ',    # 0xe8
+'Mok ',    # 0xe9
+'Ci ',    # 0xea
+'Fu ',    # 0xeb
+'Feng ',    # 0xec
+'Ping ',    # 0xed
+'Feng ',    # 0xee
+'Kai ',    # 0xef
+'Huang ',    # 0xf0
+'Kai ',    # 0xf1
+'Gan ',    # 0xf2
+'Deng ',    # 0xf3
+'Ping ',    # 0xf4
+'Qu ',    # 0xf5
+'Xiong ',    # 0xf6
+'Kuai ',    # 0xf7
+'Tu ',    # 0xf8
+'Ao ',    # 0xf9
+'Chu ',    # 0xfa
+'Ji ',    # 0xfb
+'Dang ',    # 0xfc
+'Han ',    # 0xfd
+'Han ',    # 0xfe
+'Zao ',    # 0xff
+)

lib/unidecode/x052.py

@@ -0,0 +1,258 @@
+data = (
+'Dao ',    # 0x00
+'Diao ',    # 0x01
+'Dao ',    # 0x02
+'Ren ',    # 0x03
+'Ren ',    # 0x04
+'Chuang ',    # 0x05
+'Fen ',    # 0x06
+'Qie ',    # 0x07
+'Yi ',    # 0x08
+'Ji ',    # 0x09
+'Kan ',    # 0x0a
+'Qian ',    # 0x0b
+'Cun ',    # 0x0c
+'Chu ',    # 0x0d
+'Wen ',    # 0x0e
+'Ji ',    # 0x0f
+'Dan ',    # 0x10
+'Xing ',    # 0x11
+'Hua ',    # 0x12
+'Wan ',    # 0x13
+'Jue ',    # 0x14
+'Li ',    # 0x15
+'Yue ',    # 0x16
+'Lie ',    # 0x17
+'Liu ',    # 0x18
+'Ze ',    # 0x19
+'Gang ',    # 0x1a
+'Chuang ',    # 0x1b
+'Fu ',    # 0x1c
+'Chu ',    # 0x1d
+'Qu ',    # 0x1e
+'Ju ',    # 0x1f
+'Shan ',    # 0x20
+'Min ',    # 0x21
+'Ling ',    # 0x22
+'Zhong ',    # 0x23
+'Pan ',    # 0x24
+'Bie ',    # 0x25
+'Jie ',    # 0x26
+'Jie ',    # 0x27
+'Bao ',    # 0x28
+'Li ',    # 0x29
+'Shan ',    # 0x2a
+'Bie ',    # 0x2b
+'Chan ',    # 0x2c
+'Jing ',    # 0x2d
+'Gua ',    # 0x2e
+'Gen ',    # 0x2f
+'Dao ',    # 0x30
+'Chuang ',    # 0x31
+'Kui ',    # 0x32
+'Ku ',    # 0x33
+'Duo ',    # 0x34
+'Er ',    # 0x35
+'Zhi ',    # 0x36
+'Shua ',    # 0x37
+'Quan ',    # 0x38
+'Cha ',    # 0x39
+'Ci ',    # 0x3a
+'Ke ',    # 0x3b
+'Jie ',    # 0x3c
+'Gui ',    # 0x3d
+'Ci ',    # 0x3e
+'Gui ',    # 0x3f
+'Kai ',    # 0x40
+'Duo ',    # 0x41
+'Ji ',    # 0x42
+'Ti ',    # 0x43
+'Jing ',    # 0x44
+'Lou ',    # 0x45
+'Gen ',    # 0x46
+'Ze ',    # 0x47
+'Yuan ',    # 0x48
+'Cuo ',    # 0x49
+'Xue ',    # 0x4a
+'Ke ',    # 0x4b
+'La ',    # 0x4c
+'Qian ',    # 0x4d
+'Cha ',    # 0x4e
+'Chuang ',    # 0x4f
+'Gua ',    # 0x50
+'Jian ',    # 0x51
+'Cuo ',    # 0x52
+'Li ',    # 0x53
+'Ti ',    # 0x54
+'Fei ',    # 0x55
+'Pou ',    # 0x56
+'Chan ',    # 0x57
+'Qi ',    # 0x58
+'Chuang ',    # 0x59
+'Zi ',    # 0x5a
+'Gang ',    # 0x5b
+'Wan ',    # 0x5c
+'Bo ',    # 0x5d
+'Ji ',    # 0x5e
+'Duo ',    # 0x5f
+'Qing ',    # 0x60
+'Yan ',    # 0x61
+'Zhuo ',    # 0x62
+'Jian ',    # 0x63
+'Ji ',    # 0x64
+'Bo ',    # 0x65
+'Yan ',    # 0x66
+'Ju ',    # 0x67
+'Huo ',    # 0x68
+'Sheng ',    # 0x69
+'Jian ',    # 0x6a
+'Duo ',    # 0x6b
+'Duan ',    # 0x6c
+'Wu ',    # 0x6d
+'Gua ',    # 0x6e
+'Fu ',    # 0x6f
+'Sheng ',    # 0x70
+'Jian ',    # 0x71
+'Ge ',    # 0x72
+'Zha ',    # 0x73
+'Kai ',    # 0x74
+'Chuang ',    # 0x75
+'Juan ',    # 0x76
+'Chan ',    # 0x77
+'Tuan ',    # 0x78
+'Lu ',    # 0x79
+'Li ',    # 0x7a
+'Fou ',    # 0x7b
+'Shan ',    # 0x7c
+'Piao ',    # 0x7d
+'Kou ',    # 0x7e
+'Jiao ',    # 0x7f
+'Gua ',    # 0x80
+'Qiao ',    # 0x81
+'Jue ',    # 0x82
+'Hua ',    # 0x83
+'Zha ',    # 0x84
+'Zhuo ',    # 0x85
+'Lian ',    # 0x86
+'Ju ',    # 0x87
+'Pi ',    # 0x88
+'Liu ',    # 0x89
+'Gui ',    # 0x8a
+'Jiao ',    # 0x8b
+'Gui ',    # 0x8c
+'Jian ',    # 0x8d
+'Jian ',    # 0x8e
+'Tang ',    # 0x8f
+'Huo ',    # 0x90
+'Ji ',    # 0x91
+'Jian ',    # 0x92
+'Yi ',    # 0x93
+'Jian ',    # 0x94
+'Zhi ',    # 0x95
+'Chan ',    # 0x96
+'Cuan ',    # 0x97
+'Mo ',    # 0x98
+'Li ',    # 0x99
+'Zhu ',    # 0x9a
+'Li ',    # 0x9b
+'Ya ',    # 0x9c
+'Quan ',    # 0x9d
+'Ban ',    # 0x9e
+'Gong ',    # 0x9f
+'Jia ',    # 0xa0
+'Wu ',    # 0xa1
+'Mai ',    # 0xa2
+'Lie ',    # 0xa3
+'Jin ',    # 0xa4
+'Keng ',    # 0xa5
+'Xie ',    # 0xa6
+'Zhi ',    # 0xa7
+'Dong ',    # 0xa8
+'Zhu ',    # 0xa9
+'Nu ',    # 0xaa
+'Jie ',    # 0xab
+'Qu ',    # 0xac
+'Shao ',    # 0xad
+'Yi ',    # 0xae
+'Zhu ',    # 0xaf
+'Miao ',    # 0xb0
+'Li ',    # 0xb1
+'Jing ',    # 0xb2
+'Lao ',    # 0xb3
+'Lao ',    # 0xb4
+'Juan ',    # 0xb5
+'Kou ',    # 0xb6
+'Yang ',    # 0xb7
+'Wa ',    # 0xb8
+'Xiao ',    # 0xb9
+'Mou ',    # 0xba
+'Kuang ',    # 0xbb
+'Jie ',    # 0xbc
+'Lie ',    # 0xbd
+'He ',    # 0xbe
+'Shi ',    # 0xbf
+'Ke ',    # 0xc0
+'Jing ',    # 0xc1
+'Hao ',    # 0xc2
+'Bo ',    # 0xc3
+'Min ',    # 0xc4
+'Chi ',    # 0xc5
+'Lang ',    # 0xc6
+'Yong ',    # 0xc7
+'Yong ',    # 0xc8
+'Mian ',    # 0xc9
+'Ke ',    # 0xca
+'Xun ',    # 0xcb
+'Juan ',    # 0xcc
+'Qing ',    # 0xcd
+'Lu ',    # 0xce
+'Pou ',    # 0xcf
+'Meng ',    # 0xd0
+'Lai ',    # 0xd1
+'Le ',    # 0xd2
+'Kai ',    # 0xd3
+'Mian ',    # 0xd4
+'Dong ',    # 0xd5
+'Xu ',    # 0xd6
+'Xu ',    # 0xd7
+'Kan ',    # 0xd8
+'Wu ',    # 0xd9
+'Yi ',    # 0xda
+'Xun ',    # 0xdb
+'Weng ',    # 0xdc
+'Sheng ',    # 0xdd
+'Lao ',    # 0xde
+'Mu ',    # 0xdf
+'Lu ',    # 0xe0
+'Piao ',    # 0xe1
+'Shi ',    # 0xe2
+'Ji ',    # 0xe3
+'Qin ',    # 0xe4
+'Qiang ',    # 0xe5
+'Jiao ',    # 0xe6
+'Quan ',    # 0xe7
+'Yang ',    # 0xe8
+'Yi ',    # 0xe9
+'Jue ',    # 0xea
+'Fan ',    # 0xeb
+'Juan ',    # 0xec
+'Tong ',    # 0xed
+'Ju ',    # 0xee
+'Dan ',    # 0xef
+'Xie ',    # 0xf0
+'Mai ',    # 0xf1
+'Xun ',    # 0xf2
+'Xun ',    # 0xf3
+'Lu ',    # 0xf4
+'Li ',    # 0xf5
+'Che ',    # 0xf6
+'Rang ',    # 0xf7
+'Quan ',    # 0xf8
+'Bao ',    # 0xf9
+'Shao ',    # 0xfa
+'Yun ',    # 0xfb
+'Jiu ',    # 0xfc
+'Bao ',    # 0xfd
+'Gou ',    # 0xfe
+'Wu ',    # 0xff
+)

lib/unidecode/x053.py

@@ -0,0 +1,258 @@
+data = (
+'Yun ',    # 0x00
+'Mwun ',    # 0x01
+'Nay ',    # 0x02
+'Gai ',    # 0x03
+'Gai ',    # 0x04
+'Bao ',    # 0x05
+'Cong ',    # 0x06
+'[?] ',    # 0x07
+'Xiong ',    # 0x08
+'Peng ',    # 0x09
+'Ju ',    # 0x0a
+'Tao ',    # 0x0b
+'Ge ',    # 0x0c
+'Pu ',    # 0x0d
+'An ',    # 0x0e
+'Pao ',    # 0x0f
+'Fu ',    # 0x10
+'Gong ',    # 0x11
+'Da ',    # 0x12
+'Jiu ',    # 0x13
+'Qiong ',    # 0x14
+'Bi ',    # 0x15
+'Hua ',    # 0x16
+'Bei ',    # 0x17
+'Nao ',    # 0x18
+'Chi ',    # 0x19
+'Fang ',    # 0x1a
+'Jiu ',    # 0x1b
+'Yi ',    # 0x1c
+'Za ',    # 0x1d
+'Jiang ',    # 0x1e
+'Kang ',    # 0x1f
+'Jiang ',    # 0x20
+'Kuang ',    # 0x21
+'Hu ',    # 0x22
+'Xia ',    # 0x23
+'Qu ',    # 0x24
+'Bian ',    # 0x25
+'Gui ',    # 0x26
+'Qie ',    # 0x27
+'Zang ',    # 0x28
+'Kuang ',    # 0x29
+'Fei ',    # 0x2a
+'Hu ',    # 0x2b
+'Tou ',    # 0x2c
+'Gui ',    # 0x2d
+'Gui ',    # 0x2e
+'Hui ',    # 0x2f
+'Dan ',    # 0x30
+'Gui ',    # 0x31
+'Lian ',    # 0x32
+'Lian ',    # 0x33
+'Suan ',    # 0x34
+'Du ',    # 0x35
+'Jiu ',    # 0x36
+'Qu ',    # 0x37
+'Xi ',    # 0x38
+'Pi ',    # 0x39
+'Qu ',    # 0x3a
+'Yi ',    # 0x3b
+'Qia ',    # 0x3c
+'Yan ',    # 0x3d
+'Bian ',    # 0x3e
+'Ni ',    # 0x3f
+'Qu ',    # 0x40
+'Shi ',    # 0x41
+'Xin ',    # 0x42
+'Qian ',    # 0x43
+'Nian ',    # 0x44
+'Sa ',    # 0x45
+'Zu ',    # 0x46
+'Sheng ',    # 0x47
+'Wu ',    # 0x48
+'Hui ',    # 0x49
+'Ban ',    # 0x4a
+'Shi ',    # 0x4b
+'Xi ',    # 0x4c
+'Wan ',    # 0x4d
+'Hua ',    # 0x4e
+'Xie ',    # 0x4f
+'Wan ',    # 0x50
+'Bei ',    # 0x51
+'Zu ',    # 0x52
+'Zhuo ',    # 0x53
+'Xie ',    # 0x54
+'Dan ',    # 0x55
+'Mai ',    # 0x56
+'Nan ',    # 0x57
+'Dan ',    # 0x58
+'Ji ',    # 0x59
+'Bo ',    # 0x5a
+'Shuai ',    # 0x5b
+'Bu ',    # 0x5c
+'Kuang ',    # 0x5d
+'Bian ',    # 0x5e
+'Bu ',    # 0x5f
+'Zhan ',    # 0x60
+'Qia ',    # 0x61
+'Lu ',    # 0x62
+'You ',    # 0x63
+'Lu ',    # 0x64
+'Xi ',    # 0x65
+'Gua ',    # 0x66
+'Wo ',    # 0x67
+'Xie ',    # 0x68
+'Jie ',    # 0x69
+'Jie ',    # 0x6a
+'Wei ',    # 0x6b
+'Ang ',    # 0x6c
+'Qiong ',    # 0x6d
+'Zhi ',    # 0x6e
+'Mao ',    # 0x6f
+'Yin ',    # 0x70
+'Wei ',    # 0x71
+'Shao ',    # 0x72
+'Ji ',    # 0x73
+'Que ',    # 0x74
+'Luan ',    # 0x75
+'Shi ',    # 0x76
+'Juan ',    # 0x77
+'Xie ',    # 0x78
+'Xu ',    # 0x79
+'Jin ',    # 0x7a
+'Que ',    # 0x7b
+'Wu ',    # 0x7c
+'Ji ',    # 0x7d
+'E ',    # 0x7e
+'Qing ',    # 0x7f
+'Xi ',    # 0x80
+'[?] ',    # 0x81
+'Han ',    # 0x82
+'Zhan ',    # 0x83
+'E ',    # 0x84
+'Ting ',    # 0x85
+'Li ',    # 0x86
+'Zhe ',    # 0x87
+'Han ',    # 0x88
+'Li ',    # 0x89
+'Ya ',    # 0x8a
+'Ya ',    # 0x8b
+'Yan ',    # 0x8c
+'She ',    # 0x8d
+'Zhi ',    # 0x8e
+'Zha ',    # 0x8f
+'Pang ',    # 0x90
+'[?] ',    # 0x91
+'He ',    # 0x92
+'Ya ',    # 0x93
+'Zhi ',    # 0x94
+'Ce ',    # 0x95
+'Pang ',    # 0x96
+'Ti ',    # 0x97
+'Li ',    # 0x98
+'She ',    # 0x99
+'Hou ',    # 0x9a
+'Ting ',    # 0x9b
+'Zui ',    # 0x9c
+'Cuo ',    # 0x9d
+'Fei ',    # 0x9e
+'Yuan ',    # 0x9f
+'Ce ',    # 0xa0
+'Yuan ',    # 0xa1
+'Xiang ',    # 0xa2
+'Yan ',    # 0xa3
+'Li ',    # 0xa4
+'Jue ',    # 0xa5
+'Sha ',    # 0xa6
+'Dian ',    # 0xa7
+'Chu ',    # 0xa8
+'Jiu ',    # 0xa9
+'Qin ',    # 0xaa
+'Ao ',    # 0xab
+'Gui ',    # 0xac
+'Yan ',    # 0xad
+'Si ',    # 0xae
+'Li ',    # 0xaf
+'Chang ',    # 0xb0
+'Lan ',    # 0xb1
+'Li ',    # 0xb2
+'Yan ',    # 0xb3
+'Yan ',    # 0xb4
+'Yuan ',    # 0xb5
+'Si ',    # 0xb6
+'Gong ',    # 0xb7
+'Lin ',    # 0xb8
+'Qiu ',    # 0xb9
+'Qu ',    # 0xba
+'Qu ',    # 0xbb
+'Uk ',    # 0xbc
+'Lei ',    # 0xbd
+'Du ',    # 0xbe
+'Xian ',    # 0xbf
+'Zhuan ',    # 0xc0
+'San ',    # 0xc1
+'Can ',    # 0xc2
+'Can ',    # 0xc3
+'Can ',    # 0xc4
+'Can ',    # 0xc5
+'Ai ',    # 0xc6
+'Dai ',    # 0xc7
+'You ',    # 0xc8
+'Cha ',    # 0xc9
+'Ji ',    # 0xca
+'You ',    # 0xcb
+'Shuang ',    # 0xcc
+'Fan ',    # 0xcd
+'Shou ',    # 0xce
+'Guai ',    # 0xcf
+'Ba ',    # 0xd0
+'Fa ',    # 0xd1
+'Ruo ',    # 0xd2
+'Shi ',    # 0xd3
+'Shu ',    # 0xd4
+'Zhuo ',    # 0xd5
+'Qu ',    # 0xd6
+'Shou ',    # 0xd7
+'Bian ',    # 0xd8
+'Xu ',    # 0xd9
+'Jia ',    # 0xda
+'Pan ',    # 0xdb
+'Sou ',    # 0xdc
+'Gao ',    # 0xdd
+'Wei ',    # 0xde
+'Sou ',    # 0xdf
+'Die ',    # 0xe0
+'Rui ',    # 0xe1
+'Cong ',    # 0xe2
+'Kou ',    # 0xe3
+'Gu ',    # 0xe4
+'Ju ',    # 0xe5
+'Ling ',    # 0xe6
+'Gua ',    # 0xe7
+'Tao ',    # 0xe8
+'Kou ',    # 0xe9
+'Zhi ',    # 0xea
+'Jiao ',    # 0xeb
+'Zhao ',    # 0xec
+'Ba ',    # 0xed
+'Ding ',    # 0xee
+'Ke ',    # 0xef
+'Tai ',    # 0xf0
+'Chi ',    # 0xf1
+'Shi ',    # 0xf2
+'You ',    # 0xf3
+'Qiu ',    # 0xf4
+'Po ',    # 0xf5
+'Xie ',    # 0xf6
+'Hao ',    # 0xf7
+'Si ',    # 0xf8
+'Tan ',    # 0xf9
+'Chi ',    # 0xfa
+'Le ',    # 0xfb
+'Diao ',    # 0xfc
+'Ji ',    # 0xfd
+'[?] ',    # 0xfe
+'Hong ',    # 0xff
+)