Initial python3 changes

Mostly just updating libraries, removing string encoding/decoding,
fixing some edge cases. No new functionality was added in this
commit.

lib/beets/importer.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # This file is part of beets.
 # Copyright 2016, Adrian Sampson.
 #
@@ -13,7 +12,6 @@
 # The above copyright notice and this permission notice shall be
 # included in all copies or substantial portions of the Software.
 
-from __future__ import division, absolute_import, print_function
 
 """Provides the basic, interface-agnostic workflow for importing and
 autotagging music files.
@@ -37,10 +35,10 @@ from beets import dbcore
 from beets import plugins
 from beets import util
 from beets import config
-from beets.util import pipeline, sorted_walk, ancestry
+from beets.util import pipeline, sorted_walk, ancestry, MoveOperation
 from beets.util import syspath, normpath, displayable_path
 from enum import Enum
-from beets import mediafile
+import mediafile
 
 action = Enum('action',
               ['SKIP', 'ASIS', 'TRACKS', 'APPLY', 'ALBUMS', 'RETAG'])
@@ -75,7 +73,7 @@ def _open_state():
         # unpickling, including ImportError. We use a catch-all
         # exception to avoid enumerating them all (the docs don't even have a
         # full list!).
-        log.debug(u'state file could not be read: {0}', exc)
+        log.debug('state file could not be read: {0}', exc)
         return {}
 
 
@@ -84,8 +82,8 @@ def _save_state(state):
     try:
         with open(config['statefile'].as_filename(), 'wb') as f:
             pickle.dump(state, f)
-    except IOError as exc:
-        log.error(u'state file could not be written: {0}', exc)
+    except OSError as exc:
+        log.error('state file could not be written: {0}', exc)
 
 
 # Utilities for reading and writing the beets progress file, which
@@ -174,10 +172,11 @@ def history_get():
 
 # Abstract session class.
 
-class ImportSession(object):
+class ImportSession:
     """Controls an import action. Subclasses should implement methods to
     communicate with the user or otherwise make decisions.
     """
+
     def __init__(self, lib, loghandler, paths, query):
         """Create a session. `lib` is a Library object. `loghandler` is a
         logging.Handler. Either `paths` or `query` is non-null and indicates
@@ -187,7 +186,9 @@ class ImportSession(object):
         self.logger = self._setup_logging(loghandler)
         self.paths = paths
         self.query = query
-        self._is_resuming = dict()
+        self._is_resuming = {}
+        self._merged_items = set()
+        self._merged_dirs = set()
 
         # Normalize the paths.
         if self.paths:
@@ -220,19 +221,31 @@ class ImportSession(object):
             iconfig['resume'] = False
             iconfig['incremental'] = False
 
-        # Copy, move, link, and hardlink are mutually exclusive.
+        if iconfig['reflink']:
+            iconfig['reflink'] = iconfig['reflink'] \
+                .as_choice(['auto', True, False])
+
+        # Copy, move, reflink, link, and hardlink are mutually exclusive.
         if iconfig['move']:
             iconfig['copy'] = False
             iconfig['link'] = False
             iconfig['hardlink'] = False
+            iconfig['reflink'] = False
         elif iconfig['link']:
             iconfig['copy'] = False
             iconfig['move'] = False
             iconfig['hardlink'] = False
+            iconfig['reflink'] = False
         elif iconfig['hardlink']:
             iconfig['copy'] = False
             iconfig['move'] = False
             iconfig['link'] = False
+            iconfig['reflink'] = False
+        elif iconfig['reflink']:
+            iconfig['copy'] = False
+            iconfig['move'] = False
+            iconfig['link'] = False
+            iconfig['hardlink'] = False
 
         # Only delete when copying.
         if not iconfig['copy']:
@@ -244,7 +257,7 @@ class ImportSession(object):
         """Log a message about a given album to the importer log. The status
         should reflect the reason the album couldn't be tagged.
         """
-        self.logger.info(u'{0} {1}', status, displayable_path(paths))
+        self.logger.info('{0} {1}', status, displayable_path(paths))
 
     def log_choice(self, task, duplicate=False):
         """Logs the task's current choice if it should be logged. If
@@ -255,17 +268,17 @@ class ImportSession(object):
         if duplicate:
             # Duplicate: log all three choices (skip, keep both, and trump).
             if task.should_remove_duplicates:
-                self.tag_log(u'duplicate-replace', paths)
+                self.tag_log('duplicate-replace', paths)
             elif task.choice_flag in (action.ASIS, action.APPLY):
-                self.tag_log(u'duplicate-keep', paths)
+                self.tag_log('duplicate-keep', paths)
             elif task.choice_flag is (action.SKIP):
-                self.tag_log(u'duplicate-skip', paths)
+                self.tag_log('duplicate-skip', paths)
         else:
             # Non-duplicate: log "skip" and "asis" choices.
             if task.choice_flag is action.ASIS:
-                self.tag_log(u'asis', paths)
+                self.tag_log('asis', paths)
             elif task.choice_flag is action.SKIP:
-                self.tag_log(u'skip', paths)
+                self.tag_log('skip', paths)
 
     def should_resume(self, path):
         raise NotImplementedError
@@ -282,7 +295,7 @@ class ImportSession(object):
     def run(self):
         """Run the import task.
         """
-        self.logger.info(u'import started {0}', time.asctime())
+        self.logger.info('import started {0}', time.asctime())
         self.set_config(config['import'])
 
         # Set up the pipeline.
@@ -311,6 +324,8 @@ class ImportSession(object):
                 stages += [import_asis(self)]
 
             # Plugin stages.
+            for stage_func in plugins.early_import_stages():
+                stages.append(plugin_stage(self, stage_func))
             for stage_func in plugins.import_stages():
                 stages.append(plugin_stage(self, stage_func))
 
@@ -350,6 +365,24 @@ class ImportSession(object):
             self._history_dirs = history_get()
         return self._history_dirs
 
+    def already_merged(self, paths):
+        """Returns true if all the paths being imported were part of a merge
+        during previous tasks.
+        """
+        for path in paths:
+            if path not in self._merged_items \
+               and path not in self._merged_dirs:
+                return False
+        return True
+
+    def mark_merged(self, paths):
+        """Mark paths and directories as merged for future reimport tasks.
+        """
+        self._merged_items.update(paths)
+        dirs = {os.path.dirname(path) if os.path.isfile(path) else path
+                for path in paths}
+        self._merged_dirs.update(dirs)
+
     def is_resuming(self, toppath):
         """Return `True` if user wants to resume import of this path.
 
@@ -367,7 +400,7 @@ class ImportSession(object):
             # Either accept immediately or prompt for input to decide.
             if self.want_resume is True or \
                self.should_resume(toppath):
-                log.warning(u'Resuming interrupted import of {0}',
+                log.warning('Resuming interrupted import of {0}',
                             util.displayable_path(toppath))
                 self._is_resuming[toppath] = True
             else:
@@ -377,11 +410,12 @@ class ImportSession(object):
 
 # The importer task class.
 
-class BaseImportTask(object):
+class BaseImportTask:
     """An abstract base class for importer tasks.
 
     Tasks flow through the importer pipeline. Each stage can update
     them.     """
+
     def __init__(self, toppath, paths, items):
         """Create a task. The primary fields that define a task are:
 
@@ -419,7 +453,7 @@ class ImportTask(BaseImportTask):
       from the `candidates` list.
 
     * `find_duplicates()` Returns a list of albums from `lib` with the
-       same artist and album name as the task.
+      same artist and album name as the task.
 
     * `apply_metadata()` Sets the attributes of the items from the
       task's `match` attribute.
@@ -429,17 +463,22 @@ class ImportTask(BaseImportTask):
     * `manipulate_files()` Copy, move, and write files depending on the
       session configuration.
 
+    * `set_fields()` Sets the fields given at CLI or configuration to
+      the specified values.
+
     * `finalize()` Update the import progress and cleanup the file
       system.
     """
+
     def __init__(self, toppath, paths, items):
-        super(ImportTask, self).__init__(toppath, paths, items)
+        super().__init__(toppath, paths, items)
         self.choice_flag = None
         self.cur_album = None
         self.cur_artist = None
         self.candidates = []
         self.rec = None
         self.should_remove_duplicates = False
+        self.should_merge_duplicates = False
         self.is_album = True
         self.search_ids = []  # user-supplied candidate IDs.
 
@@ -510,6 +549,10 @@ class ImportTask(BaseImportTask):
     def apply_metadata(self):
         """Copy metadata from match info to the items.
         """
+        if config['import']['from_scratch']:
+            for item in self.match.mapping:
+                item.clear()
+
         autotag.apply_metadata(self.match.info, self.match.mapping)
 
     def duplicate_items(self, lib):
@@ -520,23 +563,45 @@ class ImportTask(BaseImportTask):
 
     def remove_duplicates(self, lib):
         duplicate_items = self.duplicate_items(lib)
-        log.debug(u'removing {0} old duplicated items', len(duplicate_items))
+        log.debug('removing {0} old duplicated items', len(duplicate_items))
         for item in duplicate_items:
             item.remove()
             if lib.directory in util.ancestry(item.path):
-                log.debug(u'deleting duplicate {0}',
+                log.debug('deleting duplicate {0}',
                           util.displayable_path(item.path))
                 util.remove(item.path)
                 util.prune_dirs(os.path.dirname(item.path),
                                 lib.directory)
 
+    def set_fields(self, lib):
+        """Sets the fields given at CLI or configuration to the specified
+        values, for both the album and all its items.
+        """
+        items = self.imported_items()
+        for field, view in config['import']['set_fields'].items():
+            value = view.get()
+            log.debug('Set field {1}={2} for {0}',
+                      displayable_path(self.paths),
+                      field,
+                      value)
+            self.album[field] = value
+            for item in items:
+                item[field] = value
+        with lib.transaction():
+            for item in items:
+                item.store()
+            self.album.store()
+
     def finalize(self, session):
         """Save progress, clean up files, and emit plugin event.
         """
         # Update progress.
         if session.want_resume:
             self.save_progress()
-        if session.config['incremental']:
+        if session.config['incremental'] and not (
+            # Should we skip recording to incremental list?
+            self.skip and session.config['incremental_skip_later']
+        ):
             self.save_history()
 
         self.cleanup(copy=session.config['copy'],
@@ -609,17 +674,18 @@ class ImportTask(BaseImportTask):
             return []
 
         duplicates = []
-        task_paths = set(i.path for i in self.items if i)
+        task_paths = {i.path for i in self.items if i}
         duplicate_query = dbcore.AndQuery((
             dbcore.MatchQuery('albumartist', artist),
             dbcore.MatchQuery('album', album),
         ))
 
         for album in lib.albums(duplicate_query):
-            # Check whether the album is identical in contents, in which
-            # case it is not a duplicate (will be replaced).
-            album_paths = set(i.path for i in album.items())
-            if album_paths != task_paths:
+            # Check whether the album paths are all present in the task
+            # i.e. album is being completely re-imported by the task,
+            # in which case it is not a duplicate (will be replaced).
+            album_paths = {i.path for i in album.items()}
+            if not (album_paths <= task_paths):
                 duplicates.append(album)
         return duplicates
 
@@ -659,20 +725,28 @@ class ImportTask(BaseImportTask):
         for item in self.items:
             item.update(changes)
 
-    def manipulate_files(self, move=False, copy=False, write=False,
-                         link=False, hardlink=False, session=None):
+    def manipulate_files(self, operation=None, write=False, session=None):
+        """ Copy, move, link, hardlink or reflink (depending on `operation`) the files
+        as well as write metadata.
+
+        `operation` should be an instance of `util.MoveOperation`.
+
+        If `write` is `True` metadata is written to the files.
+        """
+
         items = self.imported_items()
         # Save the original paths of all items for deletion and pruning
         # in the next step (finalization).
         self.old_paths = [item.path for item in items]
         for item in items:
-            if move or copy or link or hardlink:
+            if operation is not None:
                 # In copy and link modes, treat re-imports specially:
                 # move in-library files. (Out-of-library files are
                 # copied/moved as usual).
                 old_path = item.path
-                if (copy or link or hardlink) and self.replaced_items[item] \
-                   and session.lib.directory in util.ancestry(old_path):
+                if (operation != MoveOperation.MOVE
+                        and self.replaced_items[item]
+                        and session.lib.directory in util.ancestry(old_path)):
                     item.move()
                     # We moved the item, so remove the
                     # now-nonexistent file from old_paths.
@@ -680,7 +754,7 @@ class ImportTask(BaseImportTask):
                 else:
                     # A normal import. Just copy files and keep track of
                     # old paths.
-                    item.move(copy, link, hardlink)
+                    item.move(operation)
 
             if write and (self.apply or self.choice_flag == action.RETAG):
                 item.try_write()
@@ -699,6 +773,8 @@ class ImportTask(BaseImportTask):
             self.record_replaced(lib)
             self.remove_replaced(lib)
             self.album = lib.add_album(self.imported_items())
+            if 'data_source' in self.imported_items()[0]:
+                self.album.data_source = self.imported_items()[0].data_source
             self.reimport_metadata(lib)
 
     def record_replaced(self, lib):
@@ -717,7 +793,7 @@ class ImportTask(BaseImportTask):
                 if (not dup_item.album_id or
                         dup_item.album_id in replaced_album_ids):
                     continue
-                replaced_album = dup_item.get_album()
+                replaced_album = dup_item._cached_album
                 if replaced_album:
                     replaced_album_ids.add(dup_item.album_id)
                     self.replaced_albums[replaced_album.path] = replaced_album
@@ -734,8 +810,8 @@ class ImportTask(BaseImportTask):
                 self.album.artpath = replaced_album.artpath
                 self.album.store()
                 log.debug(
-                    u'Reimported album: added {0}, flexible '
-                    u'attributes {1} from album {2} for {3}',
+                    'Reimported album: added {0}, flexible '
+                    'attributes {1} from album {2} for {3}',
                     self.album.added,
                     replaced_album._values_flex.keys(),
                     replaced_album.id,
@@ -748,16 +824,16 @@ class ImportTask(BaseImportTask):
                 if dup_item.added and dup_item.added != item.added:
                     item.added = dup_item.added
                     log.debug(
-                        u'Reimported item added {0} '
-                        u'from item {1} for {2}',
+                        'Reimported item added {0} '
+                        'from item {1} for {2}',
                         item.added,
                         dup_item.id,
                         displayable_path(item.path)
                     )
                 item.update(dup_item._values_flex)
                 log.debug(
-                    u'Reimported item flexible attributes {0} '
-                    u'from item {1} for {2}',
+                    'Reimported item flexible attributes {0} '
+                    'from item {1} for {2}',
                     dup_item._values_flex.keys(),
                     dup_item.id,
                     displayable_path(item.path)
@@ -770,10 +846,10 @@ class ImportTask(BaseImportTask):
         """
         for item in self.imported_items():
             for dup_item in self.replaced_items[item]:
-                log.debug(u'Replacing item {0}: {1}',
+                log.debug('Replacing item {0}: {1}',
                           dup_item.id, displayable_path(item.path))
                 dup_item.remove()
-        log.debug(u'{0} of {1} items replaced',
+        log.debug('{0} of {1} items replaced',
                   sum(bool(l) for l in self.replaced_items.values()),
                   len(self.imported_items()))
 
@@ -811,7 +887,7 @@ class SingletonImportTask(ImportTask):
     """
 
     def __init__(self, toppath, item):
-        super(SingletonImportTask, self).__init__(toppath, [item.path], [item])
+        super().__init__(toppath, [item.path], [item])
         self.item = item
         self.is_album = False
         self.paths = [item.path]
@@ -877,6 +953,19 @@ class SingletonImportTask(ImportTask):
     def reload(self):
         self.item.load()
 
+    def set_fields(self, lib):
+        """Sets the fields given at CLI or configuration to the specified
+        values, for the singleton item.
+        """
+        for field, view in config['import']['set_fields'].items():
+            value = view.get()
+            log.debug('Set field {1}={2} for {0}',
+                      displayable_path(self.paths),
+                      field,
+                      value)
+            self.item[field] = value
+        self.item.store()
+
 
 # FIXME The inheritance relationships are inverted. This is why there
 # are so many methods which pass. More responsibility should be delegated to
@@ -891,7 +980,7 @@ class SentinelImportTask(ImportTask):
     """
 
     def __init__(self, toppath, paths):
-        super(SentinelImportTask, self).__init__(toppath, paths, ())
+        super().__init__(toppath, paths, ())
         # TODO Remove the remaining attributes eventually
         self.should_remove_duplicates = False
         self.is_album = True
@@ -935,7 +1024,7 @@ class ArchiveImportTask(SentinelImportTask):
     """
 
     def __init__(self, toppath):
-        super(ArchiveImportTask, self).__init__(toppath, ())
+        super().__init__(toppath, ())
         self.extracted = False
 
     @classmethod
@@ -964,14 +1053,20 @@ class ArchiveImportTask(SentinelImportTask):
             cls._handlers = []
             from zipfile import is_zipfile, ZipFile
             cls._handlers.append((is_zipfile, ZipFile))
-            from tarfile import is_tarfile, TarFile
-            cls._handlers.append((is_tarfile, TarFile))
+            import tarfile
+            cls._handlers.append((tarfile.is_tarfile, tarfile.open))
             try:
                 from rarfile import is_rarfile, RarFile
             except ImportError:
                 pass
             else:
                 cls._handlers.append((is_rarfile, RarFile))
+            try:
+                from py7zr import is_7zfile, SevenZipFile
+            except ImportError:
+                pass
+            else:
+                cls._handlers.append((is_7zfile, SevenZipFile))
 
         return cls._handlers
 
@@ -979,7 +1074,7 @@ class ArchiveImportTask(SentinelImportTask):
         """Removes the temporary directory the archive was extracted to.
         """
         if self.extracted:
-            log.debug(u'Removing extracted directory: {0}',
+            log.debug('Removing extracted directory: {0}',
                       displayable_path(self.toppath))
             shutil.rmtree(self.toppath)
 
@@ -991,9 +1086,9 @@ class ArchiveImportTask(SentinelImportTask):
             if path_test(util.py3_path(self.toppath)):
                 break
 
+        extract_to = mkdtemp()
+        archive = handler_class(util.py3_path(self.toppath), mode='r')
         try:
-            extract_to = mkdtemp()
-            archive = handler_class(util.py3_path(self.toppath), mode='r')
             archive.extractall(extract_to)
         finally:
             archive.close()
@@ -1001,10 +1096,11 @@ class ArchiveImportTask(SentinelImportTask):
         self.toppath = extract_to
 
 
-class ImportTaskFactory(object):
+class ImportTaskFactory:
     """Generate album and singleton import tasks for all media files
     indicated by a path.
     """
+
     def __init__(self, toppath, session):
         """Create a new task factory.
 
@@ -1042,14 +1138,12 @@ class ImportTaskFactory(object):
             if self.session.config['singletons']:
                 for path in paths:
                     tasks = self._create(self.singleton(path))
-                    for task in tasks:
-                        yield task
+                    yield from tasks
                 yield self.sentinel(dirs)
 
             else:
                 tasks = self._create(self.album(paths, dirs))
-                for task in tasks:
-                    yield task
+                yield from tasks
 
         # Produce the final sentinel for this toppath to indicate that
         # it is finished. This is usually just a SentinelImportTask, but
@@ -1097,7 +1191,7 @@ class ImportTaskFactory(object):
         """Return a `SingletonImportTask` for the music file.
         """
         if self.session.already_imported(self.toppath, [path]):
-            log.debug(u'Skipping previously-imported path: {0}',
+            log.debug('Skipping previously-imported path: {0}',
                       displayable_path(path))
             self.skipped += 1
             return None
@@ -1118,10 +1212,10 @@ class ImportTaskFactory(object):
             return None
 
         if dirs is None:
-            dirs = list(set(os.path.dirname(p) for p in paths))
+            dirs = list({os.path.dirname(p) for p in paths})
 
         if self.session.already_imported(self.toppath, dirs):
-            log.debug(u'Skipping previously-imported path: {0}',
+            log.debug('Skipping previously-imported path: {0}',
                       displayable_path(dirs))
             self.skipped += 1
             return None
@@ -1151,22 +1245,22 @@ class ImportTaskFactory(object):
 
         if not (self.session.config['move'] or
                 self.session.config['copy']):
-            log.warning(u"Archive importing requires either "
-                        u"'copy' or 'move' to be enabled.")
+            log.warning("Archive importing requires either "
+                        "'copy' or 'move' to be enabled.")
             return
 
-        log.debug(u'Extracting archive: {0}',
+        log.debug('Extracting archive: {0}',
                   displayable_path(self.toppath))
         archive_task = ArchiveImportTask(self.toppath)
         try:
             archive_task.extract()
         except Exception as exc:
-            log.error(u'extraction failed: {0}', exc)
+            log.error('extraction failed: {0}', exc)
             return
 
         # Now read albums from the extracted directory.
         self.toppath = archive_task.toppath
-        log.debug(u'Archive extracted to: {0}', self.toppath)
+        log.debug('Archive extracted to: {0}', self.toppath)
         return archive_task
 
     def read_item(self, path):
@@ -1182,12 +1276,33 @@ class ImportTaskFactory(object):
                 # Silently ignore non-music files.
                 pass
             elif isinstance(exc.reason, mediafile.UnreadableFileError):
-                log.warning(u'unreadable file: {0}', displayable_path(path))
+                log.warning('unreadable file: {0}', displayable_path(path))
             else:
-                log.error(u'error reading {0}: {1}',
+                log.error('error reading {0}: {1}',
                           displayable_path(path), exc)
 
 
+# Pipeline utilities
+
+def _freshen_items(items):
+    # Clear IDs from re-tagged items so they appear "fresh" when
+    # we add them back to the library.
+    for item in items:
+        item.id = None
+        item.album_id = None
+
+
+def _extend_pipeline(tasks, *stages):
+    # Return pipeline extension for stages with list of tasks
+    if type(tasks) == list:
+        task_iter = iter(tasks)
+    else:
+        task_iter = tasks
+
+    ipl = pipeline.Pipeline([task_iter] + list(stages))
+    return pipeline.multiple(ipl.pull())
+
+
 # Full-album pipeline stages.
 
 def read_tasks(session):
@@ -1202,17 +1317,16 @@ def read_tasks(session):
 
         # Generate tasks.
         task_factory = ImportTaskFactory(toppath, session)
-        for t in task_factory.tasks():
-            yield t
+        yield from task_factory.tasks()
         skipped += task_factory.skipped
 
         if not task_factory.imported:
-            log.warning(u'No files imported from {0}',
+            log.warning('No files imported from {0}',
                         displayable_path(toppath))
 
     # Show skipped directories (due to incremental/resume).
     if skipped:
-        log.info(u'Skipped {0} paths.', skipped)
+        log.info('Skipped {0} paths.', skipped)
 
 
 def query_tasks(session):
@@ -1230,15 +1344,10 @@ def query_tasks(session):
     else:
         # Search for albums.
         for album in session.lib.albums(session.query):
-            log.debug(u'yielding album {0}: {1} - {2}',
+            log.debug('yielding album {0}: {1} - {2}',
                       album.id, album.albumartist, album.album)
             items = list(album.items())
-
-            # Clear IDs from re-tagged items so they appear "fresh" when
-            # we add them back to the library.
-            for item in items:
-                item.id = None
-                item.album_id = None
+            _freshen_items(items)
 
             task = ImportTask(None, [album.item_dir()], items)
             for task in task.handle_created(session):
@@ -1258,7 +1367,7 @@ def lookup_candidates(session, task):
         return
 
     plugins.send('import_task_start', session=session, task=task)
-    log.debug(u'Looking up: {0}', displayable_path(task.paths))
+    log.debug('Looking up: {0}', displayable_path(task.paths))
 
     # Restrict the initial lookup to IDs specified by the user via the -m
     # option. Currently all the IDs are passed onto the tasks directly.
@@ -1284,6 +1393,9 @@ def user_query(session, task):
     if task.skip:
         return task
 
+    if session.already_merged(task.paths):
+        return pipeline.BUBBLE
+
     # Ask the user for a choice.
     task.choose_match(session)
     plugins.send('import_task_choice', session=session, task=task)
@@ -1294,28 +1406,41 @@ def user_query(session, task):
         def emitter(task):
             for item in task.items:
                 task = SingletonImportTask(task.toppath, item)
-                for new_task in task.handle_created(session):
-                    yield new_task
+                yield from task.handle_created(session)
             yield SentinelImportTask(task.toppath, task.paths)
 
-        ipl = pipeline.Pipeline([
-            emitter(task),
-            lookup_candidates(session),
-            user_query(session),
-        ])
-        return pipeline.multiple(ipl.pull())
+        return _extend_pipeline(emitter(task),
+                                lookup_candidates(session),
+                                user_query(session))
 
     # As albums: group items by albums and create task for each album
     if task.choice_flag is action.ALBUMS:
-        ipl = pipeline.Pipeline([
-            iter([task]),
-            group_albums(session),
-            lookup_candidates(session),
-            user_query(session)
-        ])
-        return pipeline.multiple(ipl.pull())
+        return _extend_pipeline([task],
+                                group_albums(session),
+                                lookup_candidates(session),
+                                user_query(session))
 
     resolve_duplicates(session, task)
+
+    if task.should_merge_duplicates:
+        # Create a new task for tagging the current items
+        # and duplicates together
+        duplicate_items = task.duplicate_items(session.lib)
+
+        # Duplicates would be reimported so make them look "fresh"
+        _freshen_items(duplicate_items)
+        duplicate_paths = [item.path for item in duplicate_items]
+
+        # Record merged paths in the session so they are not reimported
+        session.mark_merged(duplicate_paths)
+
+        merged_task = ImportTask(None, task.paths + duplicate_paths,
+                                 task.items + duplicate_items)
+
+        return _extend_pipeline([merged_task],
+                                lookup_candidates(session),
+                                user_query(session))
+
     apply_choice(session, task)
     return task
 
@@ -1327,28 +1452,32 @@ def resolve_duplicates(session, task):
     if task.choice_flag in (action.ASIS, action.APPLY, action.RETAG):
         found_duplicates = task.find_duplicates(session.lib)
         if found_duplicates:
-            log.debug(u'found duplicates: {}'.format(
+            log.debug('found duplicates: {}'.format(
                 [o.id for o in found_duplicates]
             ))
 
             # Get the default action to follow from config.
             duplicate_action = config['import']['duplicate_action'].as_choice({
-                u'skip': u's',
-                u'keep': u'k',
-                u'remove': u'r',
-                u'ask': u'a',
+                'skip': 's',
+                'keep': 'k',
+                'remove': 'r',
+                'merge': 'm',
+                'ask': 'a',
             })
-            log.debug(u'default action for duplicates: {0}', duplicate_action)
+            log.debug('default action for duplicates: {0}', duplicate_action)
 
-            if duplicate_action == u's':
+            if duplicate_action == 's':
                 # Skip new.
                 task.set_choice(action.SKIP)
-            elif duplicate_action == u'k':
+            elif duplicate_action == 'k':
                 # Keep both. Do nothing; leave the choice intact.
                 pass
-            elif duplicate_action == u'r':
+            elif duplicate_action == 'r':
                 # Remove old.
                 task.should_remove_duplicates = True
+            elif duplicate_action == 'm':
+                # Merge duplicates together
+                task.should_merge_duplicates = True
             else:
                 # No default action set; ask the session.
                 session.resolve_duplicate(task, found_duplicates)
@@ -1366,7 +1495,7 @@ def import_asis(session, task):
     if task.skip:
         return
 
-    log.info(u'{}', displayable_path(task.paths))
+    log.info('{}', displayable_path(task.paths))
     task.set_choice(action.ASIS)
     apply_choice(session, task)
 
@@ -1385,6 +1514,14 @@ def apply_choice(session, task):
 
     task.add(session.lib)
 
+    # If ``set_fields`` is set, set those fields to the
+    # configured values.
+    # NOTE: This cannot be done before the ``task.add()`` call above,
+    # because then the ``ImportTask`` won't have an `album` for which
+    # it can set the fields.
+    if config['import']['set_fields']:
+        task.set_fields(session.lib)
+
 
 @pipeline.mutator_stage
 def plugin_stage(session, func, task):
@@ -1413,12 +1550,22 @@ def manipulate_files(session, task):
         if task.should_remove_duplicates:
             task.remove_duplicates(session.lib)
 
+        if session.config['move']:
+            operation = MoveOperation.MOVE
+        elif session.config['copy']:
+            operation = MoveOperation.COPY
+        elif session.config['link']:
+            operation = MoveOperation.LINK
+        elif session.config['hardlink']:
+            operation = MoveOperation.HARDLINK
+        elif session.config['reflink']:
+            operation = MoveOperation.REFLINK
+        else:
+            operation = None
+
         task.manipulate_files(
-            move=session.config['move'],
-            copy=session.config['copy'],
+            operation,
             write=session.config['write'],
-            link=session.config['link'],
-            hardlink=session.config['hardlink'],
             session=session,
         )
 
@@ -1431,11 +1578,11 @@ def log_files(session, task):
     """A coroutine (pipeline stage) to log each file to be imported.
     """
     if isinstance(task, SingletonImportTask):
-        log.info(u'Singleton: {0}', displayable_path(task.item['path']))
+        log.info('Singleton: {0}', displayable_path(task.item['path']))
     elif task.items:
-        log.info(u'Album: {0}', displayable_path(task.paths[0]))
+        log.info('Album: {0}', displayable_path(task.paths[0]))
         for item in task.items:
-            log.info(u'  {0}', displayable_path(item['path']))
+            log.info('  {0}', displayable_path(item['path']))
 
 
 def group_albums(session):
@@ -1469,6 +1616,14 @@ MULTIDISC_MARKERS = (br'dis[ck]', br'cd')
 MULTIDISC_PAT_FMT = br'^(.*%s[\W_]*)\d'
 
 
+def is_subdir_of_any_in_list(path, dirs):
+    """Returns True if path os a subdirectory of any directory in dirs
+    (a list). In other case, returns False.
+    """
+    ancestors = ancestry(path)
+    return any(d in ancestors for d in dirs)
+
+
 def albums_in_dir(path):
     """Recursively searches the given directory and returns an iterable
     of (paths, items) where paths is a list of directories and items is
@@ -1488,7 +1643,7 @@ def albums_in_dir(path):
         # and add the current directory. If so, just add the directory
         # and move on to the next directory. If not, stop collapsing.
         if collapse_paths:
-            if (not collapse_pat and collapse_paths[0] in ancestry(root)) or \
+            if (is_subdir_of_any_in_list(root, collapse_paths)) or \
                     (collapse_pat and
                      collapse_pat.match(os.path.basename(root))):
                 # Still collapsing.