From 3d3ed4ac9459357018c068a2cb7cf242c6acb10a Mon Sep 17 00:00:00 2001 From: Ade Date: Sat, 3 Aug 2013 13:36:56 +1200 Subject: [PATCH 1/4] Various - searcher, rutracker, encoder --- headphones/getXldProfile.py | 48 ++++---- headphones/music_encoder.py | 189 ++++++++++++++++++++----------- headphones/searcher.py | 53 +++++---- headphones/searcher_rutracker.py | 52 +++++---- 4 files changed, 213 insertions(+), 129 deletions(-) diff --git a/headphones/getXldProfile.py b/headphones/getXldProfile.py index e9d27015..6c744a87 100755 --- a/headphones/getXldProfile.py +++ b/headphones/getXldProfile.py @@ -9,26 +9,26 @@ def getXldProfile(xldProfile): xldProfileNotFound = xldProfile expandedPath = os.path.expanduser('~/Library/Preferences/jp.tmkk.XLD.plist') try: - preferences = plistlib.Plist.fromFile(expandedPath) + preferences = plistlib.Plist.fromFile(expandedPath) except (expat.ExpatError): - os.system("/usr/bin/plutil -convert xml1 %s" % expandedPath ) - try: - preferences = plistlib.Plist.fromFile(expandedPath) - except (ImportError): - os.system("/usr/bin/plutil -convert binary1 %s" % expandedPath ) - logger.info('The plist at "%s" has a date in it, and therefore is not useable.' % expandedPath) - return(xldProfileNotFound, None, None) + os.system("/usr/bin/plutil -convert xml1 %s" % expandedPath ) + try: + preferences = plistlib.Plist.fromFile(expandedPath) + except (ImportError): + os.system("/usr/bin/plutil -convert binary1 %s" % expandedPath ) + logger.info('The plist at "%s" has a date in it, and therefore is not useable.' % expandedPath) + return(xldProfileNotFound, None, None) except (ImportError): - logger.info('The plist at "%s" has a date in it, and therefore is not useable.' % expandedPath) + logger.info('The plist at "%s" has a date in it, and therefore is not useable.' % expandedPath) except: - logger.info('Unexpected error:', sys.exc_info()[0]) - return(xldProfileNotFound, None, None) + logger.info('Unexpected error:', sys.exc_info()[0]) + return(xldProfileNotFound, None, None) xldProfile = xldProfile.lower() profiles = preferences.get('Profiles') - + for profile in profiles: - + profilename = profile.get('XLDProfileManager_ProfileName') xldProfileForCmd = profilename profilename = profilename.lower() @@ -36,24 +36,24 @@ def getXldProfile(xldProfile): xldBitrate = None if profilename == xldProfile: - + OutputFormatName = profile.get('OutputFormatName') ShortDesc = profile.get('ShortDesc') - + # Determine format and bitrate - + if OutputFormatName == 'WAV': xldFormat = 'wav' - + elif OutputFormatName == 'AIFF': xldFormat = 'aiff' - + elif 'PCM' in OutputFormatName: xldFormat = 'pcm' - + elif OutputFormatName == 'Wave64': xldFormat = 'w64' - + elif OutputFormatName == 'MPEG-4 AAC': xldFormat = 'm4a' if 'CBR' in ShortDesc or 'ABR' in ShortDesc or 'CVBR' in ShortDesc: @@ -165,7 +165,7 @@ def getXldProfile(xldProfile): elif XLDVorbisOutput_Quality > 8 and XLDVorbisOutput_Quality <= 9: xldBitrate = 320 elif XLDVorbisOutput_Quality > 9: - xldBitrate = 500 + xldBitrate = 400 elif OutputFormatName == 'WavPack': xldFormat = 'wv' @@ -174,8 +174,8 @@ def getXldProfile(xldProfile): # Lossless if xldFormat and not xldBitrate: - xldBitrate = 500 - + xldBitrate = 400 + return(xldProfileForCmd, xldFormat, xldBitrate) - + return(xldProfileNotFound, None, None) \ No newline at end of file diff --git a/headphones/music_encoder.py b/headphones/music_encoder.py index 916c0335..3ae6216b 100644 --- a/headphones/music_encoder.py +++ b/headphones/music_encoder.py @@ -18,7 +18,7 @@ import headphones import shutil import time -from subprocess import call +import subprocess from headphones import logger from lib.beets.mediafile import MediaFile @@ -28,7 +28,6 @@ except ImportError: import lib.argparse as argparse # xld - if headphones.ENCODER == 'xld': import getXldProfile XLD = True @@ -38,12 +37,11 @@ else: def encode(albumPath): # Return if xld details not found - if XLD: global xldProfile (xldProfile, xldFormat, xldBitrate) = getXldProfile.getXldProfile(headphones.XLDPROFILE) if not xldFormat: - logger.error(u'Details for xld profile "%s" not found, will not be reencoded' % (xldProfile)) + logger.error(u'Details for xld profile %s not found, files will not be re-encoded' % (xldProfile)) return None tempDirEncode=os.path.join(albumPath,"temp") @@ -51,8 +49,6 @@ def encode(albumPath): musicFinalFiles=[] musicTempFiles=[] encoder ="" - startAlbumTime=time.time() - ifencoded=0 if not os.path.exists(tempDirEncode): os.mkdir(tempDirEncode) @@ -74,12 +70,12 @@ def encode(albumPath): if (headphones.ENCODERLOSSLESS): ext = os.path.normpath(os.path.splitext(music)[1].lstrip(".")).lower() - if not XLD and ext == 'flac' or XLD and (ext != xldFormat and (xldInfoMusic.bitrate / 1000 > 500)): + if not XLD and ext == 'flac' or XLD and (ext != xldFormat and (xldInfoMusic.bitrate / 1000 > 400)): musicFiles.append(os.path.join(r, music)) musicTemp = os.path.normpath(os.path.splitext(music)[0] + '.' + encoderFormat) musicTempFiles.append(os.path.join(tempDirEncode, musicTemp)) else: - logger.debug('Music "%s" is already encoded' % (music)) + logger.debug('%s is already encoded' % (music)) else: musicFiles.append(os.path.join(r, music)) musicTemp = os.path.normpath(os.path.splitext(music)[0] + '.' + encoderFormat) @@ -103,108 +99,170 @@ def encode(albumPath): encoder="ffmpeg" i=0 + encoder_failed = False + for music in musicFiles: infoMusic=MediaFile(music) - + encode = False + if XLD: if xldBitrate and (infoMusic.bitrate / 1000 <= xldBitrate): - logger.info('Music "%s" has bitrate <= "%skbit", will not be reencoded' % (music.decode(headphones.SYS_ENCODING, 'replace'), xldBitrate)) + logger.info('%s has bitrate <= %skb, will not be re-encoded' % (music.decode(headphones.SYS_ENCODING, 'replace'), xldBitrate)) else: - command(encoder,music,musicTempFiles[i],albumPath) - ifencoded=1 + encode = True elif headphones.ENCODER == 'lame': if not any(music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.' + x) for x in ["mp3", "wav"]): - logger.warn(u'Lame cant encode "%s" format for "%s", use ffmpeg' % (os.path.splitext(music)[1].decode(headphones.SYS_ENCODING, 'replace'),music.decode(headphones.SYS_ENCODING, 'replace'))) + logger.warn(u'Lame cannot encode %s format for %s, use ffmpeg' % (os.path.splitext(music)[1].decode(headphones.SYS_ENCODING, 'replace'),music.decode(headphones.SYS_ENCODING, 'replace'))) else: - if (music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.mp3') and (int(infoMusic.bitrate/1000)<=headphones.BITRATE)): - logger.info('Music "%s" has bitrate<="%skbit" will not be reencoded' % (music.decode(headphones.SYS_ENCODING, 'replace'),headphones.BITRATE)) + if (music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.mp3') and (int(infoMusic.bitrate / 1000) <= headphones.BITRATE)): + logger.info('%s has bitrate <= %skb, will not be re-encoded' % (music.decode(headphones.SYS_ENCODING, 'replace'),headphones.BITRATE)) else: - command(encoder,music,musicTempFiles[i],albumPath) - ifencoded=1 + encode = True else: if headphones.ENCODEROUTPUTFORMAT=='ogg': if music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.ogg'): - logger.warn('Can not reencode .ogg music "%s"' % (music.decode(headphones.SYS_ENCODING, 'replace'))) + logger.warn('Cannot re-encode .ogg %s' % (music.decode(headphones.SYS_ENCODING, 'replace'))) else: - command(encoder,music,musicTempFiles[i],albumPath) - ifencoded=1 + encode = True elif (headphones.ENCODEROUTPUTFORMAT=='mp3' or headphones.ENCODEROUTPUTFORMAT=='m4a'): - if (music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.'+headphones.ENCODEROUTPUTFORMAT) and (int(infoMusic.bitrate/1000)<=headphones.BITRATE)): - logger.info('Music "%s" has bitrate<="%skbit" will not be reencoded' % (music.decode(headphones.SYS_ENCODING, 'replace'),headphones.BITRATE)) + if (music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.'+headphones.ENCODEROUTPUTFORMAT) and (int(infoMusic.bitrate / 1000 ) <= headphones.BITRATE)): + logger.info('%s has bitrate <= %skb, will not be re-encoded' % (music.decode(headphones.SYS_ENCODING, 'replace'),headphones.BITRATE)) else: - command(encoder,music,musicTempFiles[i],albumPath) - ifencoded=1 + encode = True + # encode + if encode: + if not command(encoder,music,musicTempFiles[i],albumPath): + encoder_failed = True + break + else: + musicFiles[i] = None + musicTempFiles[i] = None + i=i+1 - + + musicFiles = filter(None, musicFiles) + musicTempFiles = filter(None, musicTempFiles) + + # check all files to be encoded now exist in temp directory + if not encoder_failed and musicTempFiles: + for dest in musicTempFiles: + if not os.path.exists(dest): + encoder_failed = True + logger.error('Encoded file %s does not exist in the destination temp directory' % (dest.decode(headphones.SYS_ENCODING, 'replace'))) + + # No errors, move from temp to parent + if not encoder_failed and musicTempFiles: + i = 0 + for dest in musicTempFiles: + if os.path.exists(dest): + source = musicFiles[i] + if headphones.DELETE_LOSSLESS_FILES: + os.remove(source) + check_dest = os.path.join(albumPath, os.path.split(dest)[1]) + if os.path.exists(check_dest): + os.remove(check_dest) + try: + shutil.move(dest, albumPath) + except Exception, e: + logger.error('Could not move %s to %s : %s' % (dest.decode(headphones.SYS_ENCODING, 'replace'), albumPath.decode(headphones.SYS_ENCODING, 'replace'), e)) + encoder_failed = True + break + i += 1 + + # remove temp directory shutil.rmtree(tempDirEncode) - time.sleep(1) + + # Return with error if any encoding errors + if encoder_failed: + logger.error('One or more files failed to encode, check debuglog and ensure you have the latest version of %s installed' % (headphones.ENCODER)) + return None + + time.sleep(1) for r,d,f in os.walk(albumPath): for music in f: if any(music.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): musicFinalFiles.append(os.path.join(r, music)) - if ifencoded==0: - logger.info('Encoding for folder "%s" is not needed' % (albumPath.decode(headphones.SYS_ENCODING, 'replace'))) + if not musicTempFiles: + logger.info('Encoding for folder %s is not required' % (albumPath.decode(headphones.SYS_ENCODING, 'replace'))) return musicFinalFiles def command(encoder,musicSource,musicDest,albumPath): - - return_code=1 - cmd='' + + cmd=[] startMusicTime=time.time() if XLD: xldDestDir = os.path.split(musicDest)[0] - cmd = '"' + encoder + '"' - cmd = cmd + ' "' + musicSource + '"' - cmd = cmd + ' --profile' - cmd = cmd + ' "' + xldProfile + '"' - cmd = cmd + ' -o' - cmd = cmd + ' "' + xldDestDir + '"' - + cmd = [encoder] + cmd.extend([musicSource]) + cmd.extend(['--profile']) + cmd.extend([xldProfile]) + cmd.extend(['-o']) + cmd.extend([xldDestDir]) + elif headphones.ENCODER == 'lame': + cmd = [encoder] + opts = [] if headphones.ADVANCEDENCODER =='': - cmd='"' + encoder + '"' + ' -h' + opts.extend(['-h']) if headphones.ENCODERVBRCBR=='cbr': - cmd=cmd+ ' --resample ' + str(headphones.SAMPLINGFREQUENCY) + ' -b ' + str(headphones.BITRATE) + opts.extend(['--resample', str(headphones.SAMPLINGFREQUENCY), '-b', str(headphones.BITRATE)]) elif headphones.ENCODERVBRCBR=='vbr': - cmd=cmd+' -V'+str(headphones.ENCODERQUALITY) - cmd=cmd+ ' ' + headphones.ADVANCEDENCODER + opts.extend(['-v', str(headphones.ENCODERQUALITY)]) else: - cmd=cmd+' '+ headphones.ADVANCEDENCODER - cmd=cmd+ ' "' + musicSource + '"' - cmd=cmd+ ' "' + musicDest +'"' - + advanced = (headphones.ADVANCEDENCODER.split()) + for tok in advanced: + opts.extend([tok.encode(headphones.SYS_ENCODING)]) + opts.extend([musicSource]) + opts.extend([musicDest]) + cmd.extend(opts) + elif headphones.ENCODER == 'ffmpeg': - cmd='"' + encoder + '"' + ' -i' - cmd=cmd+ ' "' + musicSource + '"' + cmd = [encoder, '-i', musicSource] + opts = [] if headphones.ADVANCEDENCODER =='': if headphones.ENCODEROUTPUTFORMAT=='ogg': - cmd=cmd+ ' -acodec libvorbis' + opts.extend(['-acodec libvorbis']) if headphones.ENCODEROUTPUTFORMAT=='m4a': - cmd=cmd+ ' -strict experimental' + opts.extend(['-strict experimental']) if headphones.ENCODERVBRCBR=='cbr': - cmd=cmd+ ' -ar ' + str(headphones.SAMPLINGFREQUENCY) + ' -ab ' + str(headphones.BITRATE) + 'k' + opts.extend(['-ar', str(headphones.SAMPLINGFREQUENCY), '-ab', str(headphones.BITRATE) + 'k']) elif headphones.ENCODERVBRCBR=='vbr': - cmd=cmd+' -aq ' + str(headphones.ENCODERQUALITY) - cmd=cmd+ ' -y -ac 2 -vn' + opts.extend(['-aq', str(headphones.ENCODERQUALITY)]) + opts.extend(['-y', '-ac', '2', '-vn']) else: - cmd=cmd+' '+ headphones.ADVANCEDENCODER - cmd=cmd+ ' "' + musicDest + '"' + advanced = (headphones.ADVANCEDENCODER.split()) + for tok in advanced: + opts.extend([tok.encode(headphones.SYS_ENCODING)]) + opts.extend([musicDest]) + cmd.extend(opts) - logger.debug(cmd) - try: - return_code = call(cmd, shell=True) + # Encode - if (return_code==0) and (os.path.exists(musicDest)): - if headphones.DELETE_LOSSLESS_FILES: - os.remove(musicSource) - shutil.move(musicDest,albumPath) - logger.info('Music "%s" encoded in %s' % (musicSource,getTimeEncode(startMusicTime))) + logger.info('Encoding %s...' % (musicSource.decode(headphones.SYS_ENCODING, 'replace'))) + logger.debug(subprocess.list2cmdline(cmd)) - except subprocess.CalledProcessError, e: - logger.warn('Music "%s" encoding error : %s' % (musicSource, e.output)) + p = subprocess.Popen(cmd, stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + stdout, stderr = p.communicate(headphones.ENCODER) + + # error if return code not zero + if p.returncode: + logger.error('Encoding failed for %s' % (musicSource.decode(headphones.SYS_ENCODING, 'replace'))) + out = stdout if stdout else stderr + out = out.decode(headphones.SYS_ENCODING, 'replace') + outlast2lines = '\n'.join(out.splitlines()[-2:]) + logger.error('%s error details: %s' % (headphones.ENCODER, outlast2lines)) + out = out.rstrip("\n") + logger.debug(out) + encoded = False + else: + logger.info('%s encoded in %s' % (musicSource.decode(headphones.SYS_ENCODING, 'replace'),getTimeEncode(startMusicTime))) + encoded = True + + return encoded def getTimeEncode(start): seconds =int(time.time()-start) @@ -213,3 +271,4 @@ def getTimeEncode(start): minutes = seconds / 60 seconds -= 60*minutes return "%02d:%02d:%02d" % (hours, minutes, seconds) + diff --git a/headphones/searcher.py b/headphones/searcher.py index b5993ddd..ef87243d 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -30,6 +30,7 @@ import gzip, base64 import os, re, time import string +import shutil import headphones, exceptions from headphones import logger, db, helpers, classes, sab, nzbget @@ -447,7 +448,7 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): # Add a priority if it has any of the preferred words temp_list = [] for result in resultlist: - if any(word.lower() in result[0].lower() for word in helpers.split_string(headphones.PREFERRED_WORDS)): + if headphones.PREFERRED_WORDS and any(word.lower() in result[0].lower() for word in helpers.split_string(headphones.PREFERRED_WORDS)): temp_list.append((result[0],result[1],result[2],result[3],1)) else: temp_list.append((result[0],result[1],result[2],result[3],0)) @@ -467,7 +468,7 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): if not targetsize: logger.info('No track information for %s - %s. Defaulting to highest quality' % (albums[0], albums[1])) - nzblist = sorted(resultlist, key=lambda title: (-title[4] , -title[1])) + nzblist = sorted(resultlist, key=lambda title: (title[4], title[1]), reverse=True) else: logger.info('Target size: %s' % helpers.bytes_to_mb(targetsize)) @@ -499,25 +500,27 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): logger.info(result[0] + " is too small for this album - not considering it. (Size: " + helpers.bytes_to_mb(result[1]) + ", Minsize: " + helpers.bytes_to_mb(low_size_limit) + ")") continue - delta = abs(targetsize - result[1]) + delta = abs(targetsize - int(result[1])) newlist.append((result[0], result[1], result[2], result[3], result[4], delta)) nzblist = sorted(newlist, key=lambda title: (-title[4], title[5])) if not len(nzblist) and len(flac_list) and headphones.PREFERRED_BITRATE_ALLOW_LOSSLESS: logger.info("Since there were no appropriate lossy matches (and at least one lossless match), going to use lossless instead") - nzblist = sorted(flac_list, key=lambda title: (-title[4], -title[1])) + nzblist = sorted(flac_list, key=lambda title: (title[4], title[1]), reverse=True) except Exception, e: logger.debug('Error: %s' % str(e)) logger.info('No track information for %s - %s. Defaulting to highest quality' % (albums[0], albums[1])) - nzblist = sorted(resultlist, key=lambda title: (-title[4], -title[1])) + nzblist = sorted(resultlist, key=lambda title: (title[4], title[1]), reverse=True) + else: - nzblist = sorted(resultlist, key=lambda title: (-title[4], -title[1])) + nzblist = sorted(resultlist, key=lambda title: (title[4], title[1]), reverse=True) + if new: @@ -1282,14 +1285,13 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False): # Add a priority if it has any of the preferred words temp_list = [] for result in resultlist: - if any(word.lower() in result[0].lower() for word in helpers.split_string(headphones.PREFERRED_WORDS)): + if headphones.PREFERRED_WORDS and any(word.lower() in result[0].lower() for word in helpers.split_string(headphones.PREFERRED_WORDS)): temp_list.append((result[0],result[1],result[2],result[3],1)) else: temp_list.append((result[0],result[1],result[2],result[3],0)) resultlist = temp_list - print resultlist - + if headphones.PREFERRED_QUALITY == 2 and headphones.PREFERRED_BITRATE: logger.debug('Target bitrate: %s kbps' % headphones.PREFERRED_BITRATE) @@ -1303,7 +1305,7 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False): if not targetsize: logger.info('No track information for %s - %s. Defaulting to highest quality' % (albums[0], albums[1])) - torrentlist = sorted(resultlist, key=lambda title: (-title[4] , -title[1])) + torrentlist = sorted(resultlist, key=lambda title: (title[4], title[1]), reverse=True) else: logger.info('Target size: %s' % helpers.bytes_to_mb(targetsize)) @@ -1334,27 +1336,25 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False): logger.info(result[0] + " is too small for this album - not considering it. (Size: " + helpers.bytes_to_mb(result[1]) + ", Minsize: " + helpers.bytes_to_mb(low_size_limit) + ")") continue - delta = abs(targetsize - result[1]) + delta = abs(targetsize - int(result[1])) newlist.append((result[0], result[1], result[2], result[3], result[4], delta)) - print newlist torrentlist = sorted(newlist, key=lambda title: (-title[4], title[5])) - print torrentlist if not len(torrentlist) and len(flac_list) and headphones.PREFERRED_BITRATE_ALLOW_LOSSLESS: logger.info("Since there were no appropriate lossy matches (and at least one lossless match), going to use lossless instead") - torrentlist = sorted(flac_list, key=lambda title: (-title[4], -title[1])) + torrentlist = sorted(flac_list, key=lambda title: (title[4], title[1]), reverse=True) except Exception, e: logger.debug('Error: %s' % str(e)) logger.info('No track information for %s - %s. Defaulting to highest quality' % (albums[0], albums[1])) - - torrentlist = sorted(resultlist, key=lambda title: (-title[4], -title[1])) + + torrentlist = sorted(resultlist, key=lambda title: (title[4], title[1]), reverse=True) else: - - torrentlist = sorted(resultlist, key=lambda title: (-title[4], -title[1])) + + torrentlist = sorted(resultlist, key=lambda title: (title[4], title[1]), reverse=True) if new: @@ -1420,8 +1420,23 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False): elif headphones.TORRENT_DOWNLOADER == 1: logger.info("Sending torrent to Transmission") - torrentid = transmission.addTorrent(bestqual[2]) + + # rutracker needs cookies to be set, pass the .torrent file instead of url + if bestqual[3] == 'rutracker.org': + file_or_url = rutracker.get_torrent(bestqual[2]) + else: + file_or_url = bestqual[2] + + torrentid = transmission.addTorrent(file_or_url) torrent_folder_name = transmission.getTorrentFolder(torrentid) + logger.info('Torrent folder name: %s' % torrent_folder_name) + + # remove temp .torrent file created above + if bestqual[3] == 'rutracker.org': + try: + shutil.rmtree(os.path.split(file_or_url)[0]) + except Exception, e: + logger.warning('Couldn\'t remove temp dir %s' % e) myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]]) myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", torrent_folder_name, "torrent"]) diff --git a/headphones/searcher_rutracker.py b/headphones/searcher_rutracker.py index be3ea0ed..66b4d9f9 100644 --- a/headphones/searcher_rutracker.py +++ b/headphones/searcher_rutracker.py @@ -9,9 +9,11 @@ import urllib2 import cookielib from urlparse import urlparse from bs4 import BeautifulSoup +import headphones from headphones import logger, db import lib.bencode as bencode import os +from tempfile import mkdtemp class Rutracker(): @@ -90,7 +92,7 @@ class Rutracker(): def search(self, searchurl, maxsize, minseeders, albumid, bitrate): """ - Parse the search results and return the first valid torrent + Parse the search results and return valid torrent list """ titles = [] @@ -154,10 +156,11 @@ class Rutracker(): logger.info('headphones track info not found, cannot compare to torrent') return False - # Return the first valid torrent, unless we want a preferred bitrate then we want all valid entries + # Return all valid entries, ignored, required words now checked in searcher.py - unwantedlist = ['promo', 'vinyl', '[lp]', 'songbook', 'tvrip', 'hdtv', 'dvd'] - formatlist = ['.ape', '.flac', '.ogg', '.m4a', '.aac', '.mp3', '.wav', '.aif'] + #unwantedlist = ['promo', 'vinyl', '[lp]', 'songbook', 'tvrip', 'hdtv', 'dvd'] + + formatlist = ['ape', 'flac', 'ogg', 'm4a', 'aac', 'mp3', 'wav', 'aif'] deluxelist = ['deluxe', 'edition', 'japanese', 'exclusive'] for torrent in torrentlist: @@ -167,11 +170,9 @@ class Rutracker(): seeders = torrent[2] size = torrent[3] - # Attempt to filter out unwanted - title = returntitle.lower() - if not any(unwanted in title for unwanted in unwantedlist) and int(size) <= maxsize and int(seeders) >= minseeders: + if int(size) <= maxsize and int(seeders) >= minseeders: # Check torrent info @@ -202,7 +203,7 @@ class Rutracker(): for pathfile in metainfo['files']: path = pathfile['path'] for file in path: - if any(format in file for format in formatlist): + if any(file.lower().endswith('.' + x.lower()) for x in formatlist): trackcount += 1 if '.cue' in file: cuecount += 1 @@ -255,33 +256,42 @@ class Rutracker(): if any(deluxe in title for deluxe in deluxelist): valid = True - # return 1st valid torrent if not checking by bitrate, else add to list and return at end + # Add to list if valid: rulist.append((returntitle, size, topicurl)) - if not bitrate: - return rulist - + else: + if topicurl: + logger.info(u'Torrent found with %s tracks but the selected headphones release has %s tracks, skipping for rutracker.org' % (topicurl, trackcount, hptrackcount)) + + else: + logger.info('%s is larger than the maxsize or has too little seeders for this category, skipping. (Size: %i bytes, Seeders: %i)' % (returntitle, int(size), int(seeders))) + + return rulist + def get_torrent(self, url, savelocation=None): - def get_torrent(self, url, savelocation): - torrent_id = dict([part.split('=') for part in urlparse(url)[4].split('&')])['t'] self.cookiejar.set_cookie(cookielib.Cookie(version=0, name='bb_dl', value=torrent_id, port=None, port_specified=False, domain='.rutracker.org', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False)) - downloadurl = 'http://dl.rutracker.org/forum/dl.php?t=' + torrent_id + downloadurl = 'http://dl.rutracker.org/forum/dl.php?t=' + torrent_id torrent_name = torrent_id + '.torrent' - download_path = os.path.join(savelocation, torrent_name) - + try: + prev = os.umask(headphones.UMASK) page = self.opener.open(downloadurl) torrent = page.read() + if savelocation: + download_path = os.path.join(savelocation, torrent_name) + else: + tempdir = mkdtemp(suffix='_rutracker_torrents') + download_path = os.path.join(tempdir, torrent_name) fp = open (download_path, 'wb') fp.write (torrent) fp.close () + os.umask(prev) except Exception, e: - logger.error('Error getting torrent: %s' % e) - return False - + logger.error('Error getting torrent: %s' % e) + return False + return download_path - From a32b77c589b93894d0325ab92a378f486b68e6f3 Mon Sep 17 00:00:00 2001 From: Ade Date: Sat, 3 Aug 2013 23:32:07 +1200 Subject: [PATCH 2/4] Waffles - get full title to allow filtering Currently strips the suffix from the title which has info such as [2013-CD-FLAC-Vinyl-24 96-Lossless]. Change to put this back in to allow ignore word filtering. Should also fix the parsing issue getting 'NoneType' object has no attribute 'group' --- headphones/searcher.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/headphones/searcher.py b/headphones/searcher.py index ef87243d..9a390603 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -928,22 +928,19 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False): logger.info(u'Parsing results from Waffles.fm' % searchURL) d = feedparser.parse(data) + if not len(d.entries): logger.info(u"No results found from %s for %s" % (provider, term)) pass else: for item in d.entries: - try: - title_match = re.search(r"(.+)\[(.+)\]$", item.title) - title = title_match.group(1).strip() - details = title_match.group(2).split("-") + try: + title = item.title desc_match = re.search(r"Size: (\d+)<", item.description) size = desc_match.group(1) - url = item.link - resultlist.append((title, size, url, provider)) logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) except Exception, e: From 51d7acf2c527438b164b4f999f3004502d654657 Mon Sep 17 00:00:00 2001 From: Ade Date: Sun, 4 Aug 2013 16:18:55 +1200 Subject: [PATCH 3/4] searcher sort still not quite right --- headphones/postprocessor.py | 2 +- headphones/searcher.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index c40a72b0..da1f4624 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -326,7 +326,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, # Check to see if we're preserving the torrent dir if headphones.KEEP_TORRENT_FILES and Kind=="torrent": new_folder = os.path.join(albumpath, 'headphones-modified').encode(headphones.SYS_ENCODING, 'replace') - logger.info("Copying files to 'headphones-modified' subfolder to preserve downleaded files for seeding") + logger.info("Copying files to 'headphones-modified' subfolder to preserve downloaded files for seeding") try: shutil.copytree(albumpath, new_folder) # Update the album path with the new location diff --git a/headphones/searcher.py b/headphones/searcher.py index 9a390603..ca08504f 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -468,7 +468,7 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): if not targetsize: logger.info('No track information for %s - %s. Defaulting to highest quality' % (albums[0], albums[1])) - nzblist = sorted(resultlist, key=lambda title: (title[4], title[1]), reverse=True) + nzblist = sorted(resultlist, key=lambda title: (title[4], int(title[1])), reverse=True) else: logger.info('Target size: %s' % helpers.bytes_to_mb(targetsize)) @@ -507,19 +507,19 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): if not len(nzblist) and len(flac_list) and headphones.PREFERRED_BITRATE_ALLOW_LOSSLESS: logger.info("Since there were no appropriate lossy matches (and at least one lossless match), going to use lossless instead") - nzblist = sorted(flac_list, key=lambda title: (title[4], title[1]), reverse=True) + nzblist = sorted(flac_list, key=lambda title: (title[4], int(title[1])), reverse=True) except Exception, e: logger.debug('Error: %s' % str(e)) logger.info('No track information for %s - %s. Defaulting to highest quality' % (albums[0], albums[1])) - nzblist = sorted(resultlist, key=lambda title: (title[4], title[1]), reverse=True) + nzblist = sorted(resultlist, key=lambda title: (title[4], int(title[1])), reverse=True) else: - nzblist = sorted(resultlist, key=lambda title: (title[4], title[1]), reverse=True) + nzblist = sorted(resultlist, key=lambda title: (title[4], int(title[1])), reverse=True) @@ -1302,7 +1302,7 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False): if not targetsize: logger.info('No track information for %s - %s. Defaulting to highest quality' % (albums[0], albums[1])) - torrentlist = sorted(resultlist, key=lambda title: (title[4], title[1]), reverse=True) + torrentlist = sorted(resultlist, key=lambda title: (title[4], int(title[1])), reverse=True) else: logger.info('Target size: %s' % helpers.bytes_to_mb(targetsize)) @@ -1340,18 +1340,18 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False): if not len(torrentlist) and len(flac_list) and headphones.PREFERRED_BITRATE_ALLOW_LOSSLESS: logger.info("Since there were no appropriate lossy matches (and at least one lossless match), going to use lossless instead") - torrentlist = sorted(flac_list, key=lambda title: (title[4], title[1]), reverse=True) + torrentlist = sorted(flac_list, key=lambda title: (title[4], int(title[1])), reverse=True) except Exception, e: logger.debug('Error: %s' % str(e)) logger.info('No track information for %s - %s. Defaulting to highest quality' % (albums[0], albums[1])) - torrentlist = sorted(resultlist, key=lambda title: (title[4], title[1]), reverse=True) + torrentlist = sorted(resultlist, key=lambda title: (title[4], int(title[1])), reverse=True) else: - torrentlist = sorted(resultlist, key=lambda title: (title[4], title[1]), reverse=True) + torrentlist = sorted(resultlist, key=lambda title: (title[4], int(title[1])), reverse=True) if new: From 1d7523cd48bd6f36af9ddccde7e203466fd5e1da Mon Sep 17 00:00:00 2001 From: Ade Date: Sun, 4 Aug 2013 17:07:47 +1200 Subject: [PATCH 4/4] Piratebay getting stuck if no results found --- headphones/searcher.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/headphones/searcher.py b/headphones/searcher.py index ca08504f..8e9c2869 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -1102,9 +1102,11 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False): soup = BeautifulSoup(data) table = soup.find('table') - rows = table.findAll('tr') + rows = None + if table: + rows = table.findAll('tr') - if len(rows) == '1': + if not rows or len(rows) == '1': logger.info(u"No results found from %s for %s" % (provider, term)) pass