diff --git a/headphones/__init__.py b/headphones/__init__.py index f150c2bc..606f14f8 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -657,7 +657,7 @@ def dbcheck(): conn=sqlite3.connect(DB_FILE) c=conn.cursor() - c.execute('CREATE TABLE IF NOT EXISTS artists (ArtistID TEXT UNIQUE, ArtistName TEXT, ArtistSortName TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, LatestAlbum TEXT, ReleaseDate TEXT, AlbumID TEXT, HaveTracks INTEGER, TotalTracks INTEGER)') + c.execute('CREATE TABLE IF NOT EXISTS artists (ArtistID TEXT UNIQUE, ArtistName TEXT, ArtistSortName TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, LatestAlbum TEXT, ReleaseDate TEXT, AlbumID TEXT, HaveTracks INTEGER, TotalTracks INTEGER, LastUpdated TEXT)') c.execute('CREATE TABLE IF NOT EXISTS albums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, DateAdded TEXT, AlbumID TEXT UNIQUE, Status TEXT, Type TEXT)') c.execute('CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT, Format TEXT)') c.execute('CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT)') @@ -747,7 +747,12 @@ def dbcheck(): try: c.execute('SELECT Format from tracks') except sqlite3.OperationalError: - c.execute('ALTER TABLE tracks ADD COLUMN Format TEXT DEFAULT NULL') + c.execute('ALTER TABLE tracks ADD COLUMN Format TEXT DEFAULT NULL') + + try: + c.execute('SELECT LastUpdated from artists') + except sqlite3.OperationalError: + c.execute('ALTER TABLE artists ADD COLUMN LastUpdated TEXT DEFAULT NULL') conn.commit() c.close() diff --git a/headphones/importer.py b/headphones/importer.py index c123df0a..dd6a896a 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -222,6 +222,8 @@ def addArtisttoDB(artistid, extrasonly=False): newValueDict = {"Status": "Active", "TotalTracks": totaltracks, "HaveTracks": havetracks} + + newValueDict['LastUpdated'] = helpers.now() myDB.upsert("artists", newValueDict, controlValueDict) logger.info(u"Updating complete for: " + artist['artist_name']) diff --git a/headphones/mb.py b/headphones/mb.py index 94e2c900..90ec707c 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -285,14 +285,7 @@ def getReleaseGroup(rgid): time.sleep(5) if not releaseResult: - continue - - #skip releases that do not have any release date associated with them - #if we don't do this the albums will get the release date "None" and will - #automatically be set to wanted if "Automatically Mark Upcoming Albums as Wanted" is set - #because None is not a valid date and will never be in the past so it is always upcoming - if 'date' not in releaseResult: - continue + continue if releaseGroup['type'] == 'live' and releaseResult['status'] != 'Official': logger.debug('%s is not an official live album. Skipping' % releaseResult.name) @@ -342,7 +335,7 @@ def getReleaseGroup(rgid): 'asin': unicode(releaseResult.get('asin')) if 'asin' in releaseResult else None, 'trackscount': totalTracks, 'releaseid': unicode(releaseResult.get('id')), - 'releasedate': unicode(releaseResult.get('date')), + 'releasedate': unicode(releaseResult.get('date')) if 'date' in releaseResult else None, 'format': format, 'country': country } @@ -350,6 +343,9 @@ def getReleaseGroup(rgid): releaselist.append(release_dict) #necessary to make dates that miss the month and/or day show up after full dates def getSortableReleaseDate(releaseDate): + if releaseDate == None: + return 'None';#change this value to change the sorting behaviour of none, returning 'None' will put it at the top + #which was normal behaviour for pre-ngs versions if releaseDate.count('-') == 2: return releaseDate elif releaseDate.count('-') == 1: @@ -366,7 +362,7 @@ def getReleaseGroup(rgid): a = multikeysort(releaselist, ['-hasasin', 'country', 'format', 'trackscount_delta']) release_dict = {'releaseid' :a[0]['releaseid'], - 'releasedate' : unicode(releaselist[0]['releasedate']), + 'releasedate' : releaselist[0]['releasedate'], 'trackcount' : a[0]['trackscount'], 'tracks' : a[0]['tracks'], 'asin' : a[0]['asin'], diff --git a/headphones/searcher.py b/headphones/searcher.py index 03cfe3ec..0ed33eaf 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -141,14 +141,11 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): if headphones.NZBMATRIX: provider = "nzbmatrix" if headphones.PREFERRED_QUALITY == 3 or losslessOnly: - categories = "23" - maxsize = 10000000000 + categories = "23" elif headphones.PREFERRED_QUALITY: categories = "23,22" - maxsize = 2000000000 else: categories = "22" - maxsize = 300000000 # For some reason NZBMatrix is erroring out/timing out when the term starts with a "The" right now # so we'll strip it out for the time being. This may get fixed on their end, it may not, but @@ -186,12 +183,10 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): url = item.link title = item.title size = int(item.links[1]['length']) - if size < maxsize: - resultlist.append((title, size, url, provider)) - logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) - else: - logger.info('%s is larger than the maxsize for this category, skipping. (Size: %i bytes)' % (title, size)) - + + resultlist.append((title, size, url, provider)) + logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) + except AttributeError, e: logger.info(u"No results found from NZBMatrix for %s" % term) @@ -199,13 +194,10 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): provider = "newznab" if headphones.PREFERRED_QUALITY == 3 or losslessOnly: categories = "3040" - maxsize = 10000000000 elif headphones.PREFERRED_QUALITY: categories = "3040,3010" - maxsize = 2000000000 else: - categories = "3010" - maxsize = 300000000 + categories = "3010" params = { "t": "search", "apikey": headphones.NEWZNAB_APIKEY, @@ -238,11 +230,9 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): url = item.link title = item.title size = int(item.links[1]['length']) - if size < maxsize: - resultlist.append((title, size, url, provider)) - logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) - else: - logger.info('%s is larger than the maxsize for this category, skipping. (Size: %i bytes)' % (title, size)) + + resultlist.append((title, size, url, provider)) + logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) except Exception, e: logger.error(u"An unknown error occured trying to parse the feed: %s" % e) @@ -251,13 +241,10 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): provider = "nzbsorg" if headphones.PREFERRED_QUALITY == 3 or losslessOnly: categories = "3040" - maxsize = 10000000000 elif headphones.PREFERRED_QUALITY: categories = "3040,3010" - maxsize = 2000000000 else: - categories = "3010" - maxsize = 300000000 + categories = "3010" params = { "t": "search", "apikey": headphones.NZBSORG_HASH, @@ -290,12 +277,10 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): url = item.link title = item.title size = int(item.links[1]['length']) - if size < maxsize: - resultlist.append((title, size, url, provider)) - logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) - else: - logger.info('%s is larger than the maxsize for this category, skipping. (Size: %i bytes)' % (title, size)) - + + resultlist.append((title, size, url, provider)) + logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) + except Exception, e: logger.error(u"An unknown error occured trying to parse the feed: %s" % e) @@ -305,15 +290,12 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): if headphones.PREFERRED_QUALITY == 3 or losslessOnly: categories = "7" #music format = "2" #flac - maxsize = 10000000000 elif headphones.PREFERRED_QUALITY: categories = "7" #music format = "10" #mp3+flac - maxsize = 2000000000 else: categories = "7" #music - format = "8" #mp3 - maxsize = 300000000 + format = "8" #mp3 params = { "fpn": "p", @@ -370,11 +352,11 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): logger.info("Didn't find a valid Newzbin reportid in linknode") else: url = id_match.group(1) #we have to make a post request later, need the id - if size < maxsize and url: + if url: resultlist.append((title, size, url, provider)) logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) else: - logger.info('%s is larger than the maxsize for this category, skipping. (Size: %i bytes)' % (title, size)) + logger.info('No url link found in nzb. Skipping.') else: logger.info('No results found from NEWZBIN for %s' % term) diff --git a/headphones/updater.py b/headphones/updater.py index 4ca14e3b..1ea2974b 100644 --- a/headphones/updater.py +++ b/headphones/updater.py @@ -6,7 +6,7 @@ def dbUpdate(): myDB = db.DBConnection() - activeartists = myDB.select('SELECT ArtistID, ArtistName from artists WHERE Status="Active" or Status="Loading" order by ArtistSortName collate nocase') + activeartists = myDB.select('SELECT ArtistID, ArtistName from artists WHERE Status="Active" or Status="Loading" order by LastUpdated ASC') logger.info('Starting update for %i active artists' % len(activeartists)) diff --git a/headphones/versioncheck.py b/headphones/versioncheck.py index c9a5d6e0..88c9a81f 100644 --- a/headphones/versioncheck.py +++ b/headphones/versioncheck.py @@ -1,13 +1,12 @@ -import platform, subprocess, re, os - -import urllib2 -import tarfile +import platform, subprocess, re, os, urllib2, tarfile import headphones from headphones import logger, version -from lib.pygithub import github +import lib.simplejson as simplejson +user = "rembo10" +branch = "master" def runGit(args): @@ -92,33 +91,44 @@ def getVersion(): def checkGithub(): - commits_behind = 0 - cur_commit = headphones.CURRENT_VERSION - latest_commit = None - - gh = github.GitHub() + # Get the latest commit available from github + url = 'https://api.github.com/repos/%s/headphones/commits/%s' % (user, branch) + logger.info ('Retrieving latest version information from github') + try: + result = urllib2.urlopen(url).read() + git = simplejson.JSONDecoder().decode(result) + headphones.LATEST_VERSION = git['sha'] + except: + logger.warn('Could not get the latest commit from github') + headphones.COMMITS_BEHIND = 0 + return headphones.CURRENT_VERSION - for curCommit in gh.commits.forBranch('rembo10', 'headphones', version.HEADPHONES_VERSION): - if not latest_commit: - latest_commit = curCommit.id - if not cur_commit: - break + # See how many commits behind we are + if headphones.CURRENT_VERSION: + logger.info('Comparing currently installed version with latest github version') + url = 'https://api.github.com/repos/%s/headphones/compare/%s...%s' % (user, headphones.CURRENT_VERSION, headphones.LATEST_VERSION) - if curCommit.id == cur_commit: - break + try: + result = urllib2.urlopen(url).read() + git = simplejson.JSONDecoder().decode(result) + headphones.COMMITS_BEHIND = git['total_commits'] + except: + logger.warn('Could not get commits behind from github') + headphones.COMMITS_BEHIND = 0 + return headphones.CURRENT_VERSION - commits_behind += 1 - - headphones.LATEST_VERSION = latest_commit - headphones.COMMITS_BEHIND = commits_behind - - if headphones.LATEST_VERSION == headphones.CURRENT_VERSION: - logger.info('Headphones is already up-to-date.') - - return latest_commit - - + if headphones.COMMITS_BEHIND >= 1: + logger.info('New version is available. You are %s commits behind' % headphones.COMMITS_BEHIND) + elif headphones.COMMITS_BEHIND == 0: + logger.info('Headphones is up to date') + elif headphones.COMMITS_BEHIND == -1: + logger.info('You are running an unknown version of Headphones. Run the updater to identify your version') + + else: + logger.info('You are running an unknown version of Headphones. Run the updater to identify your version') + return headphones.LATEST_VERSION + def update(): @@ -146,7 +156,7 @@ def update(): else: - tar_download_url = 'http://github.com/rembo10/headphones/tarball/'+version.HEADPHONES_VERSION + tar_download_url = 'https://github.com/%s/headphones/tarball/%s' % (user, branch) update_dir = os.path.join(headphones.PROG_DIR, 'update') version_path = os.path.join(headphones.PROG_DIR, 'version.txt') @@ -200,5 +210,5 @@ def update(): ver_file.write(headphones.LATEST_VERSION) ver_file.close() except IOError, e: - logger.error(u"Unable to write for curCommit in gh.commits.forBranch file, update not complete: "+ex(e)) - return \ No newline at end of file + logger.error(u"Unable to write current version to version.txt, update not complete: "+ex(e)) + return diff --git a/lib/pygithub/__init__.py b/lib/pygithub/__init__.py deleted file mode 100644 index ad56e087..00000000 --- a/lib/pygithub/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2005-2008 Dustin Sallings -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# -""" -github module. -""" -__all__ = ['github','ghsearch','githubsync'] diff --git a/lib/pygithub/ghsearch.py b/lib/pygithub/ghsearch.py deleted file mode 100644 index 586e502b..00000000 --- a/lib/pygithub/ghsearch.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2005-2008 Dustin Sallings -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# -""" -Search script. -""" - -import sys - -import github - -def usage(): - """display the usage and exit""" - print "Usage: %s keyword [keyword...]" % (sys.argv[0]) - sys.exit(1) - -def mk_url(repo): - return "http://github.com/%s/%s" % (repo.username, repo.name) - -if __name__ == '__main__': - g = github.GitHub() - if len(sys.argv) < 2: - usage() - res = g.repos.search(' '.join(sys.argv[1:])) - - for repo in res: - try: - print "Found %s at %s" % (repo.name, mk_url(repo)) - except AttributeError: - print "Bug: Couldn't format %s" % repo.__dict__ diff --git a/lib/pygithub/github.py b/lib/pygithub/github.py deleted file mode 100644 index bd9f4077..00000000 --- a/lib/pygithub/github.py +++ /dev/null @@ -1,520 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2005-2008 Dustin Sallings -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# -""" -Interface to github's API (v2). - -Basic usage: - -g = GitHub() - -for r in g.user.search('dustin'): - print r.name - -See the GitHub docs or README.markdown for more usage. - -Copyright (c) 2007 Dustin Sallings -""" - -# GAE friendly URL detection (theoretically) -try: - import urllib2 - default_fetcher = urllib2.urlopen -except LoadError: - pass - -import urllib -import xml -import xml.dom.minidom - -def _string_parser(x): - """Extract the data from the first child of the input.""" - return x.firstChild.data - -_types = { - 'string': _string_parser, - 'integer': lambda x: int(_string_parser(x)), - 'float': lambda x: float(_string_parser(x)), - 'datetime': _string_parser, - 'boolean': lambda x: _string_parser(x) == 'true' -} - -def _parse(el): - """Generic response parser.""" - - type = 'string' - if el.attributes and 'type' in el.attributes.keys(): - type = el.attributes['type'].value - elif el.localName in _types: - type = el.localName - elif len(el.childNodes) > 1: - # This is a container, find the child type - type = None - ch = el.firstChild - while ch and not type: - if ch.localName == 'type': - type = ch.firstChild.data - ch = ch.nextSibling - - if not type: - raise Exception("Can't parse %s, known: %s" - % (el.toxml(), repr(_types.keys()))) - - return _types[type](el) - -def parses(t): - """Parser for a specific type in the github response.""" - def f(orig): - orig.parses = t - return orig - return f - -def with_temporary_mappings(m): - """Allow temporary localized altering of type mappings.""" - def f(orig): - def every(self, *args): - global _types - o = _types.copy() - for k,v in m.items(): - if v: - _types[k] = v - else: - del _types[k] - try: - return orig(self, *args) - finally: - _types = o - return every - return f - -@parses('array') -def _parseArray(el): - rv = [] - ch = el.firstChild - while ch: - if ch.nodeType != xml.dom.Node.TEXT_NODE and ch.firstChild: - rv.append(_parse(ch)) - ch=ch.nextSibling - return rv - -class BaseResponse(object): - """Base class for XML Response Handling.""" - - def __init__(self, el): - ch = el.firstChild - while ch: - if ch.nodeType != xml.dom.Node.TEXT_NODE and ch.firstChild: - ln = ch.localName.replace('-', '_') - self.__dict__[ln] = _parse(ch) - ch=ch.nextSibling - - def __repr__(self): - return "<<%s>>" % str(self.__class__) - -class User(BaseResponse): - """A github user.""" - - parses = 'user' - - def __repr__(self): - return "<>" % self.name - -class Plan(BaseResponse): - """A github plan.""" - - parses = 'plan' - - def __repr__(self): - return "<>" % self.name - -class Repository(BaseResponse): - """A repository.""" - - parses = 'repository' - - @property - def owner_name(self): - if hasattr(self, 'owner'): - return self.owner - else: - return self.username - - def __repr__(self): - return "<>" % (self.owner_name, self.name) - -class PublicKey(BaseResponse): - """A public key.""" - - parses = 'public-key' - title = 'untitled' - - def __repr__(self): - return "<>" % self.title - -class Commit(BaseResponse): - """A commit.""" - - parses = 'commit' - - def __repr__(self): - return "<>" % self.id - -class Parent(Commit): - """A commit parent.""" - - parses = 'parent' - -class Author(User): - """A commit author.""" - - parses = 'author' - -class Committer(User): - """A commit committer.""" - - parses = 'committer' - -class Issue(BaseResponse): - """An issue within the issue tracker.""" - - parses = 'issue' - - def __repr__(self): - return "<>" % self.number - -class Label(BaseResponse): - """A Label within the issue tracker.""" - parses = 'label' - - def __repr__(self): - return "<