Merge remote-tracking branch 'cohena/develop' into develop

headphones/searcher.py

@@ -41,6 +41,10 @@ import lib.bencode as bencode
 import headphones.searcher_rutracker as rutrackersearch
 rutracker = rutrackersearch.Rutracker()
 
+# Persistent What.cd API object
+gazelle = None
+
+
 class NewzbinDownloader(urllib.FancyURLopener):
 
     def __init__(self):
@@ -735,6 +739,7 @@ def preprocess(resultlist):
 
 
 def searchTorrent(albumid=None, new=False, losslessOnly=False):
+    global gazelle  # persistent what.cd api object to reduce number of login attempts
 
     myDB = db.DBConnection()
 
@@ -1024,13 +1029,16 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False):
                 search_formats = [gazelleformat.MP3]
                 maxsize = 300000000
 
-            try:
-                gazelle = gazelleapi.GazelleAPI(headphones.WHATCD_USERNAME, headphones.WHATCD_PASSWORD)
-            except Exception, e:
-                gazelle = None
-                logger.warn(u"What.cd credentials incorrect or site is down. Error: %s %s" % (e.__class__.__name__, str(e)))
+            if not gazelle or not gazelle.logged_in():
+                try:
+                    logger.info(u"Attempting to log in to What.cd...")
+                    gazelle = gazelleapi.GazelleAPI(headphones.WHATCD_USERNAME, headphones.WHATCD_PASSWORD)
+                    gazelle._login()
+                except Exception, e:
+                    gazelle = None
+                    logger.error(u"What.cd credentials incorrect or site is down. Error: %s %s" % (e.__class__.__name__, str(e)))
 
-            if gazelle:
+            if gazelle and gazelle.logged_in():
                 logger.info(u"Searching %s..." % provider)
                 all_torrents = []
                 for search_format in search_formats:

lib/pygazelle/api.py

@@ -5,18 +5,21 @@
 #
 # Loosely based on the API implementation from 'whatbetter', by Zachary Denton
 # See https://github.com/zacharydenton/whatbetter
+from HTMLParser import HTMLParser
 
+import sys
 import json
 import time
 import lib.requests as requests
 
-from user import User
-from artist import Artist
-from tag import Tag
-from request import Request
-from torrent_group import TorrentGroup
-from torrent import Torrent
-from category import Category
+from .user import User
+from .artist import Artist
+from .tag import Tag
+from .request import Request
+from .torrent_group import TorrentGroup
+from .torrent import Torrent
+from .category import Category
+from .inbox import Mailbox
 
 class LoginException(Exception):
     pass
@@ -40,7 +43,8 @@ class GazelleAPI(object):
 
 
     def __init__(self, username=None, password=None):
-        self.session = requests.session(headers=self.default_headers)
+        self.session = requests.session()
+        self.session.headers = self.default_headers
         self.username = username
         self.password = password
         self.authkey = None
@@ -55,58 +59,102 @@ class GazelleAPI(object):
         self.cached_requests = {}
         self.cached_categories = {}
         self.site = "https://what.cd/"
-        self.rate_limit = 2.0 # seconds between requests
-        self._login()
+        self.past_request_timestamps = []
+
+    def wait_for_rate_limit(self):
+        # maximum is 5 requests within 10 secs
+        time_frame = 10
+        max_reqs = 5
+
+        slice_point = 0
+
+        while len(self.past_request_timestamps) >= max_reqs:
+            for i, timestamp in enumerate(self.past_request_timestamps):
+                if timestamp < time.time() - time_frame:
+                    slice_point = i + 1
+                else:
+                    break
+
+            if slice_point:
+                self.past_request_timestamps = self.past_request_timestamps[slice_point:]
+            else:
+                time.sleep(0.1)
+
+    def logged_in(self):
+        return self.logged_in_user is not None and self.logged_in_user.id == self.userid
 
     def _login(self):
         """
         Private method.
         Logs in user and gets authkey from server.
         """
+
+        if self.logged_in():
+            return
+
+        self.wait_for_rate_limit()
+
         loginpage = 'https://what.cd/login.php'
         data = {'username': self.username,
                 'password': self.password}
         r = self.session.post(loginpage, data=data)
         if r.status_code != 200:
-            raise LoginException("Login error, http code %s" % r.status_code)
-        accountinfo = self.request('index')
+            raise LoginException("Login returned status code %s" % r.status_code)
+
+        try:
+            accountinfo = self.request('index', autologin=False)
+        except RequestException as e:
+            raise LoginException("Login probably incorrect")
+        if not accountinfo or 'id' not in accountinfo:
+            raise LoginException("Login probably incorrect")
         self.userid = accountinfo['id']
         self.authkey = accountinfo['authkey']
         self.passkey = accountinfo['passkey']
         self.logged_in_user = User(self.userid, self)
         self.logged_in_user.set_index_data(accountinfo)
+        self.past_request_timestamps.append(time.time())
 
-    def request(self, action, **kwargs):
+    def request(self, action, autologin=True, **kwargs):
         """
         Makes an AJAX request at a given action.
         Pass an action and relevant arguments for that action.
         """
-
-        ajaxpage = 'ajax.php'
-        content = self.unparsed_request(ajaxpage, action, **kwargs)
-        try:
-            parsed = json.loads(content)
-            if parsed['status'] != 'success':
+        def make_request(action, **kwargs):
+            ajaxpage = 'ajax.php'
+            content = self.unparsed_request(ajaxpage, action, **kwargs)
+            try:
+                if not isinstance(content, text_type):
+                    content = content.decode('utf-8')
+                parsed = json.loads(content)
+                if parsed['status'] != 'success':
+                    raise RequestException
+                return parsed['response']
+            except ValueError:
                 raise RequestException
-            return parsed['response']
-        except ValueError:
-            raise RequestException
 
-    def unparsed_request(self, page, action, **kwargs):
+        try:
+            return make_request(action, **kwargs)
+        except Exception as e:
+            if autologin and not self.logged_in():
+                self._login()
+                return make_request(action, **kwargs)
+            else:
+                raise e
+
+    def unparsed_request(self, sitepage, action, **kwargs):
         """
         Makes a generic HTTP request at a given page with a given action.
         Also pass relevant arguments for that action.
         """
-        while time.time() - self.last_request < self.rate_limit:
-            time.sleep(0.1)
+        self.wait_for_rate_limit()
 
-        url = "%s/%s" % (self.site, page)
+        url = "%s%s" % (self.site, sitepage)
         params = {'action': action}
         if self.authkey:
             params['auth'] = self.authkey
         params.update(kwargs)
         r = self.session.get(url, params=params, allow_redirects=False)
-        self.last_request = time.time()
+        self.past_request_timestamps.append(time.time())
         return r.content
 
     def get_user(self, id):
@@ -141,18 +189,37 @@ class GazelleAPI(object):
 
         return found_users
 
-    def get_artist(self, id, name=None):
+    def get_inbox(self, page='1', sort='unread'):
+        """
+        Returns the inbox Mailbox for the logged in user
+        """
+        return Mailbox(self, 'inbox', page, sort)
+        
+    def get_sentbox(self, page='1', sort='unread'):
+        """
+        Returns the sentbox Mailbox for the logged in user
+        """
+        return Mailbox(self, 'sentbox', page, sort)
+
+    def get_artist(self, id=None, name=None):
         """
         Returns an Artist for the passed ID, associated with this API object. You'll need to call Artist.update_data()
         if the artist hasn't already been cached. This is done on demand to reduce unnecessary API calls.
         """
-        id = int(id)
-        if id in self.cached_artists.keys():
-            artist = self.cached_artists[id]
+        if id:
+            id = int(id)
+            if id in self.cached_artists.keys():
+                artist = self.cached_artists[id]
+            else:
+                artist = Artist(id, self)
+            if name:
+                artist.name = HTMLParser().unescape(name)
+        elif name:
+            artist = Artist(-1, self)
+            artist.name = HTMLParser().unescape(name)
         else:
-            artist = Artist(id, self)
-        if name:
-            artist.name = name
+            raise Exception("You must specify either an ID or a Name to get an artist.")
+
         return artist
 
     def get_tag(self, name):
@@ -189,7 +256,7 @@ class GazelleAPI(object):
 
     def get_torrent(self, id):
         """
-        Returns a TorrentGroup for the passed ID, associated with this API object.
+        Returns a Torrent for the passed ID, associated with this API object.
         """
         id = int(id)
         if id in self.cached_torrents.keys():
@@ -197,6 +264,24 @@ class GazelleAPI(object):
         else:
             return Torrent(id, self)
 
+    def get_torrent_from_info_hash(self, info_hash):
+        """
+        Returns a Torrent for the passed info hash (if one exists), associated with this API object.
+        """
+        try:
+            response = self.request(action='torrent', hash=info_hash.upper())
+        except RequestException:
+            return None
+
+        id = int(response['torrent']['id'])
+        if id in self.cached_torrents.keys():
+            torrent = self.cached_torrents[id]
+        else:
+            torrent = Torrent(id, self)
+
+        torrent.set_torrent_complete_data(response)
+        return torrent
+
     def get_category(self, id, name=None):
         """
         Returns a Category for the passed ID, associated with this API object.
@@ -210,6 +295,45 @@ class GazelleAPI(object):
             cat.name = name
         return cat
 
+    def get_top_10(self, type="torrents", limit=25):
+        """
+        Lists the top <limit> items of <type>. Type can be "torrents", "tags", or "users". Limit MUST be
+        10, 25, or 100...it can't just be an arbitrary number (unfortunately). Results are organized into a list of hashes.
+        Each hash contains the results for a specific time frame, like 'day', or 'week'. In the hash, the 'results' key
+        contains a list of objects appropriate to the passed <type>.
+        """
+
+        response = self.request(action='top10', type=type, limit=limit)
+        top_items = []
+        if not response:
+            raise RequestException
+        for category in response:
+            results = []
+            if type == "torrents":
+                for item in category['results']:
+                    torrent = self.get_torrent(item['torrentId'])
+                    torrent.set_torrent_top_10_data(item)
+                    results.append(torrent)
+            elif type == "tags":
+                for item in category['results']:
+                    tag = self.get_tag(item['name'])
+                    results.append(tag)
+            elif type == "users":
+                for item in category['results']:
+                    user = self.get_user(item['id'])
+                    results.append(user)
+            else:
+                raise Exception("%s is an invalid type argument for GazelleAPI.get_top_ten()" % type)
+
+            top_items.append({
+                "caption": category['caption'],
+                "tag": category['tag'],
+                "limit": category['limit'],
+                "results": results
+            })
+
+        return top_items
+
     def search_torrents(self, **kwargs):
         """
         Searches based on the args you pass and returns torrent groups filled with torrents.
@@ -281,3 +405,8 @@ class GazelleAPI(object):
             id=id, authkey=self.logged_in_user.authkey, torrent_pass=self.logged_in_user.passkey)
         with open(dest, 'w+') as dest_file:
             dest_file.write(file_data)
+
+if sys.version_info[0] == 3:
+    text_type = str
+else:
+    text_type = unicode

lib/pygazelle/artist.py

@@ -1,4 +1,4 @@
-
+from HTMLParser import HTMLParser
 
 class InvalidArtistException(Exception):
     pass
@@ -26,15 +26,28 @@ class Artist(object):
         self.parent_api.cached_artists[self.id] = self # add self to cache of known Artist objects
 
     def update_data(self):
-        response = self.parent_api.request(action='artist', id=self.id)
+        if self.id > 0:
+            response = self.parent_api.request(action='artist', id=self.id)
+        elif self.name:
+            self.name = HTMLParser().unescape(self.name)
+            try:
+                response = self.parent_api.request(action='artist', artistname=self.name)
+            except Exception:
+                self.name = self.name.split(" & ")[0]
+                response = self.parent_api.request(action='artist', artistname=self.name)
+        else:
+            raise InvalidArtistException("Neither ID or Artist Name is valid, can't update data.")
         self.set_data(response)
 
     def set_data(self, artist_json_response):
-        if self.id != artist_json_response['id']:
+        if self.id > 0 and self.id != artist_json_response['id']:
             raise InvalidArtistException("Tried to update an artists's information from an 'artist' API call with a different id." +
                                " Should be %s, got %s" % (self.id, artist_json_response['id']) )
+        elif self.name:
+            self.id = artist_json_response['id']
+            self.parent_api.cached_artists[self.id] = self
 
-        self.name = artist_json_response['name']
+        self.name = HTMLParser().unescape(artist_json_response['name'])
         self.notifications_enabled = artist_json_response['notificationsEnabled']
         self.has_bookmarked = artist_json_response['hasBookmarked']
         self.image = artist_json_response['image']

lib/pygazelle/inbox.py

@@ -0,0 +1,107 @@
+class MailboxMessage(object):
+    def __init__(self, api, message):
+        self.id = message['convId']
+        self.conv = Conversation(api, self.id)
+        self.subject = message['subject']
+        self.unread = message['unread']
+        self.sticky = message['sticky']
+        self.fwd_id = message['forwardedId']
+        self.fwd_name = message['forwardedName']
+        self.sender_id = message['senderId']
+        self.username = message['username']
+        self.donor = message['donor']
+        self.warned = message['warned']
+        self.enabled = message['enabled']
+        self.date = message['date']
+
+    def __repr__(self):
+        return "MailboxMessage ID %s - %s %s %s" % (self.id, self.subject, self.sender_id, self.username)
+
+
+class ConversationMessage(object):
+    def __init__(self, msg_resp):
+        self.id = msg_resp['messageId']
+        self.sender_id = msg_resp['senderId']
+        self.sender_name = msg_resp['senderName']
+        self.sent_date = msg_resp['sentDate']
+        self.bb_body = msg_resp['bbBody']
+        self.body = msg_resp['body']
+
+    def __repr__(self):
+        return "ConversationMessage ID %s - %s %s" % (self.id, self.sender_name, self.sent_date)
+
+
+class Conversation(object):
+    def __init__(self, api, conv_id):
+        self.id = conv_id
+        self.parent_api = api
+        self.subject = None
+        self.sticky = None
+        self.messages = []
+
+    def __repr__(self):
+        return "Conversation ID %s - %s" % (self.id, self.subject)
+
+    def set_conv_data(self, conv_resp):
+        assert self.id == conv_resp['convId']
+        self.subject = conv_resp['subject']
+        self.sticky = conv_resp['sticky']
+        self.messages = [ConversationMessage(m) for m in conv_resp['messages']]
+
+    def update_conv_data(self):
+        response = self.parent_api.request(action='inbox',
+                                           type='viewconv', id=self.id)
+        self.set_conv_data(response)
+
+
+class Mailbox(object):
+    """
+    This class represents the logged in user's inbox/sentbox
+    """
+    def __init__(self, parent_api, boxtype='inbox', page='1', sort='unread'): 
+        self.parent_api = parent_api
+        self.boxtype = boxtype 
+        self.current_page = page
+        self.total_pages = None
+        self.sort = sort
+        self.messages = None
+
+    def set_mbox_data(self, mbox_resp):
+        """
+        Takes parsed JSON response from 'inbox' action on api
+        and updates the available subset of mailbox information.
+        """
+        self.current_page = mbox_resp['currentPage']
+        self.total_pages = mbox_resp['pages']
+        self.messages = \
+          [MailboxMessage(self.parent_api, m) for m in mbox_resp['messages']]
+
+    def update_mbox_data(self):
+        response = self.parent_api.request(action='inbox',
+                     type=self.boxtype, page=self.current_page, sort=self.sort)
+        self.set_mbox_data(response)
+
+    def next_page(self):
+        if not self.total_pages:
+            raise ValueError("call update_mbox_data() first")
+        total_pages = int(self.total_pages)
+        cur_page = int(self.current_page)
+        if cur_page < total_pages:
+            return Mailbox(self.parent_api, self.boxtype,
+                           str(cur_page + 1), self.sort)
+        raise ValueError("Already at page %d/%d" % (cur_page, total_pages))
+
+    def prev_page(self):
+        if not self.total_pages:
+            raise ValueError("call update_mbox_data() first")
+        total_pages = int(self.total_pages)
+        cur_page = int(self.current_page)
+        if cur_page > 1:
+            return Mailbox(self.parent_api, self.boxtype,
+                           str(cur_page - 1), self.sort)
+        raise ValueError("Already at page %d/%d" % (cur_page, total_pages))
+
+    def __repr__(self):
+        return "Mailbox: %s %s Page %s/%s" \
+                 % (self.boxtype, self.sort,
+                    self.current_page, self.total_pages)

lib/pygazelle/torrent.py

@@ -1,3 +1,4 @@
+from HTMLParser import HTMLParser
 import re
 
 class InvalidTorrentException(Exception):
@@ -35,6 +36,40 @@ class Torrent(object):
 
         self.parent_api.cached_torrents[self.id] = self
 
+    def set_torrent_complete_data(self, torrent_json_response):
+        if self.id != torrent_json_response['torrent']['id']:
+            raise InvalidTorrentException("Tried to update a Torrent's information from an 'artist' API call with a different id." +
+                                       " Should be %s, got %s" % (self.id, torrent_json_response['id']) )
+
+        self.group = self.parent_api.get_torrent_group(torrent_json_response['group']['id'])
+        had_complete_list = self.group.has_complete_torrent_list
+        self.group.set_group_data(torrent_json_response)
+        self.group.has_complete_torrent_list = had_complete_list
+
+        self.media = torrent_json_response['torrent']['media']
+        self.format = torrent_json_response['torrent']['format']
+        self.encoding = torrent_json_response['torrent']['encoding']
+        self.remaster_year = torrent_json_response['torrent']['remasterYear']
+        self.remastered = torrent_json_response['torrent']['remastered']
+        self.remaster_title = torrent_json_response['torrent']['remasterTitle']
+        self.remaster_record_label = torrent_json_response['torrent']['remasterRecordLabel']
+        self.scene = torrent_json_response['torrent']['scene']
+        self.has_log = torrent_json_response['torrent']['hasLog']
+        self.has_cue = torrent_json_response['torrent']['hasCue']
+        self.log_score = torrent_json_response['torrent']['logScore']
+        self.file_count = torrent_json_response['torrent']['fileCount']
+        self.free_torrent = torrent_json_response['torrent']['freeTorrent']
+        self.size = torrent_json_response['torrent']['size']
+        self.leechers = torrent_json_response['torrent']['leechers']
+        self.seeders = torrent_json_response['torrent']['seeders']
+        self.snatched = torrent_json_response['torrent']['snatched']
+        self.time = torrent_json_response['torrent']['time']
+        self.description = torrent_json_response['torrent']['description']
+        self.file_list = [ re.match("(.+){{{(\d+)}}}", item).groups()
+                           for item in torrent_json_response['torrent']['fileList'].split("|||") ] # tuple ( filename, filesize )
+        self.file_path = torrent_json_response['torrent']['filePath']
+        self.user = self.parent_api.get_user(torrent_json_response['torrent']['userId'])
+
     def set_torrent_artist_data(self, artist_torrent_json_response):
         if self.id != artist_torrent_json_response['id']:
             raise InvalidTorrentException("Tried to update a Torrent's information from an 'artist' API call with a different id." +
@@ -118,6 +153,26 @@ class Torrent(object):
         self.free_torrent = search_torrent_json_response['isFreeleech'] or search_torrent_json_response['isPersonalFreeleech']
         self.time = search_torrent_json_response['time']
 
+    def set_torrent_top_10_data(self, top_10_json_response):
+        if self.id != top_10_json_response['torrentId']:
+            raise InvalidTorrentException("Tried to update a Torrent's information from a 'browse'/search API call with a different id." +
+                                          " Should be %s, got %s" % (self.id, top_10_json_response['torrentId']) )
+
+        # TODO: Add conditionals to handle torrents that aren't music
+        self.group = self.parent_api.get_torrent_group(top_10_json_response['groupId'])
+        self.group.name = top_10_json_response['groupName']
+        if not self.group.music_info and top_10_json_response['artist']:
+            self.group.music_info = {'artists': [self.parent_api.get_artist(name=HTMLParser().unescape(top_10_json_response['artist']))]}
+        self.remaster_title = top_10_json_response['remasterTitle']
+        self.media = top_10_json_response['media']
+        self.format = top_10_json_response['format']
+        self.encoding = top_10_json_response['encoding']
+        self.has_log = top_10_json_response['hasLog']
+        self.has_cue = top_10_json_response['hasCue']
+        self.scene = top_10_json_response['scene']
+        self.seeders = top_10_json_response['seeders']
+        self.leechers = top_10_json_response['leechers']
+        self.snatched = top_10_json_response['snatched']
 
 
     def __repr__(self):
@@ -125,4 +180,4 @@ class Torrent(object):
             groupname = self.group.name
         else:
             groupname = "Unknown Group"
-        return "Torrent: %s - %s - ID: %s" % (groupname, self.encoding, self.id)
+        return "Torrent: %s - %s - ID: %s" % (groupname, self.encoding, self.id)

lib/pygazelle/torrent_group.py

@@ -1,4 +1,4 @@
-from torrent import Torrent
+from .torrent import Torrent
 
 class InvalidTorrentGroupException(Exception):
     pass
@@ -62,13 +62,17 @@ class TorrentGroup(object):
         self.music_info['with'] = [ self.parent_api.get_artist(artist['id'], artist['name'])
                                        for artist in self.music_info['with'] ]
 
-        self.torrents = []
-        for torrent_dict in torrent_group_json_response['torrents']:
-            torrent_dict['groupId'] = self.id
-            torrent = self.parent_api.get_torrent(torrent_dict['id'])
-            torrent.set_torrent_group_data(torrent_dict)
+        if 'torrents' in torrent_group_json_response:
+            self.torrents = []
+            for torrent_dict in torrent_group_json_response['torrents']:
+                torrent_dict['groupId'] = self.id
+                torrent = self.parent_api.get_torrent(torrent_dict['id'])
+                torrent.set_torrent_group_data(torrent_dict)
+                self.torrents.append(torrent)
+            self.has_complete_torrent_list = True
+        elif 'torrent' in torrent_group_json_response:
+            torrent = self.parent_api.get_torrent(torrent_group_json_response['torrent']['id'])
             self.torrents.append(torrent)
-        self.has_complete_torrent_list = True
 
     def set_artist_group_data(self, artist_group_json_response):
         """
@@ -132,4 +136,4 @@ class TorrentGroup(object):
 
 
     def __repr__(self):
-        return "TorrentGroup: %s - ID: %s" % (self.name, self.id)
+        return "TorrentGroup: %s - ID: %s" % (self.name, self.id)

lib/pygazelle/user.py

@@ -67,9 +67,9 @@ class User(object):
         Takes parsed JSON response from 'user' action on api, and updates relevant user information.
         To avoid problems, only pass in user data from an API call that used this user's ID as an argument.
         """
-        if self.id != user_json_response['id']:
-            raise InvalidUserException("Tried to update a user's information from a 'user' API call with a different id." +
-                               " Should be %s, got %s" % (self.id, user_json_response['id']) )
+        if self.username and self.username != user_json_response['username']:
+            raise InvalidUserException("Tried to update a user's information from a 'user' API call with a different username." +
+                                       " Should be %s, got %s" % (self.username, user_json_response['username']) )
 
         self.username = user_json_response['username']
         self.avatar = user_json_response['avatar']