diff --git a/headphones/searcher.py b/headphones/searcher.py index eb227622..a12c8cdd 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -41,6 +41,10 @@ import lib.bencode as bencode import headphones.searcher_rutracker as rutrackersearch rutracker = rutrackersearch.Rutracker() +# Persistent What.cd API object +gazelle = None + + class NewzbinDownloader(urllib.FancyURLopener): def __init__(self): @@ -735,6 +739,7 @@ def preprocess(resultlist): def searchTorrent(albumid=None, new=False, losslessOnly=False): + global gazelle # persistent what.cd api object to reduce number of login attempts myDB = db.DBConnection() @@ -1024,13 +1029,16 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False): search_formats = [gazelleformat.MP3] maxsize = 300000000 - try: - gazelle = gazelleapi.GazelleAPI(headphones.WHATCD_USERNAME, headphones.WHATCD_PASSWORD) - except Exception, e: - gazelle = None - logger.warn(u"What.cd credentials incorrect or site is down. Error: %s %s" % (e.__class__.__name__, str(e))) + if not gazelle or not gazelle.logged_in(): + try: + logger.info(u"Attempting to log in to What.cd...") + gazelle = gazelleapi.GazelleAPI(headphones.WHATCD_USERNAME, headphones.WHATCD_PASSWORD) + gazelle._login() + except Exception, e: + gazelle = None + logger.error(u"What.cd credentials incorrect or site is down. Error: %s %s" % (e.__class__.__name__, str(e))) - if gazelle: + if gazelle and gazelle.logged_in(): logger.info(u"Searching %s..." % provider) all_torrents = [] for search_format in search_formats: diff --git a/lib/pygazelle/api.py b/lib/pygazelle/api.py index 97240fc4..75272cd5 100644 --- a/lib/pygazelle/api.py +++ b/lib/pygazelle/api.py @@ -5,18 +5,21 @@ # # Loosely based on the API implementation from 'whatbetter', by Zachary Denton # See https://github.com/zacharydenton/whatbetter +from HTMLParser import HTMLParser +import sys import json import time import lib.requests as requests -from user import User -from artist import Artist -from tag import Tag -from request import Request -from torrent_group import TorrentGroup -from torrent import Torrent -from category import Category +from .user import User +from .artist import Artist +from .tag import Tag +from .request import Request +from .torrent_group import TorrentGroup +from .torrent import Torrent +from .category import Category +from .inbox import Mailbox class LoginException(Exception): pass @@ -40,7 +43,8 @@ class GazelleAPI(object): def __init__(self, username=None, password=None): - self.session = requests.session(headers=self.default_headers) + self.session = requests.session() + self.session.headers = self.default_headers self.username = username self.password = password self.authkey = None @@ -55,58 +59,102 @@ class GazelleAPI(object): self.cached_requests = {} self.cached_categories = {} self.site = "https://what.cd/" - self.rate_limit = 2.0 # seconds between requests - self._login() + self.past_request_timestamps = [] + + def wait_for_rate_limit(self): + # maximum is 5 requests within 10 secs + time_frame = 10 + max_reqs = 5 + + slice_point = 0 + + while len(self.past_request_timestamps) >= max_reqs: + for i, timestamp in enumerate(self.past_request_timestamps): + if timestamp < time.time() - time_frame: + slice_point = i + 1 + else: + break + + if slice_point: + self.past_request_timestamps = self.past_request_timestamps[slice_point:] + else: + time.sleep(0.1) + + def logged_in(self): + return self.logged_in_user is not None and self.logged_in_user.id == self.userid def _login(self): """ Private method. Logs in user and gets authkey from server. """ + + if self.logged_in(): + return + + self.wait_for_rate_limit() + loginpage = 'https://what.cd/login.php' data = {'username': self.username, 'password': self.password} r = self.session.post(loginpage, data=data) if r.status_code != 200: - raise LoginException("Login error, http code %s" % r.status_code) - accountinfo = self.request('index') + raise LoginException("Login returned status code %s" % r.status_code) + + try: + accountinfo = self.request('index', autologin=False) + except RequestException as e: + raise LoginException("Login probably incorrect") + if not accountinfo or 'id' not in accountinfo: + raise LoginException("Login probably incorrect") self.userid = accountinfo['id'] self.authkey = accountinfo['authkey'] self.passkey = accountinfo['passkey'] self.logged_in_user = User(self.userid, self) self.logged_in_user.set_index_data(accountinfo) + self.past_request_timestamps.append(time.time()) - def request(self, action, **kwargs): + def request(self, action, autologin=True, **kwargs): """ Makes an AJAX request at a given action. Pass an action and relevant arguments for that action. """ - - ajaxpage = 'ajax.php' - content = self.unparsed_request(ajaxpage, action, **kwargs) - try: - parsed = json.loads(content) - if parsed['status'] != 'success': + def make_request(action, **kwargs): + ajaxpage = 'ajax.php' + content = self.unparsed_request(ajaxpage, action, **kwargs) + try: + if not isinstance(content, text_type): + content = content.decode('utf-8') + parsed = json.loads(content) + if parsed['status'] != 'success': + raise RequestException + return parsed['response'] + except ValueError: raise RequestException - return parsed['response'] - except ValueError: - raise RequestException - def unparsed_request(self, page, action, **kwargs): + try: + return make_request(action, **kwargs) + except Exception as e: + if autologin and not self.logged_in(): + self._login() + return make_request(action, **kwargs) + else: + raise e + + def unparsed_request(self, sitepage, action, **kwargs): """ Makes a generic HTTP request at a given page with a given action. Also pass relevant arguments for that action. """ - while time.time() - self.last_request < self.rate_limit: - time.sleep(0.1) + self.wait_for_rate_limit() - url = "%s/%s" % (self.site, page) + url = "%s%s" % (self.site, sitepage) params = {'action': action} if self.authkey: params['auth'] = self.authkey params.update(kwargs) r = self.session.get(url, params=params, allow_redirects=False) - self.last_request = time.time() + self.past_request_timestamps.append(time.time()) return r.content def get_user(self, id): @@ -141,18 +189,37 @@ class GazelleAPI(object): return found_users - def get_artist(self, id, name=None): + def get_inbox(self, page='1', sort='unread'): + """ + Returns the inbox Mailbox for the logged in user + """ + return Mailbox(self, 'inbox', page, sort) + + def get_sentbox(self, page='1', sort='unread'): + """ + Returns the sentbox Mailbox for the logged in user + """ + return Mailbox(self, 'sentbox', page, sort) + + def get_artist(self, id=None, name=None): """ Returns an Artist for the passed ID, associated with this API object. You'll need to call Artist.update_data() if the artist hasn't already been cached. This is done on demand to reduce unnecessary API calls. """ - id = int(id) - if id in self.cached_artists.keys(): - artist = self.cached_artists[id] + if id: + id = int(id) + if id in self.cached_artists.keys(): + artist = self.cached_artists[id] + else: + artist = Artist(id, self) + if name: + artist.name = HTMLParser().unescape(name) + elif name: + artist = Artist(-1, self) + artist.name = HTMLParser().unescape(name) else: - artist = Artist(id, self) - if name: - artist.name = name + raise Exception("You must specify either an ID or a Name to get an artist.") + return artist def get_tag(self, name): @@ -189,7 +256,7 @@ class GazelleAPI(object): def get_torrent(self, id): """ - Returns a TorrentGroup for the passed ID, associated with this API object. + Returns a Torrent for the passed ID, associated with this API object. """ id = int(id) if id in self.cached_torrents.keys(): @@ -197,6 +264,24 @@ class GazelleAPI(object): else: return Torrent(id, self) + def get_torrent_from_info_hash(self, info_hash): + """ + Returns a Torrent for the passed info hash (if one exists), associated with this API object. + """ + try: + response = self.request(action='torrent', hash=info_hash.upper()) + except RequestException: + return None + + id = int(response['torrent']['id']) + if id in self.cached_torrents.keys(): + torrent = self.cached_torrents[id] + else: + torrent = Torrent(id, self) + + torrent.set_torrent_complete_data(response) + return torrent + def get_category(self, id, name=None): """ Returns a Category for the passed ID, associated with this API object. @@ -210,6 +295,45 @@ class GazelleAPI(object): cat.name = name return cat + def get_top_10(self, type="torrents", limit=25): + """ + Lists the top items of . Type can be "torrents", "tags", or "users". Limit MUST be + 10, 25, or 100...it can't just be an arbitrary number (unfortunately). Results are organized into a list of hashes. + Each hash contains the results for a specific time frame, like 'day', or 'week'. In the hash, the 'results' key + contains a list of objects appropriate to the passed . + """ + + response = self.request(action='top10', type=type, limit=limit) + top_items = [] + if not response: + raise RequestException + for category in response: + results = [] + if type == "torrents": + for item in category['results']: + torrent = self.get_torrent(item['torrentId']) + torrent.set_torrent_top_10_data(item) + results.append(torrent) + elif type == "tags": + for item in category['results']: + tag = self.get_tag(item['name']) + results.append(tag) + elif type == "users": + for item in category['results']: + user = self.get_user(item['id']) + results.append(user) + else: + raise Exception("%s is an invalid type argument for GazelleAPI.get_top_ten()" % type) + + top_items.append({ + "caption": category['caption'], + "tag": category['tag'], + "limit": category['limit'], + "results": results + }) + + return top_items + def search_torrents(self, **kwargs): """ Searches based on the args you pass and returns torrent groups filled with torrents. @@ -281,3 +405,8 @@ class GazelleAPI(object): id=id, authkey=self.logged_in_user.authkey, torrent_pass=self.logged_in_user.passkey) with open(dest, 'w+') as dest_file: dest_file.write(file_data) + +if sys.version_info[0] == 3: + text_type = str +else: + text_type = unicode diff --git a/lib/pygazelle/artist.py b/lib/pygazelle/artist.py index e4c4c513..60779db1 100644 --- a/lib/pygazelle/artist.py +++ b/lib/pygazelle/artist.py @@ -1,4 +1,4 @@ - +from HTMLParser import HTMLParser class InvalidArtistException(Exception): pass @@ -26,15 +26,28 @@ class Artist(object): self.parent_api.cached_artists[self.id] = self # add self to cache of known Artist objects def update_data(self): - response = self.parent_api.request(action='artist', id=self.id) + if self.id > 0: + response = self.parent_api.request(action='artist', id=self.id) + elif self.name: + self.name = HTMLParser().unescape(self.name) + try: + response = self.parent_api.request(action='artist', artistname=self.name) + except Exception: + self.name = self.name.split(" & ")[0] + response = self.parent_api.request(action='artist', artistname=self.name) + else: + raise InvalidArtistException("Neither ID or Artist Name is valid, can't update data.") self.set_data(response) def set_data(self, artist_json_response): - if self.id != artist_json_response['id']: + if self.id > 0 and self.id != artist_json_response['id']: raise InvalidArtistException("Tried to update an artists's information from an 'artist' API call with a different id." + " Should be %s, got %s" % (self.id, artist_json_response['id']) ) + elif self.name: + self.id = artist_json_response['id'] + self.parent_api.cached_artists[self.id] = self - self.name = artist_json_response['name'] + self.name = HTMLParser().unescape(artist_json_response['name']) self.notifications_enabled = artist_json_response['notificationsEnabled'] self.has_bookmarked = artist_json_response['hasBookmarked'] self.image = artist_json_response['image'] diff --git a/lib/pygazelle/inbox.py b/lib/pygazelle/inbox.py new file mode 100644 index 00000000..e5016286 --- /dev/null +++ b/lib/pygazelle/inbox.py @@ -0,0 +1,107 @@ +class MailboxMessage(object): + def __init__(self, api, message): + self.id = message['convId'] + self.conv = Conversation(api, self.id) + self.subject = message['subject'] + self.unread = message['unread'] + self.sticky = message['sticky'] + self.fwd_id = message['forwardedId'] + self.fwd_name = message['forwardedName'] + self.sender_id = message['senderId'] + self.username = message['username'] + self.donor = message['donor'] + self.warned = message['warned'] + self.enabled = message['enabled'] + self.date = message['date'] + + def __repr__(self): + return "MailboxMessage ID %s - %s %s %s" % (self.id, self.subject, self.sender_id, self.username) + + +class ConversationMessage(object): + def __init__(self, msg_resp): + self.id = msg_resp['messageId'] + self.sender_id = msg_resp['senderId'] + self.sender_name = msg_resp['senderName'] + self.sent_date = msg_resp['sentDate'] + self.bb_body = msg_resp['bbBody'] + self.body = msg_resp['body'] + + def __repr__(self): + return "ConversationMessage ID %s - %s %s" % (self.id, self.sender_name, self.sent_date) + + +class Conversation(object): + def __init__(self, api, conv_id): + self.id = conv_id + self.parent_api = api + self.subject = None + self.sticky = None + self.messages = [] + + def __repr__(self): + return "Conversation ID %s - %s" % (self.id, self.subject) + + def set_conv_data(self, conv_resp): + assert self.id == conv_resp['convId'] + self.subject = conv_resp['subject'] + self.sticky = conv_resp['sticky'] + self.messages = [ConversationMessage(m) for m in conv_resp['messages']] + + def update_conv_data(self): + response = self.parent_api.request(action='inbox', + type='viewconv', id=self.id) + self.set_conv_data(response) + + +class Mailbox(object): + """ + This class represents the logged in user's inbox/sentbox + """ + def __init__(self, parent_api, boxtype='inbox', page='1', sort='unread'): + self.parent_api = parent_api + self.boxtype = boxtype + self.current_page = page + self.total_pages = None + self.sort = sort + self.messages = None + + def set_mbox_data(self, mbox_resp): + """ + Takes parsed JSON response from 'inbox' action on api + and updates the available subset of mailbox information. + """ + self.current_page = mbox_resp['currentPage'] + self.total_pages = mbox_resp['pages'] + self.messages = \ + [MailboxMessage(self.parent_api, m) for m in mbox_resp['messages']] + + def update_mbox_data(self): + response = self.parent_api.request(action='inbox', + type=self.boxtype, page=self.current_page, sort=self.sort) + self.set_mbox_data(response) + + def next_page(self): + if not self.total_pages: + raise ValueError("call update_mbox_data() first") + total_pages = int(self.total_pages) + cur_page = int(self.current_page) + if cur_page < total_pages: + return Mailbox(self.parent_api, self.boxtype, + str(cur_page + 1), self.sort) + raise ValueError("Already at page %d/%d" % (cur_page, total_pages)) + + def prev_page(self): + if not self.total_pages: + raise ValueError("call update_mbox_data() first") + total_pages = int(self.total_pages) + cur_page = int(self.current_page) + if cur_page > 1: + return Mailbox(self.parent_api, self.boxtype, + str(cur_page - 1), self.sort) + raise ValueError("Already at page %d/%d" % (cur_page, total_pages)) + + def __repr__(self): + return "Mailbox: %s %s Page %s/%s" \ + % (self.boxtype, self.sort, + self.current_page, self.total_pages) diff --git a/lib/pygazelle/torrent.py b/lib/pygazelle/torrent.py index 01d528d0..730cb841 100644 --- a/lib/pygazelle/torrent.py +++ b/lib/pygazelle/torrent.py @@ -1,3 +1,4 @@ +from HTMLParser import HTMLParser import re class InvalidTorrentException(Exception): @@ -35,6 +36,40 @@ class Torrent(object): self.parent_api.cached_torrents[self.id] = self + def set_torrent_complete_data(self, torrent_json_response): + if self.id != torrent_json_response['torrent']['id']: + raise InvalidTorrentException("Tried to update a Torrent's information from an 'artist' API call with a different id." + + " Should be %s, got %s" % (self.id, torrent_json_response['id']) ) + + self.group = self.parent_api.get_torrent_group(torrent_json_response['group']['id']) + had_complete_list = self.group.has_complete_torrent_list + self.group.set_group_data(torrent_json_response) + self.group.has_complete_torrent_list = had_complete_list + + self.media = torrent_json_response['torrent']['media'] + self.format = torrent_json_response['torrent']['format'] + self.encoding = torrent_json_response['torrent']['encoding'] + self.remaster_year = torrent_json_response['torrent']['remasterYear'] + self.remastered = torrent_json_response['torrent']['remastered'] + self.remaster_title = torrent_json_response['torrent']['remasterTitle'] + self.remaster_record_label = torrent_json_response['torrent']['remasterRecordLabel'] + self.scene = torrent_json_response['torrent']['scene'] + self.has_log = torrent_json_response['torrent']['hasLog'] + self.has_cue = torrent_json_response['torrent']['hasCue'] + self.log_score = torrent_json_response['torrent']['logScore'] + self.file_count = torrent_json_response['torrent']['fileCount'] + self.free_torrent = torrent_json_response['torrent']['freeTorrent'] + self.size = torrent_json_response['torrent']['size'] + self.leechers = torrent_json_response['torrent']['leechers'] + self.seeders = torrent_json_response['torrent']['seeders'] + self.snatched = torrent_json_response['torrent']['snatched'] + self.time = torrent_json_response['torrent']['time'] + self.description = torrent_json_response['torrent']['description'] + self.file_list = [ re.match("(.+){{{(\d+)}}}", item).groups() + for item in torrent_json_response['torrent']['fileList'].split("|||") ] # tuple ( filename, filesize ) + self.file_path = torrent_json_response['torrent']['filePath'] + self.user = self.parent_api.get_user(torrent_json_response['torrent']['userId']) + def set_torrent_artist_data(self, artist_torrent_json_response): if self.id != artist_torrent_json_response['id']: raise InvalidTorrentException("Tried to update a Torrent's information from an 'artist' API call with a different id." + @@ -118,6 +153,26 @@ class Torrent(object): self.free_torrent = search_torrent_json_response['isFreeleech'] or search_torrent_json_response['isPersonalFreeleech'] self.time = search_torrent_json_response['time'] + def set_torrent_top_10_data(self, top_10_json_response): + if self.id != top_10_json_response['torrentId']: + raise InvalidTorrentException("Tried to update a Torrent's information from a 'browse'/search API call with a different id." + + " Should be %s, got %s" % (self.id, top_10_json_response['torrentId']) ) + + # TODO: Add conditionals to handle torrents that aren't music + self.group = self.parent_api.get_torrent_group(top_10_json_response['groupId']) + self.group.name = top_10_json_response['groupName'] + if not self.group.music_info and top_10_json_response['artist']: + self.group.music_info = {'artists': [self.parent_api.get_artist(name=HTMLParser().unescape(top_10_json_response['artist']))]} + self.remaster_title = top_10_json_response['remasterTitle'] + self.media = top_10_json_response['media'] + self.format = top_10_json_response['format'] + self.encoding = top_10_json_response['encoding'] + self.has_log = top_10_json_response['hasLog'] + self.has_cue = top_10_json_response['hasCue'] + self.scene = top_10_json_response['scene'] + self.seeders = top_10_json_response['seeders'] + self.leechers = top_10_json_response['leechers'] + self.snatched = top_10_json_response['snatched'] def __repr__(self): @@ -125,4 +180,4 @@ class Torrent(object): groupname = self.group.name else: groupname = "Unknown Group" - return "Torrent: %s - %s - ID: %s" % (groupname, self.encoding, self.id) \ No newline at end of file + return "Torrent: %s - %s - ID: %s" % (groupname, self.encoding, self.id) diff --git a/lib/pygazelle/torrent_group.py b/lib/pygazelle/torrent_group.py index cb47af61..e9d15625 100644 --- a/lib/pygazelle/torrent_group.py +++ b/lib/pygazelle/torrent_group.py @@ -1,4 +1,4 @@ -from torrent import Torrent +from .torrent import Torrent class InvalidTorrentGroupException(Exception): pass @@ -62,13 +62,17 @@ class TorrentGroup(object): self.music_info['with'] = [ self.parent_api.get_artist(artist['id'], artist['name']) for artist in self.music_info['with'] ] - self.torrents = [] - for torrent_dict in torrent_group_json_response['torrents']: - torrent_dict['groupId'] = self.id - torrent = self.parent_api.get_torrent(torrent_dict['id']) - torrent.set_torrent_group_data(torrent_dict) + if 'torrents' in torrent_group_json_response: + self.torrents = [] + for torrent_dict in torrent_group_json_response['torrents']: + torrent_dict['groupId'] = self.id + torrent = self.parent_api.get_torrent(torrent_dict['id']) + torrent.set_torrent_group_data(torrent_dict) + self.torrents.append(torrent) + self.has_complete_torrent_list = True + elif 'torrent' in torrent_group_json_response: + torrent = self.parent_api.get_torrent(torrent_group_json_response['torrent']['id']) self.torrents.append(torrent) - self.has_complete_torrent_list = True def set_artist_group_data(self, artist_group_json_response): """ @@ -132,4 +136,4 @@ class TorrentGroup(object): def __repr__(self): - return "TorrentGroup: %s - ID: %s" % (self.name, self.id) \ No newline at end of file + return "TorrentGroup: %s - ID: %s" % (self.name, self.id) diff --git a/lib/pygazelle/user.py b/lib/pygazelle/user.py index 8d191107..fc1d6a9a 100644 --- a/lib/pygazelle/user.py +++ b/lib/pygazelle/user.py @@ -67,9 +67,9 @@ class User(object): Takes parsed JSON response from 'user' action on api, and updates relevant user information. To avoid problems, only pass in user data from an API call that used this user's ID as an argument. """ - if self.id != user_json_response['id']: - raise InvalidUserException("Tried to update a user's information from a 'user' API call with a different id." + - " Should be %s, got %s" % (self.id, user_json_response['id']) ) + if self.username and self.username != user_json_response['username']: + raise InvalidUserException("Tried to update a user's information from a 'user' API call with a different username." + + " Should be %s, got %s" % (self.username, user_json_response['username']) ) self.username = user_json_response['username'] self.avatar = user_json_response['avatar']