diff --git a/.gitignore b/.gitignore index b22d691a..1117d204 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,6 @@ _ReSharper*/ /logs .project .pydevproject + + +headphones_docs \ No newline at end of file diff --git a/.pep8 b/.pep8 new file mode 100644 index 00000000..9dc3362c --- /dev/null +++ b/.pep8 @@ -0,0 +1,17 @@ +[pep8] +# E111 indentation is not a multiple of four +# E121 continuation line under-indented for hanging indent +# E122 continuation line missing indentation or outdented +# E124 closing bracket does not match visual indentation +# E125 continuation line with same indent as next logical line +# E126 continuation line over-indented for hanging indent +# E127 continuation line over-indented for visual indent +# E128 continuation line under-indented for visual indent +# E261 at least two spaces before inline comment +# E262 inline comment should start with '# ' +# E265 block comment should start with '# ' +# E302 expected 2 blank lines, found 1 +# E501 line too long (312 > 160 characters) +# E502 the backslash is redundant between brackets +ignore = E111,E121,E122,E123,E124,E125,E126,E127,E128,E261,E262,E265,E302,E501,E502 +max-line-length = 160 \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..6aaf8f33 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,20 @@ +# Travis CI configuration file +# http://about.travis-ci.org/docs/ + +language: python + +# Available Python versions: +# http://about.travis-ci.org/docs/user/ci-environment/#Python-VM-images +python: + - "2.6" + - "2.7" +install: + - pip install pyOpenSSL + - pip install pylint + - pip install pyflakes + - pip install pep8 +script: + - pep8 headphones + - pylint --rcfile=pylintrc headphones + - pyflakes headphones + - nosetests headphones diff --git a/API.md b/API.md new file mode 100644 index 00000000..cf69ef66 --- /dev/null +++ b/API.md @@ -0,0 +1,109 @@ +# API Reference +The API is still pretty new and needs some serious cleaning up on the backend but should be reasonably functional. There are no error codes yet. + +## General structure +The API endpoint is `http://ip:port + HTTP_ROOT + /api?apikey=$apikey&cmd=$command` + +Data response in JSON formatted. If executing a command like "delArtist" or "addArtist" you'll get back an "OK", else, you'll get the data you requested. + +## API methods + +### getIndex +Fetch data from index page. Returns: ArtistName, ArtistSortName, ArtistID, Status, DateAdded, [LatestAlbum, ReleaseDate, AlbumID], HaveTracks, TotalTracks, IncludeExtras, LastUpdated, [ArtworkURL, ThumbURL]: a remote url to the artwork/thumbnail. + +To get the cached image path, see getArtistArt command. ThumbURL is added/updated when an artist is added/updated. If your using the database method to get the artwork, it's more reliable to use the ThumbURL than the ArtworkURL) + +### getArtist&id=$artistid +Fetch artist data. returns the artist object (see above) and album info: Status, AlbumASIN, DateAdded, AlbumTitle, ArtistName, ReleaseDate, AlbumID, ArtistID, Type, ArtworkURL: hosted image path. For cached image, see getAlbumArt command) + +### getAlbum&id=$albumid +Fetch data from album page. Returns the album object, a description object and a tracks object. Tracks contain: AlbumASIN, AlbumTitle, TrackID, Format, TrackDuration (ms), ArtistName, TrackTitle, AlbumID, ArtistID, Location, TrackNumber, CleanName (stripped of punctuation /styling), BitRate) + +### getUpcoming +Returns: Status, AlbumASIN, DateAdded, AlbumTitle, ArtistName, ReleaseDate, AlbumID, ArtistID, Type + +### getWanted +Returns: Status, AlbumASIN, DateAdded, AlbumTitle, ArtistName, ReleaseDate, AlbumID, ArtistID, Type + +### getSimilar +Returns similar artists - with a higher "Count" being more likely to be similar. Returns: Count, ArtistName, ArtistID + +### getHistory +Returns: Status, DateAdded, Title, URL (nzb), FolderName, AlbumID, Size (bytes) + +### getLogs +Not working yet + +### findArtist&name=$artistname[&limit=$limit] +Perform artist query on musicbrainz. Returns: url, score, name, uniquename (contains disambiguation info), id) + +### findAlbum&name=$albumname[&limit=$limit] +Perform album query on musicbrainz. Returns: title, url (artist), id (artist), albumurl, albumid, score, uniquename (artist - with disambiguation) + +### addArtist&id=$artistid +Add an artist to the db by artistid) + +### addAlbum&id=$releaseid +Add an album to the db by album release id + +### delArtist&id=$artistid +Delete artist from db by artistid) + +### pauseArtist&id=$artistid +Pause an artist in db) + +### resumeArtist&id=$artistid +Resume an artist in db) + +### refreshArtist&id=$artistid +Refresh info for artist in db from musicbrainz + +### queueAlbum&id=$albumid[&new=True&lossless=True] +Mark an album as wanted and start the searcher. Optional paramters: 'new' looks for new versions, 'lossless' looks only for lossless versions + +### unqueueAlbum&id=$albumid +Unmark album as wanted / i.e. mark as skipped + +### forceSearch +force search for wanted albums - not launched in a separate thread so it may take a bit to complete +### forceProcess +Force post process albums in download directory - also not launched in a separate thread + +### getVersion +Returns some version information: git_path, install_type, current_version, installed_version, commits_behind + +### checkGithub +Updates the version information above and returns getVersion data + +### shutdown +Shut down headphones + +### restart +Restart headphones + +### update +Update headphones - you may want to check the install type in get version and not allow this if type==exe + +### getArtistArt&id=$artistid +Returns either a relative path to the cached image, or a remote url if the image can't be saved to the cache dir + +getAlbumArt&id=$albumid +see above + +### getArtistInfo&id=$artistid +Returns Summary and Content, both formatted in html. + +### getAlbumInfo&id=$albumid +See above, returns Summary and Content. + +### getArtistThumb&id=$artistid +Returns either a relative path to the cached thumbnail artist image, or an http:// address if the cache dir can't be written to. + +### getAlbumThumb&id=$albumid +See above. + +### choose_specific_download&id=$albumid +Gives you a list of results from searcher.searchforalbum(). Basically runs a normal search, but rather than sorting them and downloading the best result, it dumps the data, which you can then pass on to download_specific_release(). Returns a list of dictionaries with params: title, size, url, provider & kind - all of these values must be passed back to download_specific_release + +### download_specific_release&id=albumid&title=$title&size=$size&url=$url&provider=$provider&kind=$kind +Allows you to manually pass a choose_specific_download release back to searcher.send_to_downloader() \ No newline at end of file diff --git a/API_REFERENCE b/API_REFERENCE deleted file mode 100644 index 79431747..00000000 --- a/API_REFERENCE +++ /dev/null @@ -1,71 +0,0 @@ -The API is still pretty new and needs some serious cleaning up on the backend but should be -reasonably functional. There are no error codes yet, - - -General structure: -http://localhost:8181 + HTTP_ROOT + /api?apikey=$apikey&cmd=$command - -Data returned in json format. If executing a command like "delArtist" or "addArtist" you'll get back an "OK", else, you'll get the data you requested - -$commands¶meters[&optionalparameters]: - - -getIndex (fetch data from index page. Returns: ArtistName, ArtistSortName, ArtistID, Status, DateAdded, - [LatestAlbum, ReleaseDate, AlbumID], HaveTracks, TotalTracks, - IncludeExtras, LastUpdated, [ArtworkURL, ThumbURL]: a remote url to the artwork/thumbnail. To get the cached image path, see getArtistArt command. - ThumbURL is added/updated when an artist is added/updated. If your using the database method to get the artwork, - it's more reliable to use the ThumbURL than the ArtworkURL) - -getArtist&id=$artistid (fetch artist data. returns the artist object (see above) and album info: Status, AlbumASIN, DateAdded, AlbumTitle, ArtistName, ReleaseDate, AlbumID, ArtistID, Type, ArtworkURL: hosted image path. For cached image, see getAlbumArt command) - -getAlbum&id=$albumid (fetch data from album page. Returns the album object, a description object and a tracks object. Tracks contain: AlbumASIN, AlbumTitle, TrackID, Format, TrackDuration (ms), ArtistName, TrackTitle, AlbumID, ArtistID, Location, TrackNumber, CleanName (stripped of punctuation /styling), BitRate) - -getUpcoming (Returns: Status, AlbumASIN, DateAdded, AlbumTitle, ArtistName, ReleaseDate, AlbumID, ArtistID, Type) - -getWanted (Returns: Status, AlbumASIN, DateAdded, AlbumTitle, ArtistName, ReleaseDate, AlbumID, ArtistID, Type) - -getSimilar (Returns similar artists - with a higher "Count" being more likely to be similar. Returns: Count, ArtistName, ArtistID) - -getHistory (Returns: Status, DateAdded, Title, URL (nzb), FolderName, AlbumID, Size (bytes)) - -getLogs (not working yet) - -findArtist&name=$artistname[&limit=$limit] (perform artist query on musicbrainz. Returns: url, score, name, uniquename (contains disambiguation info), id) - -findAlbum&name=$albumname[&limit=$limit] (perform album query on musicbrainz. Returns: title, url (artist), id (artist), albumurl, albumid, score, uniquename (artist - with disambiguation) - -addArtist&id=$artistid (add an artist to the db by artistid) -addAlbum&id=$releaseid (add an album to the db by album release id) - -delArtist&id=$artistid (delete artist from db by artistid) - -pauseArtist&id=$artistid (pause an artist in db) -resumeArtist&id=$artistid (resume an artist in db) - -refreshArtist&id=$artistid (refresh info for artist in db from musicbrainz) - -queueAlbum&id=$albumid[&new=True&lossless=True] (Mark an album as wanted and start the searcher. Optional paramters: 'new' looks for new versions, 'lossless' looks only for lossless versions) -unqueueAlbum&id=$albumid (Unmark album as wanted / i.e. mark as skipped) - -forceSearch (force search for wanted albums - not launched in a separate thread so it may take a bit to complete) -forceProcess (force post process albums in download directory - also not launched in a separate thread) - -getVersion (Returns some version information: git_path, install_type, current_version, installed_version, commits_behind -checkGithub (updates the version information above and returns getVersion data) - -shutdown (shut down headphones) -restart (restart headphones) -update (update headphones - you may want to check the install type in get version and not allow this if type==exe) - -getArtistArt&id=$artistid (Returns either a relative path to the cached image, or a remote url if the image can't be saved to the cache dir) -getAlbumArt&id=$albumid (see above) - -getArtistInfo&id=$artistid (Returns Summary and Content, both formatted in html) -getAlbumInfo&id=$albumid (See above, returns Summary and Content) - -getArtistThumb&id=$artistid (Returns either a relative path to the cached thumbnail artist image, or an http:// address if the cache dir can't be written to) -getAlbumThumb&id=$albumid (see above) - -choose_specific_download&id=$albumid (Gives you a list of results from searcher.searchforalbum(). Basically runs a normal search, but rather than sorting them and downloading the best result, it dumps the data, which you can then pass on to download_specific_release(). Returns a list of dictionaries with params: title, size, url, provider & kind - all of these values must be passed back to download_specific_release) - -download_specific_release&id=albumid&title=$title&size=$size&url=$url&provider=$provider&kind=$kind (Allows you to manually pass a choose_specific_download release back to searcher.send_to_downloader()) \ No newline at end of file diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 00000000..94103e14 --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,7 @@ +v0.5 Released 10 Nov 2014 +------------------------- +- Several bug fixes (please retest your posted issues) +- Cue splitter +- Other improvements + + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..a8d27cca --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,35 @@ +# Contributing to Headphones + +## For users +In case you read this because you are posting an issue, please take a minute and conside the things below. The issue tracker is not a support forum. It is primarily intended to submit bugs, improvements or feature requests. However, we are glad to help you, and make sure the problem is not caused by Headphones, but don't expect step-by-step answers. + +* Use the search function. Chances are that your problem is already discussed. +* Visit the [Troubleshooting](../../wiki/TroubleShooting) wiki first. +* Use [proper formatting](https://help.github.com/articles/github-flavored-markdown/). Paste your logs in code blocks. +* Close your issue if you resolved it. + +## For developers +If you think you can contribute code to the Headphones repository, do not hesitate to submit a pull request. + +### Branches +All pull requests should be based on the `develop` branch. When you want to develop a new feature, clone the repository with `git clone origin/develop -b FEATURE_NAME`. Use meaningful commit messages. + +### Code compatibility +The code should work with Python 2.6 and 2.7. Note that Headphones runs on different platforms, including Network Attached Storage devices such as Synology. + +Re-use existing code. Do not hesitate to add logging in your code. You can the logger module `headphones.logger.*` for this. Web requests are invoked via `headphones.request.*` and derived ones. Use these methods to automatically add proper and meaningful error handling. + +### Code conventions +Altough Headphones did not adapt a code convention in the past, we try to follow the [PEP8](http://legacy.python.org/dev/peps/pep-0008/) conventions for future code. A short summary to remind you (copied from http://wiki.ros.org/PyStyleGuide): + + * 4 space indentation + * 80 characters per line + * `package_name` + * `ClassName` + * `method_name` + * `field_name` + * `_private_something` + * `self.__really_private_field` + * `_global` + +Document your code! \ No newline at end of file diff --git a/Headphones.py b/Headphones.py index 5d83fc68..118494cd 100755 --- a/Headphones.py +++ b/Headphones.py @@ -14,15 +14,14 @@ # You should have received a copy of the GNU General Public License # along with Headphones. If not, see . -import os, sys +import os +import sys # Ensure lib added to path, before any other imports sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'lib/')) from headphones import webstart, logger -from configobj import ConfigObj - import locale import time import signal @@ -33,6 +32,7 @@ import headphones signal.signal(signal.SIGINT, headphones.sig_handler) signal.signal(signal.SIGTERM, headphones.sig_handler) + def main(): """ Headphones application entry point. Parses arguments, setups encoding and @@ -63,16 +63,24 @@ def main(): headphones.SYS_ENCODING = 'UTF-8' # Set up and gather command line arguments - parser = argparse.ArgumentParser(description='Music add-on for SABnzbd+, Transmission and more.') + parser = argparse.ArgumentParser( + description='Music add-on for SABnzbd+, Transmission and more.') - parser.add_argument('-v', '--verbose', action='store_true', help='Increase console logging verbosity') - parser.add_argument('-q', '--quiet', action='store_true', help='Turn off console logging') - parser.add_argument('-d', '--daemon', action='store_true', help='Run as a daemon') - parser.add_argument('-p', '--port', type=int, help='Force Headphones to run on a specified port') - parser.add_argument('--datadir', help='Specify a directory where to store your data files') + parser.add_argument( + '-v', '--verbose', action='store_true', help='Increase console logging verbosity') + parser.add_argument( + '-q', '--quiet', action='store_true', help='Turn off console logging') + parser.add_argument( + '-d', '--daemon', action='store_true', help='Run as a daemon') + parser.add_argument( + '-p', '--port', type=int, help='Force Headphones to run on a specified port') + parser.add_argument( + '--datadir', help='Specify a directory where to store your data files') parser.add_argument('--config', help='Specify a config file to use') - parser.add_argument('--nolaunch', action='store_true', help='Prevent browser from launching on startup') - parser.add_argument('--pidfile', help='Create a pid file (only relevant when running as a daemon)') + parser.add_argument('--nolaunch', action='store_true', + help='Prevent browser from launching on startup') + parser.add_argument( + '--pidfile', help='Create a pid file (only relevant when running as a daemon)') args = parser.parse_args() @@ -83,7 +91,8 @@ def main(): if args.daemon: if sys.platform == 'win32': - sys.stderr.write("Daemonizing not supported under Windows, starting normally\n") + sys.stderr.write( + "Daemonizing not supported under Windows, starting normally\n") else: headphones.DAEMON = True headphones.QUIET = True @@ -91,11 +100,14 @@ def main(): if args.pidfile: headphones.PIDFILE = str(args.pidfile) - # If the pidfile already exists, headphones may still be running, so exit + # If the pidfile already exists, headphones may still be running, so + # exit if os.path.exists(headphones.PIDFILE): - sys.exit("PID file '" + headphones.PIDFILE + "' already exists. Exiting.") + sys.exit( + "PID file '" + headphones.PIDFILE + "' already exists. Exiting.") - # The pidfile is only useful in daemon mode, make sure we can write the file properly + # The pidfile is only useful in daemon mode, make sure we can write the + # file properly if headphones.DAEMON: headphones.CREATEPID = True @@ -103,9 +115,11 @@ def main(): with open(headphones.PIDFILE, 'w') as fp: fp.write("pid\n") except IOError as e: - raise SystemExit("Unable to write PID file: %s [%d]", e.strerror, e.errno) + raise SystemExit( + "Unable to write PID file: %s [%d]", e.strerror, e.errno) else: - logger.warn("Not running in daemon mode. PID file creation disabled.") + logger.warn( + "Not running in daemon mode. PID file creation disabled.") # Determine which data directory and config file to use if args.datadir: @@ -114,27 +128,28 @@ def main(): headphones.DATA_DIR = headphones.PROG_DIR if args.config: - headphones.CONFIG_FILE = args.config + config_file = args.config else: - headphones.CONFIG_FILE = os.path.join(headphones.DATA_DIR, 'config.ini') + config_file = os.path.join(headphones.DATA_DIR, 'config.ini') # Try to create the DATA_DIR if it doesn't exist if not os.path.exists(headphones.DATA_DIR): try: os.makedirs(headphones.DATA_DIR) except OSError: - raise SystemExit('Could not create data directory: ' + headphones.DATA_DIR + '. Exiting....') + raise SystemExit( + 'Could not create data directory: ' + headphones.DATA_DIR + '. Exiting....') # Make sure the DATA_DIR is writeable if not os.access(headphones.DATA_DIR, os.W_OK): - raise SystemExit('Cannot write to the data directory: ' + headphones.DATA_DIR + '. Exiting...') + raise SystemExit( + 'Cannot write to the data directory: ' + headphones.DATA_DIR + '. Exiting...') # Put the database in the DATA_DIR headphones.DB_FILE = os.path.join(headphones.DATA_DIR, 'headphones.db') - headphones.CFG = ConfigObj(headphones.CONFIG_FILE, encoding='utf-8') # Read config and start logging - headphones.initialize() + headphones.initialize(config_file) if headphones.DAEMON: headphones.daemonize() @@ -147,24 +162,25 @@ def main(): http_port = args.port logger.info('Using forced web server port: %i', http_port) else: - http_port = int(headphones.HTTP_PORT) + http_port = int(headphones.CONFIG.HTTP_PORT) # Try to start the server. Will exit here is address is already in use. - webstart.initialize({ + web_config = { 'http_port': http_port, - 'http_host': headphones.HTTP_HOST, - 'http_root': headphones.HTTP_ROOT, - 'http_proxy': headphones.HTTP_PROXY, - 'enable_https': headphones.ENABLE_HTTPS, - 'https_cert': headphones.HTTPS_CERT, - 'https_key': headphones.HTTPS_KEY, - 'http_username': headphones.HTTP_USERNAME, - 'http_password': headphones.HTTP_PASSWORD, - }) + 'http_host': headphones.CONFIG.HTTP_HOST, + 'http_root': headphones.CONFIG.HTTP_ROOT, + 'http_proxy': headphones.CONFIG.HTTP_PROXY, + 'enable_https': headphones.CONFIG.ENABLE_HTTPS, + 'https_cert': headphones.CONFIG.HTTPS_CERT, + 'https_key': headphones.CONFIG.HTTPS_KEY, + 'http_username': headphones.CONFIG.HTTP_USERNAME, + 'http_password': headphones.CONFIG.HTTP_PASSWORD, + } + webstart.initialize(web_config) - if headphones.LAUNCH_BROWSER and not args.nolaunch: - headphones.launch_browser(headphones.HTTP_HOST, http_port, - headphones.HTTP_ROOT) + if headphones.CONFIG.LAUNCH_BROWSER and not args.nolaunch: + headphones.launch_browser(headphones.CONFIG.HTTP_HOST, http_port, + headphones.CONFIG.HTTP_ROOT) # Start the background threads headphones.start() @@ -190,4 +206,4 @@ def main(): # Call main() if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/COPYING b/LICENSE similarity index 100% rename from COPYING rename to LICENSE diff --git a/README.md b/README.md index d1e0af34..eb3a9b35 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,18 @@ -#![preview thumb](https://github.com/rembo10/headphones/raw/master/data/images/headphoneslogo.png)Headphones +#![Headphones Logo](https://github.com/rembo10/headphones/raw/master/data/images/headphoneslogo.png) Headphones -###Support & Discuss +Headphones is an automated music downloader for NZB and Torrent, written in Python. It supports SABnzbd, NZBget, Transmission, µTorrent and Blackhole. -You are free to join the HP support community on IRC where you can ask questions, hang around and discuss anything related to HP. +## Support & Discuss +You are free to join the Headphones support community on IRC where you can ask questions, hang around and discuss anything related to HP. -1. Use any IRC client and connect to the Freenode server. -2. Join #headphones +1. Use any IRC client and connect to the Freenode server, `irc.freenode.net`. +2. Join the `#headphones` channel. -###Installation and Notes +## Installation and Notes -[Read our Wiki](../../wiki) on how to install and use HeadPhones properly. - -[**Troubleshooting** page](../../wiki/TroubleShooting) in the wiki can help you with comon problems. +* [Installation page](../../wiki/Usage-guide) shows you how to install Headphones. +* [Usage guide](../../wiki/Usage-guide) introduces you to Headphones. +* [Troubleshooting page](../../wiki/TroubleShooting) in the wiki can help you with common problems. **Issues** can be reported on the GitHub issue tracker considering these rules: @@ -27,17 +28,15 @@ You are free to join the HP support community on IRC where you can ask questions If you **comply with these rules** you can [post your request/issue](http://github.com/rembo10/headphones/issues). -**Support** the project by implementing new features, solving support tickets and provide bug fixes. -If you change something in the code always make a PR to the developer branch instead of the master branch. +**Support** the project by implementing new features, solving support tickets and provide bug fixes. +## Screenshots -###Screenshots - -Homepage (Artist Overview) +Homepage (Artist Overview): ![preview thumb](http://i.imgur.com/LZO9a.png) -One of the many settings pages.... +One of the many settings pages: ![preview thumb](http://i.imgur.com/xcWNy.png) @@ -49,7 +48,7 @@ Import Your Favorite Artists: ![preview thumb](http://i.imgur.com/6tZoC.png) -Artist Search Results (also search by album!): +Search Results: ![preview thumb](http://i.imgur.com/rIV0P.png) @@ -61,5 +60,5 @@ Album Page with track overview: ![preview thumb](http://i.imgur.com/kcjES.png) - -This is free software under the GPL v3 open source license - so feel free to do with it what you wish. +## License +This is free software under the GPL v3 open source license. Feel free to do with it what you wish, but any modification must be open sourced. A copy of the license is included. diff --git a/data/interfaces/default/artist.html b/data/interfaces/default/artist.html index 1f2753c5..229cef67 100644 --- a/data/interfaces/default/artist.html +++ b/data/interfaces/default/artist.html @@ -163,21 +163,21 @@ } <% - if headphones.SONGKICK_FILTER_ENABLED: + if headphones.CONFIG.SONGKICK_FILTER_ENABLED: songkick_filter_enabled = "true" else: songkick_filter_enabled = "false" - if not headphones.SONGKICK_LOCATION: + if not headphones.CONFIG.SONGKICK_LOCATION: songkick_location = "none" else: - songkick_location = headphones.SONGKICK_LOCATION - - if headphones.SONGKICK_ENABLED: + songkick_location = headphones.CONFIG.SONGKICK_LOCATION + + if headphones.CONFIG.SONGKICK_ENABLED: songkick_enabled = "true" else: songkick_enabled = "false" - + %> function getArtistsCalendar() { var template, calendarDomNode; @@ -186,16 +186,16 @@ template = '
  • NAMELOC
    • '; - $.getJSON("https://api.songkick.com/api/3.0/artists/mbid:${artist['ArtistID']}/calendar.json?apikey=${headphones.SONGKICK_APIKEY}&jsoncallback=?", + $.getJSON("https://api.songkick.com/api/3.0/artists/mbid:${artist['ArtistID']}/calendar.json?apikey=${headphones.CONFIG.SONGKICK_APIKEY}&jsoncallback=?", function(data){ - if (data['resultsPage'].totalEntries >= 1) { - + if (data['resultsPage'].totalEntries >= 1) { + if( ${songkick_filter_enabled} ) { data.resultsPage.results.event = $.grep(data.resultsPage.results.event, function(element,index){ return element.venue.metroArea.id == ${songkick_location}; }); } - + var tourDate; calendarDomNode.show(); @@ -211,7 +211,7 @@ }); calendarDomNode.append('
    • '); - + $(function() { $("#artistCalendar").each(function() { $("li:gt(4)", this).hide(); /* :gt() is zero-indexed */ @@ -269,7 +269,7 @@ $('#dialog').dialog(); event.preventDefault(); }); - $('#menu_link_modifyextra').click(function() { + $('#menu_link_modifyextra').click(function(event) { $('#dialog').dialog(); event.preventDefault(); }); diff --git a/data/interfaces/default/base.html b/data/interfaces/default/base.html index 7b53f1d3..7edfd7cf 100644 --- a/data/interfaces/default/base.html +++ b/data/interfaces/default/base.html @@ -38,7 +38,7 @@ % elif headphones.CURRENT_VERSION != headphones.LATEST_VERSION and headphones.COMMITS_BEHIND > 0 and headphones.INSTALL_TYPE != 'win':
    - A newer version is available. You're ${headphones.COMMITS_BEHIND} commits behind. Update or Close + A newer version is available. You're ${headphones.COMMITS_BEHIND} commits behind. Update or Close
    % endif @@ -96,8 +96,8 @@ %if version.HEADPHONES_VERSION != 'master': (${version.HEADPHONES_VERSION}) %endif - %if headphones.GIT_BRANCH != 'master': - (${headphones.GIT_BRANCH}) + %if headphones.CONFIG.GIT_BRANCH != 'master': + (${headphones.CONFIG.GIT_BRANCH}) %endif diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 3fdb7c1f..47d81587 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -50,13 +50,13 @@ - +
    - +
    @@ -86,7 +86,7 @@
    API
    - + @@ -166,25 +166,25 @@ - +
    - +
    - +
    - +
    @@ -200,19 +200,19 @@ - +
    - +
    - +
    <% if config['nzbget_priority'] == -100: @@ -317,10 +317,27 @@ Folder your Download program watches for Torrents -
    - - - Allow Headphones to open magnet links + +
    + + + + + + + +
    + + Note: opening magnet URL's is not suitable for headless/console/terminal servers.
    @@ -331,11 +348,11 @@
    - +
    - +
    Note: With Transmission, you can specify a different download directory for downloads sent from Headphones. @@ -351,11 +368,11 @@
    - +
    - +
    @@ -382,7 +399,7 @@ NZBs Torrents No Preference -
    + @@ -397,7 +414,8 @@
    Headphones Indexer
    - + +
    @@ -417,7 +435,7 @@
    Custom Newznab Providers
    - +
    @@ -428,7 +446,7 @@
    - +
    @@ -471,7 +489,7 @@
    NZBs.org
    - +
    @@ -483,7 +501,7 @@
    omgwtfnzbs
    - +
    @@ -504,7 +522,7 @@
    The Pirate Bay
    - +
    @@ -521,7 +539,7 @@
    Kick Ass Torrents
    - +
    @@ -538,7 +556,7 @@
    Waffles.fm
    - +
    @@ -559,7 +577,7 @@
    rutracker.org
    - +
    @@ -580,7 +598,7 @@
    What.cd
    - +
    @@ -601,7 +619,7 @@
    Mininova
    - +
    @@ -650,15 +668,15 @@
    Target bitrate - kbps
    + kbps
    - Reject if less than % or more than % of the target size (leave blank for no limit)
    + Reject if less than % or more than % of the target size (leave blank for no limit)
    - +
    @@ -691,6 +709,11 @@
    Post-Processing +
    +
    - + e.g. /Users/name/Music/iTunes or /Volumes/share/music
    - + Set this if you have a separate directory for lossless music
    @@ -1241,7 +1264,7 @@
    - +
    @@ -1303,7 +1326,7 @@ - %for mirror in config['mirror_list']: + %for mirror in config['mirrorlist']: <% - if mirror == headphones.MIRROR: + if mirror == headphones.CONFIG.MIRROR: selected = 'selected="selected"' else: selected = '' @@ -2013,17 +2036,17 @@ $( "#tabs" ).tabs(); }); initActions(); - initConfigCheckbox("#use_headphones_indexer"); - initConfigCheckbox("#usenewznab"); - initConfigCheckbox("#usenzbsorg"); - initConfigCheckbox("#useomgwtfnzbs"); - initConfigCheckbox("#usekat"); - initConfigCheckbox("#usepiratebay"); - initConfigCheckbox("#usemininova"); - initConfigCheckbox("#usewaffles"); - initConfigCheckbox("#userutracker"); - initConfigCheckbox("#usewhatcd"); - initConfigCheckbox("#useapi"); + initConfigCheckbox("#headphones_indexer"); + initConfigCheckbox("#use_newznab"); + initConfigCheckbox("#use_nzbsorg"); + initConfigCheckbox("#use_omgwtfnzbs"); + initConfigCheckbox("#use_kat"); + initConfigCheckbox("#use_piratebay"); + initConfigCheckbox("#use_mininova"); + initConfigCheckbox("#use_waffles"); + initConfigCheckbox("#use_rutracker"); + initConfigCheckbox("#use_whatcd"); + initConfigCheckbox("#api_enabled"); initConfigCheckbox("#enable_https"); diff --git a/data/interfaces/default/css/style.css b/data/interfaces/default/css/style.css index 16a9f735..0fd4a4ea 100644 --- a/data/interfaces/default/css/style.css +++ b/data/interfaces/default/css/style.css @@ -336,6 +336,10 @@ form .row label { padding-top: 7px; width: 175px; } +form .row label.inline { + margin-right: 5px; + width: auto; +} form .row input { margin-right: 5px; } diff --git a/data/interfaces/default/css/style.less b/data/interfaces/default/css/style.less index 5cdc8e21..b03057c2 100644 --- a/data/interfaces/default/css/style.less +++ b/data/interfaces/default/css/style.less @@ -191,6 +191,11 @@ form { line-height: normal; padding-top: 7px; width: 175px; + + &.inline { + margin-right: 5px; + width: auto; + } } input { margin-right: 5px; } input[type=text], input[type=password] { diff --git a/data/interfaces/default/history.html b/data/interfaces/default/history.html index 45ceb31d..4a4531b7 100644 --- a/data/interfaces/default/history.html +++ b/data/interfaces/default/history.html @@ -81,8 +81,11 @@ "sInfoFiltered":"(filtered from _MAX_ total items)"}, "iDisplayLength": 25, "sPaginationType": "full_numbers", - "aaSorting": [] - + "aaSorting": [], + "fnDrawCallback": function (o) { + // Jump to top of page + $('html,body').scrollTop(0); + } }); resetFilters("history"); } diff --git a/data/interfaces/default/index.html b/data/interfaces/default/index.html index 44501a3b..260c0a1a 100644 --- a/data/interfaces/default/index.html +++ b/data/interfaces/default/index.html @@ -131,6 +131,10 @@ }, "fnInitComplete": function(oSettings, json) { + }, + "fnDrawCallback": function (o) { + // Jump to top of page + $('html,body').scrollTop(0); } }); $('#artist_table').on("draw.dt", function () { diff --git a/data/interfaces/default/logs.html b/data/interfaces/default/logs.html index ff7c63b0..8caa608e 100644 --- a/data/interfaces/default/logs.html +++ b/data/interfaces/default/logs.html @@ -46,51 +46,50 @@ <%def name="javascriptIncludes()"> diff --git a/data/interfaces/default/manageartists.html b/data/interfaces/default/manageartists.html index 0bfcc9f9..7a8efa37 100644 --- a/data/interfaces/default/manageartists.html +++ b/data/interfaces/default/manageartists.html @@ -86,31 +86,31 @@ <%def name="javascriptIncludes()"> diff --git a/data/interfaces/default/managemanual.html b/data/interfaces/default/managemanual.html index f4868884..2be6d5a8 100644 --- a/data/interfaces/default/managemanual.html +++ b/data/interfaces/default/managemanual.html @@ -85,24 +85,26 @@ <%def name="javascriptIncludes()"> diff --git a/data/interfaces/default/manageunmatched.html b/data/interfaces/default/manageunmatched.html index 946c8a5b..ab0f7a9c 100644 --- a/data/interfaces/default/manageunmatched.html +++ b/data/interfaces/default/manageunmatched.html @@ -118,22 +118,24 @@ <%def name="javascriptIncludes()"> + + diff --git a/data/interfaces/default/upcoming.html b/data/interfaces/default/upcoming.html index 259e8b3c..dbe7b254 100644 --- a/data/interfaces/default/upcoming.html +++ b/data/interfaces/default/upcoming.html @@ -97,7 +97,7 @@ "oLanguage": { "sEmptyTable": " " }, - "bDestroy":true, + "bDestroy": true, "bFilter": false, "bInfo": false, "bPaginate": false diff --git a/headphones/__init__.py b/headphones/__init__.py index 037cf2e9..8d29811c 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -13,7 +13,8 @@ # You should have received a copy of the GNU General Public License # along with Headphones. If not, see . -# NZBGet support added by CurlyMo as a part of XBian - XBMC on the Raspberry Pi +# NZBGet support added by CurlyMo as a part of +# XBian - XBMC on the Raspberry Pi import os import sys @@ -21,17 +22,34 @@ import subprocess import threading import webbrowser import sqlite3 -import itertools import cherrypy -from apscheduler.scheduler import Scheduler -from configobj import ConfigObj +from apscheduler.schedulers.background import BackgroundScheduler +from apscheduler.triggers.interval import IntervalTrigger -from headphones import versioncheck, logger, version -from headphones.common import * +from headphones import versioncheck, logger +import headphones.config + +# (append new extras to the end) +POSSIBLE_EXTRAS = [ + "single", + "ep", + "compilation", + "soundtrack", + "live", + "remix", + "spokenword", + "audiobook", + "other", + "dj-mix", + "mixtape/street", + "broadcast", + "interview", + "demo" +] -FULL_PATH = None PROG_DIR = None +FULL_PATH = None ARGS = None SIGNAL = None @@ -43,722 +61,105 @@ QUIET = False VERBOSE = False DAEMON = False CREATEPID = False -PIDFILE= None +PIDFILE = None -SCHED = Scheduler() +SCHED = BackgroundScheduler() INIT_LOCK = threading.Lock() -__INITIALIZED__ = False +_INITIALIZED = False started = False DATA_DIR = None -CONFIG_FILE = None -CFG = None -CONFIG_VERSION = None +CONFIG = None DB_FILE = None -LOG_DIR = None LOG_LIST = [] -CACHE_DIR = None - -HTTP_PORT = None -HTTP_HOST = None -HTTP_USERNAME = None -HTTP_PASSWORD = None -HTTP_ROOT = None -HTTP_PROXY = False -LAUNCH_BROWSER = False - -ENABLE_HTTPS = False -HTTPS_CERT = None -HTTPS_KEY = None - -API_ENABLED = False -API_KEY = None - -GIT_PATH = None -GIT_USER = None -GIT_BRANCH = None -DO_NOT_OVERRIDE_GIT_BRANCH = False INSTALL_TYPE = None CURRENT_VERSION = None LATEST_VERSION = None COMMITS_BEHIND = None -CHECK_GITHUB = False -CHECK_GITHUB_ON_STARTUP = False -CHECK_GITHUB_INTERVAL = None - -MUSIC_DIR = None -DESTINATION_DIR = None -LOSSLESS_DESTINATION_DIR = None -FOLDER_FORMAT = None -FILE_FORMAT = None -FILE_UNDERSCORES = False -PATH_TO_XML = None -PREFERRED_QUALITY = None -PREFERRED_BITRATE = None -PREFERRED_BITRATE_HIGH_BUFFER = None -PREFERRED_BITRATE_LOW_BUFFER = None -PREFERRED_BITRATE_ALLOW_LOSSLESS = False -DETECT_BITRATE = False -LOSSLESS_BITRATE_FROM = None -LOSSLESS_BITRATE_TO = None -ADD_ARTISTS = False -CORRECT_METADATA = False -FREEZE_DB = False -MOVE_FILES = False -RENAME_FILES = False -CLEANUP_FILES = False -KEEP_NFO = False -ADD_ALBUM_ART = False -ALBUM_ART_FORMAT = None -EMBED_ALBUM_ART = False -EMBED_LYRICS = False -REPLACE_EXISTING_FOLDERS = False -NZB_DOWNLOADER = None # 0: sabnzbd, 1: nzbget, 2: blackhole -TORRENT_DOWNLOADER = None # 0: blackhole, 1: transmission, 2: utorrent -DOWNLOAD_DIR = None -BLACKHOLE = None -BLACKHOLE_DIR = None -USENET_RETENTION = None -INCLUDE_EXTRAS = False -EXTRAS = None -AUTOWANT_UPCOMING = False -AUTOWANT_ALL = False -AUTOWANT_MANUALLY_ADDED = True -KEEP_TORRENT_FILES = False -PREFER_TORRENTS = None # 0: nzbs, 1: torrents, 2: no preference -OPEN_MAGNET_LINKS = False - -SEARCH_INTERVAL = 360 -LIBRARYSCAN = False -LIBRARYSCAN_INTERVAL = 300 -DOWNLOAD_SCAN_INTERVAL = 5 -UPDATE_DB_INTERVAL = 24 -MB_IGNORE_AGE = 365 -TORRENT_REMOVAL_INTERVAL = 720 - -SAB_HOST = None -SAB_USERNAME = None -SAB_PASSWORD = None -SAB_APIKEY = None -SAB_CATEGORY = None - -NZBGET_USERNAME = None -NZBGET_PASSWORD = None -NZBGET_CATEGORY = None -NZBGET_HOST = None -NZBGET_PRIORITY = 0 - -HEADPHONES_INDEXER = False - -TRANSMISSION_HOST = None -TRANSMISSION_USERNAME = None -TRANSMISSION_PASSWORD = None - -UTORRENT_HOST = None -UTORRENT_USERNAME = None -UTORRENT_PASSWORD = None -UTORRENT_LABEL = None - -NEWZNAB = False -NEWZNAB_HOST = None -NEWZNAB_APIKEY = None -NEWZNAB_ENABLED = False -EXTRA_NEWZNABS = [] - -NZBSORG = False -NZBSORG_UID = None -NZBSORG_HASH = None - -OMGWTFNZBS = False -OMGWTFNZBS_UID = None -OMGWTFNZBS_APIKEY = None - -PREFERRED_WORDS = None -IGNORED_WORDS = None -REQUIRED_WORDS = None - -LASTFM_USERNAME = None - LOSSY_MEDIA_FORMATS = ["mp3", "aac", "ogg", "ape", "m4a", "asf", "wma"] LOSSLESS_MEDIA_FORMATS = ["flac"] MEDIA_FORMATS = LOSSY_MEDIA_FORMATS + LOSSLESS_MEDIA_FORMATS -ALBUM_COMPLETION_PCT = None # This is used in importer.py to determine how complete an album needs to be - to be considered "downloaded". Percentage from 0-100 - -TORRENTBLACKHOLE_DIR = None -NUMBEROFSEEDERS = 10 -KAT = None -KAT_PROXY_URL = None -KAT_RATIO = None -MININOVA = None -MININOVA_RATIO = None -PIRATEBAY = None -PIRATEBAY_PROXY_URL = None -PIRATEBAY_RATIO = None -WAFFLES = None -WAFFLES_UID = None -WAFFLES_PASSKEY = None -WAFFLES_RATIO = None -RUTRACKER = None -RUTRACKER_USER = None -RUTRACKER_PASSWORD = None -RUTRACKER_RATIO = None -WHATCD = None -WHATCD_USERNAME = None -WHATCD_PASSWORD = None -WHATCD_RATIO = None -DOWNLOAD_TORRENT_DIR = None - -INTERFACE = None -FOLDER_PERMISSIONS = None -FILE_PERMISSIONS = None - -MUSIC_ENCODER = False -ENCODERFOLDER = None -ENCODER_PATH = None -ENCODER = None -XLDPROFILE = None -BITRATE = None -SAMPLINGFREQUENCY = None -ADVANCEDENCODER = None -ENCODEROUTPUTFORMAT = None -ENCODERQUALITY = None -ENCODERVBRCBR = None -ENCODERLOSSLESS = False -ENCODER_MULTICORE = False -ENCODER_MULTICORE_COUNT = 0 -DELETE_LOSSLESS_FILES = False -GROWL_ENABLED = True -GROWL_HOST = None -GROWL_PASSWORD = None -GROWL_ONSNATCH = True -PROWL_ENABLED = True -PROWL_PRIORITY = 1 -PROWL_KEYS = None -PROWL_ONSNATCH = True -XBMC_ENABLED = False -XBMC_HOST = None -XBMC_USERNAME = None -XBMC_PASSWORD = None -XBMC_UPDATE = False -XBMC_NOTIFY = False -LMS_ENABLED = False -LMS_HOST = None -PLEX_ENABLED = False -PLEX_SERVER_HOST = None -PLEX_CLIENT_HOST = None -PLEX_USERNAME = None -PLEX_PASSWORD = None -PLEX_UPDATE = False -PLEX_NOTIFY = False -NMA_ENABLED = False -NMA_APIKEY = None -NMA_PRIORITY = 0 -NMA_ONSNATCH = None -PUSHALOT_ENABLED = False -PUSHALOT_APIKEY = None -PUSHALOT_ONSNATCH = None -SYNOINDEX_ENABLED = False -PUSHOVER_ENABLED = True -PUSHOVER_PRIORITY = 1 -PUSHOVER_KEYS = None -PUSHOVER_ONSNATCH = True -PUSHOVER_APITOKEN = None -PUSHBULLET_ENABLED = True -PUSHBULLET_APIKEY = None -PUSHBULLET_DEVICEID = None -PUSHBULLET_ONSNATCH = True -TWITTER_ENABLED = False -TWITTER_ONSNATCH = False -TWITTER_USERNAME = None -TWITTER_PASSWORD = None -TWITTER_PREFIX = None -OSX_NOTIFY_ENABLED = False -OSX_NOTIFY_ONSNATCH = False -OSX_NOTIFY_APP = None -BOXCAR_ENABLED = False -BOXCAR_ONSNATCH = False -BOXCAR_TOKEN = None -SUBSONIC_ENABLED = False -SUBSONIC_HOST = None -SUBSONIC_USERNAME = None -SUBSONIC_PASSWORD = None -MIRRORLIST = ["musicbrainz.org","headphones","custom"] -MIRROR = None -CUSTOMHOST = None -CUSTOMPORT = None -CUSTOMSLEEP = None -HPUSER = None -HPPASS = None -SONGKICK_ENABLED = False -SONGKICK_APIKEY = None -SONGKICK_LOCATION = None -SONGKICK_FILTER_ENABLED = False -MPC_ENABLED = False - -CACHE_SIZEMB = 32 -JOURNAL_MODE = None +MIRRORLIST = ["musicbrainz.org", "headphones", "custom"] UMASK = None -VERIFY_SSL_CERT = True -def CheckSection(sec): - """ Check if INI section exists, if not create it """ - try: - CFG[sec] - return True - except: - CFG[sec] = {} - return False - -################################################################################ -# Check_setting_int # -################################################################################ -def check_setting_int(config, cfg_name, item_name, def_val): - try: - my_val = int(config[cfg_name][item_name]) - except: - my_val = def_val - try: - config[cfg_name][item_name] = my_val - except: - config[cfg_name] = {} - config[cfg_name][item_name] = my_val - logger.debug("%s -> %s", item_name, my_val) - return my_val - -################################################################################ -# Check_setting_str # -################################################################################ -def check_setting_str(config, cfg_name, item_name, def_val, log=True): - try: - my_val = config[cfg_name][item_name] - except: - my_val = def_val - try: - config[cfg_name][item_name] = my_val - except: - config[cfg_name] = {} - config[cfg_name][item_name] = my_val - - logger.debug("%s -> %s", item_name, my_val if log else "******") - return my_val - -def initialize(): +def initialize(config_file): with INIT_LOCK: - global __INITIALIZED__, FULL_PATH, PROG_DIR, VERBOSE, QUIET, DAEMON, SYS_PLATFORM, DATA_DIR, CONFIG_FILE, CFG, CONFIG_VERSION, LOG_DIR, CACHE_DIR, \ - HTTP_PORT, HTTP_HOST, HTTP_USERNAME, HTTP_PASSWORD, HTTP_ROOT, HTTP_PROXY, LAUNCH_BROWSER, API_ENABLED, API_KEY, GIT_PATH, GIT_USER, GIT_BRANCH, DO_NOT_OVERRIDE_GIT_BRANCH, \ - CURRENT_VERSION, LATEST_VERSION, CHECK_GITHUB, CHECK_GITHUB_ON_STARTUP, CHECK_GITHUB_INTERVAL, MUSIC_DIR, DESTINATION_DIR, \ - LOSSLESS_DESTINATION_DIR, PREFERRED_QUALITY, PREFERRED_BITRATE, DETECT_BITRATE, ADD_ARTISTS, CORRECT_METADATA, FREEZE_DB, MOVE_FILES, \ - RENAME_FILES, FOLDER_FORMAT, FILE_FORMAT, FILE_UNDERSCORES, CLEANUP_FILES, KEEP_NFO, INCLUDE_EXTRAS, EXTRAS, AUTOWANT_UPCOMING, AUTOWANT_ALL, AUTOWANT_MANUALLY_ADDED, KEEP_TORRENT_FILES, PREFER_TORRENTS, OPEN_MAGNET_LINKS, \ - ADD_ALBUM_ART, ALBUM_ART_FORMAT, EMBED_ALBUM_ART, EMBED_LYRICS, REPLACE_EXISTING_FOLDERS, DOWNLOAD_DIR, BLACKHOLE, BLACKHOLE_DIR, USENET_RETENTION, SEARCH_INTERVAL, \ - TORRENTBLACKHOLE_DIR, NUMBEROFSEEDERS, KAT, KAT_PROXY_URL, KAT_RATIO, PIRATEBAY, PIRATEBAY_PROXY_URL, PIRATEBAY_RATIO, MININOVA, MININOVA_RATIO, WAFFLES, WAFFLES_UID, WAFFLES_PASSKEY, WAFFLES_RATIO, \ - RUTRACKER, RUTRACKER_USER, RUTRACKER_PASSWORD, RUTRACKER_RATIO, WHATCD, WHATCD_USERNAME, WHATCD_PASSWORD, WHATCD_RATIO, DOWNLOAD_TORRENT_DIR, \ - LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, UPDATE_DB_INTERVAL, MB_IGNORE_AGE, TORRENT_REMOVAL_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \ - NZBGET_USERNAME, NZBGET_PASSWORD, NZBGET_CATEGORY, NZBGET_PRIORITY, NZBGET_HOST, HEADPHONES_INDEXER, NZBMATRIX, TRANSMISSION_HOST, TRANSMISSION_USERNAME, TRANSMISSION_PASSWORD, \ - UTORRENT_HOST, UTORRENT_USERNAME, UTORRENT_PASSWORD, UTORRENT_LABEL, NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, NEWZNAB_ENABLED, EXTRA_NEWZNABS, \ - NZBSORG, NZBSORG_UID, NZBSORG_HASH, OMGWTFNZBS, OMGWTFNZBS_UID, OMGWTFNZBS_APIKEY, \ - NZB_DOWNLOADER, TORRENT_DOWNLOADER, PREFERRED_WORDS, REQUIRED_WORDS, IGNORED_WORDS, LASTFM_USERNAME, \ - INTERFACE, FOLDER_PERMISSIONS, FILE_PERMISSIONS, ENCODERFOLDER, ENCODER_PATH, ENCODER, XLDPROFILE, BITRATE, SAMPLINGFREQUENCY, \ - MUSIC_ENCODER, ADVANCEDENCODER, ENCODEROUTPUTFORMAT, ENCODERQUALITY, ENCODERVBRCBR, ENCODERLOSSLESS, ENCODER_MULTICORE, ENCODER_MULTICORE_COUNT, DELETE_LOSSLESS_FILES, \ - GROWL_ENABLED, GROWL_HOST, GROWL_PASSWORD, GROWL_ONSNATCH, PROWL_ENABLED, PROWL_PRIORITY, PROWL_KEYS, PROWL_ONSNATCH, PUSHOVER_ENABLED, PUSHOVER_PRIORITY, PUSHOVER_KEYS, PUSHOVER_ONSNATCH, PUSHOVER_APITOKEN, MIRRORLIST, \ - TWITTER_ENABLED, TWITTER_ONSNATCH, TWITTER_USERNAME, TWITTER_PASSWORD, TWITTER_PREFIX, OSX_NOTIFY_ENABLED, OSX_NOTIFY_ONSNATCH, OSX_NOTIFY_APP, BOXCAR_ENABLED, BOXCAR_ONSNATCH, BOXCAR_TOKEN, \ - PUSHBULLET_ENABLED, PUSHBULLET_APIKEY, PUSHBULLET_DEVICEID, PUSHBULLET_ONSNATCH, \ - MIRROR, CUSTOMHOST, CUSTOMPORT, CUSTOMSLEEP, HPUSER, HPPASS, XBMC_ENABLED, XBMC_HOST, XBMC_USERNAME, XBMC_PASSWORD, XBMC_UPDATE, \ - XBMC_NOTIFY, LMS_ENABLED, LMS_HOST, NMA_ENABLED, NMA_APIKEY, NMA_PRIORITY, NMA_ONSNATCH, SYNOINDEX_ENABLED, ALBUM_COMPLETION_PCT, PREFERRED_BITRATE_HIGH_BUFFER, \ - PREFERRED_BITRATE_LOW_BUFFER, PREFERRED_BITRATE_ALLOW_LOSSLESS, LOSSLESS_BITRATE_FROM, LOSSLESS_BITRATE_TO, CACHE_SIZEMB, JOURNAL_MODE, UMASK, ENABLE_HTTPS, HTTPS_CERT, HTTPS_KEY, \ - PLEX_ENABLED, PLEX_SERVER_HOST, PLEX_CLIENT_HOST, PLEX_USERNAME, PLEX_PASSWORD, PLEX_UPDATE, PLEX_NOTIFY, PUSHALOT_ENABLED, PUSHALOT_APIKEY, \ - PUSHALOT_ONSNATCH, SONGKICK_ENABLED, SONGKICK_APIKEY, SONGKICK_LOCATION, SONGKICK_FILTER_ENABLED, SUBSONIC_ENABLED, SUBSONIC_HOST, SUBSONIC_USERNAME, SUBSONIC_PASSWORD, VERIFY_SSL_CERT + global CONFIG + global _INITIALIZED + global CURRENT_VERSION + global LATEST_VERSION + global UMASK + CONFIG = headphones.config.Config(config_file) - if __INITIALIZED__: + assert CONFIG is not None + + if _INITIALIZED: return False - # Make sure all the config sections exist - for section in ('General', 'SABnzbd', 'NZBget', 'Transmission', - 'uTorrent', 'Headphones', 'Newznab', 'NZBsorg', - 'omgwtfnzbs', 'Piratebay', 'Kat', 'Mininova', 'Waffles', - 'Rutracker', 'What.cd', 'Growl', 'Prowl', 'Pushover', - 'PushBullet', 'XBMC', 'LMS', 'Plex', 'NMA', 'Pushalot', - 'Synoindex', 'Twitter', 'OSX_Notify', 'Boxcar', - 'Songkick', 'Advanced'): - CheckSection(section) + if CONFIG.HTTP_PORT < 21 or CONFIG.HTTP_PORT > 65535: + headphones.logger.warn( + 'HTTP_PORT out of bounds: 21 < %s < 65535', CONFIG.HTTP_PORT) + CONFIG.HTTP_PORT = 8181 - # Set global variables based on config file or use defaults - CONFIG_VERSION = check_setting_str(CFG, 'General', 'config_version', '0') + if CONFIG.HTTPS_CERT == '': + CONFIG.HTTPS_CERT = os.path.join(DATA_DIR, 'server.crt') + if CONFIG.HTTPS_KEY == '': + CONFIG.HTTPS_KEY = os.path.join(DATA_DIR, 'server.key') - try: - HTTP_PORT = check_setting_int(CFG, 'General', 'http_port', 8181) - except: - HTTP_PORT = 8181 + if not CONFIG.LOG_DIR: + CONFIG.LOG_DIR = os.path.join(DATA_DIR, 'logs') - if HTTP_PORT < 21 or HTTP_PORT > 65535: - HTTP_PORT = 8181 - - HTTP_HOST = check_setting_str(CFG, 'General', 'http_host', '0.0.0.0') - HTTP_USERNAME = check_setting_str(CFG, 'General', 'http_username', '') - HTTP_PASSWORD = check_setting_str(CFG, 'General', 'http_password', '') - HTTP_ROOT = check_setting_str(CFG, 'General', 'http_root', '/') - HTTP_PROXY = bool(check_setting_int(CFG, 'General', 'http_proxy', 0)) - ENABLE_HTTPS = bool(check_setting_int(CFG, 'General', 'enable_https', 0)) - HTTPS_CERT = check_setting_str(CFG, 'General', 'https_cert', os.path.join(DATA_DIR, 'server.crt')) - HTTPS_KEY = check_setting_str(CFG, 'General', 'https_key', os.path.join(DATA_DIR, 'server.key')) - LAUNCH_BROWSER = bool(check_setting_int(CFG, 'General', 'launch_browser', 1)) - API_ENABLED = bool(check_setting_int(CFG, 'General', 'api_enabled', 0)) - API_KEY = check_setting_str(CFG, 'General', 'api_key', '') - GIT_PATH = check_setting_str(CFG, 'General', 'git_path', '') - GIT_USER = check_setting_str(CFG, 'General', 'git_user', 'rembo10') - GIT_BRANCH = check_setting_str(CFG, 'General', 'git_branch', 'master') - DO_NOT_OVERRIDE_GIT_BRANCH = check_setting_int(CFG, 'General', 'do_not_override_git_branch', 0) - LOG_DIR = check_setting_str(CFG, 'General', 'log_dir', '') - CACHE_DIR = check_setting_str(CFG, 'General', 'cache_dir', '') - - CHECK_GITHUB = bool(check_setting_int(CFG, 'General', 'check_github', 1)) - CHECK_GITHUB_ON_STARTUP = bool(check_setting_int(CFG, 'General', 'check_github_on_startup', 1)) - CHECK_GITHUB_INTERVAL = check_setting_int(CFG, 'General', 'check_github_interval', 360) - - MUSIC_DIR = check_setting_str(CFG, 'General', 'music_dir', '') - DESTINATION_DIR = check_setting_str(CFG, 'General', 'destination_dir', '') - LOSSLESS_DESTINATION_DIR = check_setting_str(CFG, 'General', 'lossless_destination_dir', '') - PREFERRED_QUALITY = check_setting_int(CFG, 'General', 'preferred_quality', 0) - PREFERRED_BITRATE = check_setting_str(CFG, 'General', 'preferred_bitrate', '') - PREFERRED_BITRATE_HIGH_BUFFER = check_setting_int(CFG, 'General', 'preferred_bitrate_high_buffer', '') - PREFERRED_BITRATE_LOW_BUFFER = check_setting_int(CFG, 'General', 'preferred_bitrate_low_buffer', '') - PREFERRED_BITRATE_ALLOW_LOSSLESS = bool(check_setting_int(CFG, 'General', 'preferred_bitrate_allow_lossless', 0)) - DETECT_BITRATE = bool(check_setting_int(CFG, 'General', 'detect_bitrate', 0)) - LOSSLESS_BITRATE_FROM = check_setting_int(CFG, 'General', 'lossless_bitrate_from', '') - LOSSLESS_BITRATE_TO = check_setting_int(CFG, 'General', 'lossless_bitrate_to', '') - ADD_ARTISTS = bool(check_setting_int(CFG, 'General', 'auto_add_artists', 1)) - CORRECT_METADATA = bool(check_setting_int(CFG, 'General', 'correct_metadata', 0)) - FREEZE_DB = bool(check_setting_int(CFG, 'General', 'freeze_db', 0)) - MOVE_FILES = bool(check_setting_int(CFG, 'General', 'move_files', 0)) - RENAME_FILES = bool(check_setting_int(CFG, 'General', 'rename_files', 0)) - FOLDER_FORMAT = check_setting_str(CFG, 'General', 'folder_format', 'Artist/Album [Year]') - FILE_FORMAT = check_setting_str(CFG, 'General', 'file_format', 'Track Artist - Album [Year] - Title') - FILE_UNDERSCORES = bool(check_setting_int(CFG, 'General', 'file_underscores', 0)) - CLEANUP_FILES = bool(check_setting_int(CFG, 'General', 'cleanup_files', 0)) - KEEP_NFO = bool(check_setting_int(CFG, 'General', 'keep_nfo', 0)) - ADD_ALBUM_ART = bool(check_setting_int(CFG, 'General', 'add_album_art', 0)) - ALBUM_ART_FORMAT = check_setting_str(CFG, 'General', 'album_art_format', 'folder') - EMBED_ALBUM_ART = bool(check_setting_int(CFG, 'General', 'embed_album_art', 0)) - EMBED_LYRICS = bool(check_setting_int(CFG, 'General', 'embed_lyrics', 0)) - REPLACE_EXISTING_FOLDERS = bool(check_setting_int(CFG, 'General', 'replace_existing_folders', 0)) - NZB_DOWNLOADER = check_setting_int(CFG, 'General', 'nzb_downloader', 0) - TORRENT_DOWNLOADER = check_setting_int(CFG, 'General', 'torrent_downloader', 0) - DOWNLOAD_DIR = check_setting_str(CFG, 'General', 'download_dir', '') - BLACKHOLE = bool(check_setting_int(CFG, 'General', 'blackhole', 0)) - BLACKHOLE_DIR = check_setting_str(CFG, 'General', 'blackhole_dir', '') - USENET_RETENTION = check_setting_int(CFG, 'General', 'usenet_retention', '1500') - INCLUDE_EXTRAS = bool(check_setting_int(CFG, 'General', 'include_extras', 0)) - EXTRAS = check_setting_str(CFG, 'General', 'extras', '') - AUTOWANT_UPCOMING = bool(check_setting_int(CFG, 'General', 'autowant_upcoming', 1)) - AUTOWANT_ALL = bool(check_setting_int(CFG, 'General', 'autowant_all', 0)) - AUTOWANT_MANUALLY_ADDED = bool(check_setting_int(CFG, 'General', 'autowant_manually_added', 1)) - KEEP_TORRENT_FILES = bool(check_setting_int(CFG, 'General', 'keep_torrent_files', 0)) - PREFER_TORRENTS = check_setting_int(CFG, 'General', 'prefer_torrents', 0) - OPEN_MAGNET_LINKS = bool(check_setting_int(CFG, 'General', 'open_magnet_links', 0)) - - SEARCH_INTERVAL = check_setting_int(CFG, 'General', 'search_interval', 1440) - LIBRARYSCAN = bool(check_setting_int(CFG, 'General', 'libraryscan', 1)) - LIBRARYSCAN_INTERVAL = check_setting_int(CFG, 'General', 'libraryscan_interval', 300) - DOWNLOAD_SCAN_INTERVAL = check_setting_int(CFG, 'General', 'download_scan_interval', 5) - UPDATE_DB_INTERVAL = check_setting_int(CFG, 'General', 'update_db_interval', 24) - MB_IGNORE_AGE = check_setting_int(CFG, 'General', 'mb_ignore_age', 365) - TORRENT_REMOVAL_INTERVAL = check_setting_int(CFG, 'General', 'torrent_removal_interval', 720) - - TORRENTBLACKHOLE_DIR = check_setting_str(CFG, 'General', 'torrentblackhole_dir', '') - NUMBEROFSEEDERS = check_setting_str(CFG, 'General', 'numberofseeders', '10') - DOWNLOAD_TORRENT_DIR = check_setting_str(CFG, 'General', 'download_torrent_dir', '') - - KAT = bool(check_setting_int(CFG, 'Kat', 'kat', 0)) - KAT_PROXY_URL = check_setting_str(CFG, 'Kat', 'kat_proxy_url', '') - KAT_RATIO = check_setting_str(CFG, 'Kat', 'kat_ratio', '') - - PIRATEBAY = bool(check_setting_int(CFG, 'Piratebay', 'piratebay', 0)) - PIRATEBAY_PROXY_URL = check_setting_str(CFG, 'Piratebay', 'piratebay_proxy_url', '') - PIRATEBAY_RATIO = check_setting_str(CFG, 'Piratebay', 'piratebay_ratio', '') - - MININOVA = bool(check_setting_int(CFG, 'Mininova', 'mininova', 0)) - MININOVA_RATIO = check_setting_str(CFG, 'Mininova', 'mininova_ratio', '') - - WAFFLES = bool(check_setting_int(CFG, 'Waffles', 'waffles', 0)) - WAFFLES_UID = check_setting_str(CFG, 'Waffles', 'waffles_uid', '') - WAFFLES_PASSKEY = check_setting_str(CFG, 'Waffles', 'waffles_passkey', '') - WAFFLES_RATIO = check_setting_str(CFG, 'Waffles', 'waffles_ratio', '') - - RUTRACKER = bool(check_setting_int(CFG, 'Rutracker', 'rutracker', 0)) - RUTRACKER_USER = check_setting_str(CFG, 'Rutracker', 'rutracker_user', '') - RUTRACKER_PASSWORD = check_setting_str(CFG, 'Rutracker', 'rutracker_password', '') - RUTRACKER_RATIO = check_setting_str(CFG, 'Rutracker', 'rutracker_ratio', '') - - WHATCD = bool(check_setting_int(CFG, 'What.cd', 'whatcd', 0)) - WHATCD_USERNAME = check_setting_str(CFG, 'What.cd', 'whatcd_username', '') - WHATCD_PASSWORD = check_setting_str(CFG, 'What.cd', 'whatcd_password', '') - WHATCD_RATIO = check_setting_str(CFG, 'What.cd', 'whatcd_ratio', '') - - SAB_HOST = check_setting_str(CFG, 'SABnzbd', 'sab_host', '') - SAB_USERNAME = check_setting_str(CFG, 'SABnzbd', 'sab_username', '') - SAB_PASSWORD = check_setting_str(CFG, 'SABnzbd', 'sab_password', '') - SAB_APIKEY = check_setting_str(CFG, 'SABnzbd', 'sab_apikey', '') - SAB_CATEGORY = check_setting_str(CFG, 'SABnzbd', 'sab_category', '') - - NZBGET_USERNAME = check_setting_str(CFG, 'NZBget', 'nzbget_username', 'nzbget') - NZBGET_PASSWORD = check_setting_str(CFG, 'NZBget', 'nzbget_password', '') - NZBGET_CATEGORY = check_setting_str(CFG, 'NZBget', 'nzbget_category', '') - NZBGET_HOST = check_setting_str(CFG, 'NZBget', 'nzbget_host', '') - NZBGET_PRIORITY = check_setting_int(CFG, 'NZBget', 'nzbget_priority', 0) - - HEADPHONES_INDEXER = bool(check_setting_int(CFG, 'Headphones', 'headphones_indexer', 0)) - - TRANSMISSION_HOST = check_setting_str(CFG, 'Transmission', 'transmission_host', '') - TRANSMISSION_USERNAME = check_setting_str(CFG, 'Transmission', 'transmission_username', '') - TRANSMISSION_PASSWORD = check_setting_str(CFG, 'Transmission', 'transmission_password', '') - - UTORRENT_HOST = check_setting_str(CFG, 'uTorrent', 'utorrent_host', '') - UTORRENT_USERNAME = check_setting_str(CFG, 'uTorrent', 'utorrent_username', '') - UTORRENT_PASSWORD = check_setting_str(CFG, 'uTorrent', 'utorrent_password', '') - UTORRENT_LABEL = check_setting_str(CFG, 'uTorrent', 'utorrent_label', '') - - NEWZNAB = bool(check_setting_int(CFG, 'Newznab', 'newznab', 0)) - NEWZNAB_HOST = check_setting_str(CFG, 'Newznab', 'newznab_host', '') - NEWZNAB_APIKEY = check_setting_str(CFG, 'Newznab', 'newznab_apikey', '') - NEWZNAB_ENABLED = bool(check_setting_int(CFG, 'Newznab', 'newznab_enabled', 1)) - - # Need to pack the extra newznabs back into a list of tuples - flattened_newznabs = check_setting_str(CFG, 'Newznab', 'extra_newznabs', [], log=False) - EXTRA_NEWZNABS = list(itertools.izip(*[itertools.islice(flattened_newznabs, i, None, 3) for i in range(3)])) - - NZBSORG = bool(check_setting_int(CFG, 'NZBsorg', 'nzbsorg', 0)) - NZBSORG_UID = check_setting_str(CFG, 'NZBsorg', 'nzbsorg_uid', '') - NZBSORG_HASH = check_setting_str(CFG, 'NZBsorg', 'nzbsorg_hash', '') - - OMGWTFNZBS = bool(check_setting_int(CFG, 'omgwtfnzbs', 'omgwtfnzbs', 0)) - OMGWTFNZBS_UID = check_setting_str(CFG, 'omgwtfnzbs', 'omgwtfnzbs_uid', '') - OMGWTFNZBS_APIKEY = check_setting_str(CFG, 'omgwtfnzbs', 'omgwtfnzbs_apikey', '') - - PREFERRED_WORDS = check_setting_str(CFG, 'General', 'preferred_words', '') - IGNORED_WORDS = check_setting_str(CFG, 'General', 'ignored_words', '') - REQUIRED_WORDS = check_setting_str(CFG, 'General', 'required_words', '') - - LASTFM_USERNAME = check_setting_str(CFG, 'General', 'lastfm_username', '') - - INTERFACE = check_setting_str(CFG, 'General', 'interface', 'default') - FOLDER_PERMISSIONS = check_setting_str(CFG, 'General', 'folder_permissions', '0755') - FILE_PERMISSIONS = check_setting_str(CFG, 'General', 'file_permissions', '0644') - - ENCODERFOLDER = check_setting_str(CFG, 'General', 'encoderfolder', '') - ENCODER_PATH = check_setting_str(CFG, 'General', 'encoder_path', '') - ENCODER = check_setting_str(CFG, 'General', 'encoder', 'ffmpeg') - XLDPROFILE = check_setting_str(CFG, 'General', 'xldprofile', '') - BITRATE = check_setting_int(CFG, 'General', 'bitrate', 192) - SAMPLINGFREQUENCY= check_setting_int(CFG, 'General', 'samplingfrequency', 44100) - MUSIC_ENCODER = bool(check_setting_int(CFG, 'General', 'music_encoder', 0)) - ADVANCEDENCODER = check_setting_str(CFG, 'General', 'advancedencoder', '') - ENCODEROUTPUTFORMAT = check_setting_str(CFG, 'General', 'encoderoutputformat', 'mp3') - ENCODERQUALITY = check_setting_int(CFG, 'General', 'encoderquality', 2) - ENCODERVBRCBR = check_setting_str(CFG, 'General', 'encodervbrcbr', 'cbr') - ENCODERLOSSLESS = bool(check_setting_int(CFG, 'General', 'encoderlossless', 1)) - ENCODER_MULTICORE = bool(check_setting_int(CFG, 'General', 'encoder_multicore', 0)) - ENCODER_MULTICORE_COUNT = max(0, check_setting_int(CFG, 'General', 'encoder_multicore_count', 0)) - DELETE_LOSSLESS_FILES = bool(check_setting_int(CFG, 'General', 'delete_lossless_files', 1)) - - GROWL_ENABLED = bool(check_setting_int(CFG, 'Growl', 'growl_enabled', 0)) - GROWL_HOST = check_setting_str(CFG, 'Growl', 'growl_host', '') - GROWL_PASSWORD = check_setting_str(CFG, 'Growl', 'growl_password', '') - GROWL_ONSNATCH = bool(check_setting_int(CFG, 'Growl', 'growl_onsnatch', 0)) - - PROWL_ENABLED = bool(check_setting_int(CFG, 'Prowl', 'prowl_enabled', 0)) - PROWL_KEYS = check_setting_str(CFG, 'Prowl', 'prowl_keys', '') - PROWL_ONSNATCH = bool(check_setting_int(CFG, 'Prowl', 'prowl_onsnatch', 0)) - PROWL_PRIORITY = check_setting_int(CFG, 'Prowl', 'prowl_priority', 0) - - XBMC_ENABLED = bool(check_setting_int(CFG, 'XBMC', 'xbmc_enabled', 0)) - XBMC_HOST = check_setting_str(CFG, 'XBMC', 'xbmc_host', '') - XBMC_USERNAME = check_setting_str(CFG, 'XBMC', 'xbmc_username', '') - XBMC_PASSWORD = check_setting_str(CFG, 'XBMC', 'xbmc_password', '') - XBMC_UPDATE = bool(check_setting_int(CFG, 'XBMC', 'xbmc_update', 0)) - XBMC_NOTIFY = bool(check_setting_int(CFG, 'XBMC', 'xbmc_notify', 0)) - - LMS_ENABLED = bool(check_setting_int(CFG, 'LMS', 'lms_enabled', 0)) - LMS_HOST = check_setting_str(CFG, 'LMS', 'lms_host', '') - - PLEX_ENABLED = bool(check_setting_int(CFG, 'Plex', 'plex_enabled', 0)) - PLEX_SERVER_HOST = check_setting_str(CFG, 'Plex', 'plex_server_host', '') - PLEX_CLIENT_HOST = check_setting_str(CFG, 'Plex', 'plex_client_host', '') - PLEX_USERNAME = check_setting_str(CFG, 'Plex', 'plex_username', '') - PLEX_PASSWORD = check_setting_str(CFG, 'Plex', 'plex_password', '') - PLEX_UPDATE = bool(check_setting_int(CFG, 'Plex', 'plex_update', 0)) - PLEX_NOTIFY = bool(check_setting_int(CFG, 'Plex', 'plex_notify', 0)) - - NMA_ENABLED = bool(check_setting_int(CFG, 'NMA', 'nma_enabled', 0)) - NMA_APIKEY = check_setting_str(CFG, 'NMA', 'nma_apikey', '') - NMA_PRIORITY = check_setting_int(CFG, 'NMA', 'nma_priority', 0) - NMA_ONSNATCH = bool(check_setting_int(CFG, 'NMA', 'nma_onsnatch', 0)) - - PUSHALOT_ENABLED = bool(check_setting_int(CFG, 'Pushalot', 'pushalot_enabled', 0)) - PUSHALOT_APIKEY = check_setting_str(CFG, 'Pushalot', 'pushalot_apikey', '') - PUSHALOT_ONSNATCH = bool(check_setting_int(CFG, 'Pushalot', 'pushalot_onsnatch', 0)) - - SYNOINDEX_ENABLED = bool(check_setting_int(CFG, 'Synoindex', 'synoindex_enabled', 0)) - - PUSHOVER_ENABLED = bool(check_setting_int(CFG, 'Pushover', 'pushover_enabled', 0)) - PUSHOVER_KEYS = check_setting_str(CFG, 'Pushover', 'pushover_keys', '') - PUSHOVER_ONSNATCH = bool(check_setting_int(CFG, 'Pushover', 'pushover_onsnatch', 0)) - PUSHOVER_PRIORITY = check_setting_int(CFG, 'Pushover', 'pushover_priority', 0) - PUSHOVER_APITOKEN = check_setting_str(CFG, 'Pushover', 'pushover_apitoken', '') - - PUSHBULLET_ENABLED = bool(check_setting_int(CFG, 'PushBullet', 'pushbullet_enabled', 0)) - PUSHBULLET_APIKEY = check_setting_str(CFG, 'PushBullet', 'pushbullet_apikey', '') - PUSHBULLET_DEVICEID = check_setting_str(CFG, 'PushBullet', 'pushbullet_deviceid', '') - PUSHBULLET_ONSNATCH = bool(check_setting_int(CFG, 'PushBullet', 'pushbullet_onsnatch', 0)) - - TWITTER_ENABLED = bool(check_setting_int(CFG, 'Twitter', 'twitter_enabled', 0)) - TWITTER_ONSNATCH = bool(check_setting_int(CFG, 'Twitter', 'twitter_onsnatch', 0)) - TWITTER_USERNAME = check_setting_str(CFG, 'Twitter', 'twitter_username', '') - TWITTER_PASSWORD = check_setting_str(CFG, 'Twitter', 'twitter_password', '') - TWITTER_PREFIX = check_setting_str(CFG, 'Twitter', 'twitter_prefix', 'Headphones') - - OSX_NOTIFY_ENABLED = bool(check_setting_int(CFG, 'OSX_Notify', 'osx_notify_enabled', 0)) - OSX_NOTIFY_ONSNATCH = bool(check_setting_int(CFG, 'OSX_Notify', 'osx_notify_onsnatch', 0)) - OSX_NOTIFY_APP = check_setting_str(CFG, 'OSX_Notify', 'osx_notify_app', '/Applications/Headphones') - - BOXCAR_ENABLED = bool(check_setting_int(CFG, 'Boxcar', 'boxcar_enabled', 0)) - BOXCAR_ONSNATCH = bool(check_setting_int(CFG, 'Boxcar', 'boxcar_onsnatch', 0)) - BOXCAR_TOKEN = check_setting_str(CFG, 'Boxcar', 'boxcar_token', '') - - SUBSONIC_ENABLED = bool(check_setting_int(CFG, 'Subsonic', 'subsonic_enabled', 0)) - SUBSONIC_HOST = check_setting_str(CFG, 'Subsonic', 'subsonic_host', '') - SUBSONIC_USERNAME = check_setting_str(CFG, 'Subsonic', 'subsonic_username', '') - SUBSONIC_PASSWORD = check_setting_str(CFG, 'Subsonic', 'subsonic_password', '') - - SONGKICK_ENABLED = bool(check_setting_int(CFG, 'Songkick', 'songkick_enabled', 1)) - SONGKICK_APIKEY = check_setting_str(CFG, 'Songkick', 'songkick_apikey', 'nd1We7dFW2RqxPw8') - SONGKICK_LOCATION = check_setting_str(CFG, 'Songkick', 'songkick_location', '') - SONGKICK_FILTER_ENABLED = bool(check_setting_int(CFG, 'Songkick', 'songkick_filter_enabled', 0)) - - MIRROR = check_setting_str(CFG, 'General', 'mirror', 'musicbrainz.org') - CUSTOMHOST = check_setting_str(CFG, 'General', 'customhost', 'localhost') - CUSTOMPORT = check_setting_int(CFG, 'General', 'customport', 5000) - CUSTOMSLEEP = check_setting_int(CFG, 'General', 'customsleep', 1) - HPUSER = check_setting_str(CFG, 'General', 'hpuser', '') - HPPASS = check_setting_str(CFG, 'General', 'hppass', '') - - CACHE_SIZEMB = check_setting_int(CFG,'Advanced','cache_sizemb',32) - JOURNAL_MODE = check_setting_int(CFG,'Advanced', 'journal_mode', 'wal') - - ALBUM_COMPLETION_PCT = check_setting_int(CFG, 'Advanced', 'album_completion_pct', 80) - - VERIFY_SSL_CERT = bool(check_setting_int(CFG, 'Advanced', 'verify_ssl_cert', 1)) - - # update folder formats in the config & bump up config version - if CONFIG_VERSION == '0': - from headphones.helpers import replace_all - file_values = { 'tracknumber': 'Track', 'title': 'Title','artist' : 'Artist', 'album' : 'Album', 'year' : 'Year' } - folder_values = { 'artist' : 'Artist', 'album':'Album', 'year' : 'Year', 'releasetype' : 'Type', 'first' : 'First', 'lowerfirst' : 'first' } - FILE_FORMAT = replace_all(FILE_FORMAT, file_values) - FOLDER_FORMAT = replace_all(FOLDER_FORMAT, folder_values) - - CONFIG_VERSION = '1' - - if CONFIG_VERSION == '1': - - from headphones.helpers import replace_all - - file_values = { 'Track': '$Track', - 'Title': '$Title', - 'Artist': '$Artist', - 'Album': '$Album', - 'Year': '$Year', - 'track': '$track', - 'title': '$title', - 'artist': '$artist', - 'album': '$album', - 'year': '$year' - } - folder_values = { 'Artist': '$Artist', - 'Album': '$Album', - 'Year': '$Year', - 'Type': '$Type', - 'First': '$First', - 'artist': '$artist', - 'album': '$album', - 'year': '$year', - 'type': '$type', - 'first': '$first' - } - FILE_FORMAT = replace_all(FILE_FORMAT, file_values) - FOLDER_FORMAT = replace_all(FOLDER_FORMAT, folder_values) - - CONFIG_VERSION = '2' - - if CONFIG_VERSION == '2': - - # Update the config to use direct path to the encoder rather than the encoder folder - if ENCODERFOLDER: - ENCODER_PATH = os.path.join(ENCODERFOLDER, ENCODER) - CONFIG_VERSION = '3' - - if CONFIG_VERSION == '3': - #Update the BLACKHOLE option to the NZB_DOWNLOADER format - if BLACKHOLE: - NZB_DOWNLOADER = 2 - CONFIG_VERSION = '4' - - # Enable Headphones Indexer if they have a VIP account - if CONFIG_VERSION == '4': - if HPUSER and HPPASS: - HEADPHONES_INDEXER = True - CONFIG_VERSION = '5' - - if not LOG_DIR: - LOG_DIR = os.path.join(DATA_DIR, 'logs') - - if not os.path.exists(LOG_DIR): + if not os.path.exists(CONFIG.LOG_DIR): try: - os.makedirs(LOG_DIR) + os.makedirs(CONFIG.LOG_DIR) except OSError: if VERBOSE: - sys.stderr.write('Unable to create the log directory. Logging to screen only.\n') + sys.stderr.write( + 'Unable to create the log directory. Logging to screen only.\n') # Start the logger, disable console if needed logger.initLogger(console=not QUIET, verbose=VERBOSE) - if not CACHE_DIR: + if not CONFIG.CACHE_DIR: # Put the cache dir in the data dir for now - CACHE_DIR = os.path.join(DATA_DIR, 'cache') - if not os.path.exists(CACHE_DIR): + CONFIG.CACHE_DIR = os.path.join(DATA_DIR, 'cache') + if not os.path.exists(CONFIG.CACHE_DIR): try: - os.makedirs(CACHE_DIR) + os.makedirs(CONFIG.CACHE_DIR) except OSError: - logger.error('Could not create cache dir. Check permissions of datadir: %s', DATA_DIR) + logger.error( + 'Could not create cache dir. Check permissions of datadir: %s', DATA_DIR) # Sanity check for search interval. Set it to at least 6 hours - if SEARCH_INTERVAL < 360: + if CONFIG.SEARCH_INTERVAL < 360: logger.info("Search interval too low. Resetting to 6 hour minimum") - SEARCH_INTERVAL = 360 + CONFIG.SEARCH_INTERVAL = 360 # Initialize the database logger.info('Checking to see if the database has all tables....') try: dbcheck() - except Exception, e: + except Exception as e: logger.error("Can't connect to the database: %s", e) # Get the currently installed version - returns None, 'win32' or the git hash # Also sets INSTALL_TYPE variable to 'win', 'git' or 'source' - CURRENT_VERSION, GIT_BRANCH = versioncheck.getVersion() + CURRENT_VERSION, CONFIG.GIT_BRANCH = versioncheck.getVersion() # Check for new versions - if CHECK_GITHUB_ON_STARTUP: + if CONFIG.CHECK_GITHUB_ON_STARTUP: try: LATEST_VERSION = versioncheck.checkGithub() except: @@ -771,14 +172,16 @@ def initialize(): UMASK = os.umask(0) os.umask(UMASK) - __INITIALIZED__ = True + _INITIALIZED = True return True -def daemonize(): +def daemonize(): if threading.activeCount() != 1: - logger.warn('There are %r active threads. Daemonizing may cause \ - strange behavior.' % threading.enumerate()) + logger.warn( + 'There are %r active threads. Daemonizing may cause' + ' strange behavior.', + threading.enumerate()) sys.stdout.flush() sys.stderr.flush() @@ -824,12 +227,13 @@ def daemonize(): with file(PIDFILE, 'w') as fp: fp.write("%s\n" % pid) + def launch_browser(host, port, root): if host == '0.0.0.0': host = 'localhost' - if ENABLE_HTTPS: + if CONFIG.ENABLE_HTTPS: protocol = 'https' else: protocol = 'http' @@ -839,377 +243,111 @@ def launch_browser(host, port, root): except Exception as e: logger.error('Could not launch browser: %s', e) -def config_write(): - """ - Write configuration to file. If an IOError occures during a write, it will - be caught. - """ - - new_config = ConfigObj(encoding="UTF-8") - new_config.filename = CONFIG_FILE - - new_config['General'] = {} - new_config['General']['config_version'] = CONFIG_VERSION - new_config['General']['http_port'] = HTTP_PORT - new_config['General']['http_host'] = HTTP_HOST - new_config['General']['http_username'] = HTTP_USERNAME - new_config['General']['http_password'] = HTTP_PASSWORD - new_config['General']['http_root'] = HTTP_ROOT - new_config['General']['http_proxy'] = int(HTTP_PROXY) - new_config['General']['enable_https'] = int(ENABLE_HTTPS) - new_config['General']['https_cert'] = HTTPS_CERT - new_config['General']['https_key'] = HTTPS_KEY - new_config['General']['launch_browser'] = int(LAUNCH_BROWSER) - new_config['General']['api_enabled'] = int(API_ENABLED) - new_config['General']['api_key'] = API_KEY - new_config['General']['log_dir'] = LOG_DIR - new_config['General']['cache_dir'] = CACHE_DIR - new_config['General']['git_path'] = GIT_PATH - new_config['General']['git_user'] = GIT_USER - new_config['General']['git_branch'] = GIT_BRANCH - new_config['General']['do_not_override_git_branch'] = int(DO_NOT_OVERRIDE_GIT_BRANCH) - - new_config['General']['check_github'] = int(CHECK_GITHUB) - new_config['General']['check_github_on_startup'] = int(CHECK_GITHUB_ON_STARTUP) - new_config['General']['check_github_interval'] = CHECK_GITHUB_INTERVAL - - new_config['General']['music_dir'] = MUSIC_DIR - new_config['General']['destination_dir'] = DESTINATION_DIR - new_config['General']['lossless_destination_dir'] = LOSSLESS_DESTINATION_DIR - new_config['General']['preferred_quality'] = PREFERRED_QUALITY - new_config['General']['preferred_bitrate'] = PREFERRED_BITRATE - new_config['General']['preferred_bitrate_high_buffer'] = PREFERRED_BITRATE_HIGH_BUFFER - new_config['General']['preferred_bitrate_low_buffer'] = PREFERRED_BITRATE_LOW_BUFFER - new_config['General']['preferred_bitrate_allow_lossless'] = int(PREFERRED_BITRATE_ALLOW_LOSSLESS) - new_config['General']['detect_bitrate'] = int(DETECT_BITRATE) - new_config['General']['lossless_bitrate_from'] = LOSSLESS_BITRATE_FROM - new_config['General']['lossless_bitrate_to'] = LOSSLESS_BITRATE_TO - new_config['General']['auto_add_artists'] = int(ADD_ARTISTS) - new_config['General']['correct_metadata'] = int(CORRECT_METADATA) - new_config['General']['freeze_db'] = int(FREEZE_DB) - new_config['General']['move_files'] = int(MOVE_FILES) - new_config['General']['rename_files'] = int(RENAME_FILES) - new_config['General']['folder_format'] = FOLDER_FORMAT - new_config['General']['file_format'] = FILE_FORMAT - new_config['General']['file_underscores'] = int(FILE_UNDERSCORES) - new_config['General']['cleanup_files'] = int(CLEANUP_FILES) - new_config['General']['keep_nfo'] = int(KEEP_NFO) - new_config['General']['add_album_art'] = int(ADD_ALBUM_ART) - new_config['General']['album_art_format'] = ALBUM_ART_FORMAT - new_config['General']['embed_album_art'] = int(EMBED_ALBUM_ART) - new_config['General']['embed_lyrics'] = int(EMBED_LYRICS) - new_config['General']['replace_existing_folders'] = int(REPLACE_EXISTING_FOLDERS) - new_config['General']['nzb_downloader'] = NZB_DOWNLOADER - new_config['General']['torrent_downloader'] = TORRENT_DOWNLOADER - new_config['General']['download_dir'] = DOWNLOAD_DIR - new_config['General']['blackhole_dir'] = BLACKHOLE_DIR - new_config['General']['usenet_retention'] = USENET_RETENTION - new_config['General']['include_extras'] = int(INCLUDE_EXTRAS) - new_config['General']['extras'] = EXTRAS - new_config['General']['autowant_upcoming'] = int(AUTOWANT_UPCOMING) - new_config['General']['autowant_all'] = int(AUTOWANT_ALL) - new_config['General']['autowant_manually_added'] = int(AUTOWANT_MANUALLY_ADDED) - new_config['General']['keep_torrent_files'] = int(KEEP_TORRENT_FILES) - new_config['General']['prefer_torrents'] = PREFER_TORRENTS - new_config['General']['open_magnet_links'] = OPEN_MAGNET_LINKS - - new_config['General']['numberofseeders'] = NUMBEROFSEEDERS - new_config['General']['torrentblackhole_dir'] = TORRENTBLACKHOLE_DIR - new_config['General']['download_torrent_dir'] = DOWNLOAD_TORRENT_DIR - - new_config['Kat'] = {} - new_config['Kat']['kat'] = int(KAT) - new_config['Kat']['kat_proxy_url'] = KAT_PROXY_URL - new_config['Kat']['kat_ratio'] = KAT_RATIO - - new_config['Mininova'] = {} - new_config['Mininova']['mininova'] = int(MININOVA) - new_config['Mininova']['mininova_ratio'] = MININOVA_RATIO - - new_config['Piratebay'] = {} - new_config['Piratebay']['piratebay'] = int(PIRATEBAY) - new_config['Piratebay']['piratebay_proxy_url'] = PIRATEBAY_PROXY_URL - new_config['Piratebay']['piratebay_ratio'] = PIRATEBAY_RATIO - - new_config['Waffles'] = {} - new_config['Waffles']['waffles'] = int(WAFFLES) - new_config['Waffles']['waffles_uid'] = WAFFLES_UID - new_config['Waffles']['waffles_passkey'] = WAFFLES_PASSKEY - new_config['Waffles']['waffles_ratio'] = WAFFLES_RATIO - - new_config['Rutracker'] = {} - new_config['Rutracker']['rutracker'] = int(RUTRACKER) - new_config['Rutracker']['rutracker_user'] = RUTRACKER_USER - new_config['Rutracker']['rutracker_password'] = RUTRACKER_PASSWORD - new_config['Rutracker']['rutracker_ratio'] = RUTRACKER_RATIO - - new_config['What.cd'] = {} - new_config['What.cd']['whatcd'] = int(WHATCD) - new_config['What.cd']['whatcd_username'] = WHATCD_USERNAME - new_config['What.cd']['whatcd_password'] = WHATCD_PASSWORD - new_config['What.cd']['whatcd_ratio'] = WHATCD_RATIO - - new_config['General']['search_interval'] = SEARCH_INTERVAL - new_config['General']['libraryscan'] = int(LIBRARYSCAN) - new_config['General']['libraryscan_interval'] = LIBRARYSCAN_INTERVAL - new_config['General']['download_scan_interval'] = DOWNLOAD_SCAN_INTERVAL - new_config['General']['update_db_interval'] = UPDATE_DB_INTERVAL - new_config['General']['mb_ignore_age'] = MB_IGNORE_AGE - new_config['General']['torrent_removal_interval'] = TORRENT_REMOVAL_INTERVAL - - new_config['SABnzbd'] = {} - new_config['SABnzbd']['sab_host'] = SAB_HOST - new_config['SABnzbd']['sab_username'] = SAB_USERNAME - new_config['SABnzbd']['sab_password'] = SAB_PASSWORD - new_config['SABnzbd']['sab_apikey'] = SAB_APIKEY - new_config['SABnzbd']['sab_category'] = SAB_CATEGORY - - new_config['NZBget'] = {} - new_config['NZBget']['nzbget_username'] = NZBGET_USERNAME - new_config['NZBget']['nzbget_password'] = NZBGET_PASSWORD - new_config['NZBget']['nzbget_category'] = NZBGET_CATEGORY - new_config['NZBget']['nzbget_host'] = NZBGET_HOST - new_config['NZBget']['nzbget_priority'] = NZBGET_PRIORITY - - new_config['Headphones'] = {} - new_config['Headphones']['headphones_indexer'] = int(HEADPHONES_INDEXER) - - new_config['Transmission'] = {} - new_config['Transmission']['transmission_host'] = TRANSMISSION_HOST - new_config['Transmission']['transmission_username'] = TRANSMISSION_USERNAME - new_config['Transmission']['transmission_password'] = TRANSMISSION_PASSWORD - - new_config['uTorrent'] = {} - new_config['uTorrent']['utorrent_host'] = UTORRENT_HOST - new_config['uTorrent']['utorrent_username'] = UTORRENT_USERNAME - new_config['uTorrent']['utorrent_password'] = UTORRENT_PASSWORD - new_config['uTorrent']['utorrent_label'] = UTORRENT_LABEL - - new_config['Newznab'] = {} - new_config['Newznab']['newznab'] = int(NEWZNAB) - new_config['Newznab']['newznab_host'] = NEWZNAB_HOST - new_config['Newznab']['newznab_apikey'] = NEWZNAB_APIKEY - new_config['Newznab']['newznab_enabled'] = int(NEWZNAB_ENABLED) - # Need to unpack the extra newznabs for saving in config.ini - flattened_newznabs = [] - for newznab in EXTRA_NEWZNABS: - for item in newznab: - flattened_newznabs.append(item) - - new_config['Newznab']['extra_newznabs'] = flattened_newznabs - - new_config['NZBsorg'] = {} - new_config['NZBsorg']['nzbsorg'] = int(NZBSORG) - new_config['NZBsorg']['nzbsorg_uid'] = NZBSORG_UID - new_config['NZBsorg']['nzbsorg_hash'] = NZBSORG_HASH - - new_config['omgwtfnzbs'] = {} - new_config['omgwtfnzbs']['omgwtfnzbs'] = int(OMGWTFNZBS) - new_config['omgwtfnzbs']['omgwtfnzbs_uid'] = OMGWTFNZBS_UID - new_config['omgwtfnzbs']['omgwtfnzbs_apikey'] = OMGWTFNZBS_APIKEY - - new_config['General']['preferred_words'] = PREFERRED_WORDS - new_config['General']['ignored_words'] = IGNORED_WORDS - new_config['General']['required_words'] = REQUIRED_WORDS - - new_config['Growl'] = {} - new_config['Growl']['growl_enabled'] = int(GROWL_ENABLED) - new_config['Growl']['growl_host'] = GROWL_HOST - new_config['Growl']['growl_password'] = GROWL_PASSWORD - new_config['Growl']['growl_onsnatch'] = int(GROWL_ONSNATCH) - - new_config['Prowl'] = {} - new_config['Prowl']['prowl_enabled'] = int(PROWL_ENABLED) - new_config['Prowl']['prowl_keys'] = PROWL_KEYS - new_config['Prowl']['prowl_onsnatch'] = int(PROWL_ONSNATCH) - new_config['Prowl']['prowl_priority'] = int(PROWL_PRIORITY) - - new_config['XBMC'] = {} - new_config['XBMC']['xbmc_enabled'] = int(XBMC_ENABLED) - new_config['XBMC']['xbmc_host'] = XBMC_HOST - new_config['XBMC']['xbmc_username'] = XBMC_USERNAME - new_config['XBMC']['xbmc_password'] = XBMC_PASSWORD - new_config['XBMC']['xbmc_update'] = int(XBMC_UPDATE) - new_config['XBMC']['xbmc_notify'] = int(XBMC_NOTIFY) - - new_config['LMS'] = {} - new_config['LMS']['lms_enabled'] = int(LMS_ENABLED) - new_config['LMS']['lms_host'] = LMS_HOST - - new_config['Plex'] = {} - new_config['Plex']['plex_enabled'] = int(PLEX_ENABLED) - new_config['Plex']['plex_server_host'] = PLEX_SERVER_HOST - new_config['Plex']['plex_client_host'] = PLEX_CLIENT_HOST - new_config['Plex']['plex_username'] = PLEX_USERNAME - new_config['Plex']['plex_password'] = PLEX_PASSWORD - new_config['Plex']['plex_update'] = int(PLEX_UPDATE) - new_config['Plex']['plex_notify'] = int(PLEX_NOTIFY) - - new_config['NMA'] = {} - new_config['NMA']['nma_enabled'] = int(NMA_ENABLED) - new_config['NMA']['nma_apikey'] = NMA_APIKEY - new_config['NMA']['nma_priority'] = int(NMA_PRIORITY) - new_config['NMA']['nma_onsnatch'] = int(NMA_ONSNATCH) - - new_config['Pushalot'] = {} - new_config['Pushalot']['pushalot_enabled'] = int(PUSHALOT_ENABLED) - new_config['Pushalot']['pushalot_apikey'] = PUSHALOT_APIKEY - new_config['Pushalot']['pushalot_onsnatch'] = int(PUSHALOT_ONSNATCH) - - new_config['Pushover'] = {} - new_config['Pushover']['pushover_enabled'] = int(PUSHOVER_ENABLED) - new_config['Pushover']['pushover_keys'] = PUSHOVER_KEYS - new_config['Pushover']['pushover_onsnatch'] = int(PUSHOVER_ONSNATCH) - new_config['Pushover']['pushover_priority'] = int(PUSHOVER_PRIORITY) - new_config['Pushover']['pushover_apitoken'] = PUSHOVER_APITOKEN - - new_config['PushBullet'] = {} - new_config['PushBullet']['pushbullet_enabled'] = int(PUSHBULLET_ENABLED) - new_config['PushBullet']['pushbullet_apikey'] = PUSHBULLET_APIKEY - new_config['PushBullet']['pushbullet_deviceid'] = PUSHBULLET_DEVICEID - new_config['PushBullet']['pushbullet_onsnatch'] = int(PUSHBULLET_ONSNATCH) - - new_config['Twitter'] = {} - new_config['Twitter']['twitter_enabled'] = int(TWITTER_ENABLED) - new_config['Twitter']['twitter_onsnatch'] = int(TWITTER_ONSNATCH) - new_config['Twitter']['twitter_username'] = TWITTER_USERNAME - new_config['Twitter']['twitter_password'] = TWITTER_PASSWORD - new_config['Twitter']['twitter_prefix'] = TWITTER_PREFIX - - new_config['OSX_Notify'] = {} - new_config['OSX_Notify']['osx_notify_enabled'] = int(OSX_NOTIFY_ENABLED) - new_config['OSX_Notify']['osx_notify_onsnatch'] = int(OSX_NOTIFY_ONSNATCH) - new_config['OSX_Notify']['osx_notify_app'] = OSX_NOTIFY_APP - - new_config['Boxcar'] = {} - new_config['Boxcar']['boxcar_enabled'] = int(BOXCAR_ENABLED) - new_config['Boxcar']['boxcar_onsnatch'] = int(BOXCAR_ONSNATCH) - new_config['Boxcar']['boxcar_token'] = BOXCAR_TOKEN - - new_config['Subsonic'] = {} - new_config['Subsonic']['subsonic_enabled'] = int(SUBSONIC_ENABLED) - new_config['Subsonic']['subsonic_host'] = SUBSONIC_HOST - new_config['Subsonic']['subsonic_username'] = SUBSONIC_USERNAME - new_config['Subsonic']['subsonic_password'] = SUBSONIC_PASSWORD - - new_config['Songkick'] = {} - new_config['Songkick']['songkick_enabled'] = int(SONGKICK_ENABLED) - new_config['Songkick']['songkick_apikey'] = SONGKICK_APIKEY - new_config['Songkick']['songkick_location'] = SONGKICK_LOCATION - new_config['Songkick']['songkick_filter_enabled'] = int(SONGKICK_FILTER_ENABLED) - - new_config['Synoindex'] = {} - new_config['Synoindex']['synoindex_enabled'] = int(SYNOINDEX_ENABLED) - - new_config['General']['lastfm_username'] = LASTFM_USERNAME - new_config['General']['interface'] = INTERFACE - new_config['General']['folder_permissions'] = FOLDER_PERMISSIONS - new_config['General']['file_permissions'] = FILE_PERMISSIONS - - new_config['General']['music_encoder'] = int(MUSIC_ENCODER) - new_config['General']['encoder'] = ENCODER - new_config['General']['xldprofile'] = XLDPROFILE - new_config['General']['bitrate'] = int(BITRATE) - new_config['General']['samplingfrequency'] = int(SAMPLINGFREQUENCY) - new_config['General']['encoder_path'] = ENCODER_PATH - new_config['General']['advancedencoder'] = ADVANCEDENCODER - new_config['General']['encoderoutputformat'] = ENCODEROUTPUTFORMAT - new_config['General']['encoderquality'] = ENCODERQUALITY - new_config['General']['encodervbrcbr'] = ENCODERVBRCBR - new_config['General']['encoderlossless'] = int(ENCODERLOSSLESS) - new_config['General']['encoder_multicore'] = int(ENCODER_MULTICORE) - new_config['General']['encoder_multicore_count'] = int(ENCODER_MULTICORE_COUNT) - new_config['General']['delete_lossless_files'] = int(DELETE_LOSSLESS_FILES) - - new_config['General']['mirror'] = MIRROR - new_config['General']['customhost'] = CUSTOMHOST - new_config['General']['customport'] = CUSTOMPORT - new_config['General']['customsleep'] = CUSTOMSLEEP - new_config['General']['hpuser'] = HPUSER - new_config['General']['hppass'] = HPPASS - - new_config['Advanced'] = {} - new_config['Advanced']['album_completion_pct'] = ALBUM_COMPLETION_PCT - new_config['Advanced']['cache_sizemb'] = CACHE_SIZEMB - new_config['Advanced']['journal_mode'] = JOURNAL_MODE - new_config['Advanced']['verify_ssl_cert'] = int(VERIFY_SSL_CERT) - - # Write it to file - logger.info("Writing configuration to file") - - try: - new_config.write() - except IOError as e: - logger.error("Error writing configuration file: %s", e) def start(): - global __INITIALIZED__, started + global started - if __INITIALIZED__: + if _INITIALIZED: # Start our scheduled background tasks from headphones import updater, searcher, librarysync, postprocessor, torrentfinished + SCHED.add_job(updater.dbUpdate, trigger=IntervalTrigger( + hours=CONFIG.UPDATE_DB_INTERVAL)) + SCHED.add_job(searcher.searchforalbum, trigger=IntervalTrigger( + minutes=CONFIG.SEARCH_INTERVAL)) + SCHED.add_job(librarysync.libraryScan, trigger=IntervalTrigger( + hours=CONFIG.LIBRARYSCAN_INTERVAL)) - SCHED.add_interval_job(updater.dbUpdate, hours=UPDATE_DB_INTERVAL) - SCHED.add_interval_job(searcher.searchforalbum, minutes=SEARCH_INTERVAL) - SCHED.add_interval_job(librarysync.libraryScan, hours=LIBRARYSCAN_INTERVAL, kwargs={'cron':True}) + if CONFIG.CHECK_GITHUB: + SCHED.add_job(versioncheck.checkGithub, trigger=IntervalTrigger( + minutes=CONFIG.CHECK_GITHUB_INTERVAL)) - if CHECK_GITHUB: - SCHED.add_interval_job(versioncheck.checkGithub, minutes=CHECK_GITHUB_INTERVAL) - - if DOWNLOAD_SCAN_INTERVAL > 0: - SCHED.add_interval_job(postprocessor.checkFolder, minutes=DOWNLOAD_SCAN_INTERVAL) + if CONFIG.DOWNLOAD_SCAN_INTERVAL > 0: + SCHED.add_job(postprocessor.checkFolder, trigger=IntervalTrigger( + minutes=CONFIG.DOWNLOAD_SCAN_INTERVAL)) # Remove Torrent + data if Post Processed and finished Seeding - if TORRENT_REMOVAL_INTERVAL > 0: - SCHED.add_interval_job(torrentfinished.checkTorrentFinished, minutes=TORRENT_REMOVAL_INTERVAL) + if CONFIG.TORRENT_REMOVAL_INTERVAL > 0: + SCHED.add_job(torrentfinished.checkTorrentFinished, trigger=IntervalTrigger( + minutes=CONFIG.TORRENT_REMOVAL_INTERVAL)) SCHED.start() started = True + def sig_handler(signum=None, frame=None): if signum is not None: logger.info("Signal %i caught, saving and exiting...", signum) shutdown() + def dbcheck(): - conn=sqlite3.connect(DB_FILE) - c=conn.cursor() - c.execute('CREATE TABLE IF NOT EXISTS artists (ArtistID TEXT UNIQUE, ArtistName TEXT, ArtistSortName TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, LatestAlbum TEXT, ReleaseDate TEXT, AlbumID TEXT, HaveTracks INTEGER, TotalTracks INTEGER, LastUpdated TEXT, ArtworkURL TEXT, ThumbURL TEXT, Extras TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS albums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, DateAdded TEXT, AlbumID TEXT UNIQUE, Status TEXT, Type TEXT, ArtworkURL TEXT, ThumbURL TEXT, ReleaseID TEXT, ReleaseCountry TEXT, ReleaseFormat TEXT, SearchTerm TEXT)') # ReleaseFormat here means CD,Digital,Vinyl, etc. If using the default Headphones hybrid release, ReleaseID will equal AlbumID (AlbumID is releasegroup id) - c.execute('CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT, Format TEXT, ReleaseID TEXT)') # Format here means mp3, flac, etc. - c.execute('CREATE TABLE IF NOT EXISTS allalbums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, AlbumID TEXT, Type TEXT, ReleaseID TEXT, ReleaseCountry TEXT, ReleaseFormat TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS alltracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT, Format TEXT, ReleaseID TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT, Kind TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS have (ArtistName TEXT, AlbumTitle TEXT, TrackNumber TEXT, TrackTitle TEXT, TrackLength TEXT, BitRate TEXT, Genre TEXT, Date TEXT, TrackID TEXT, Location TEXT, CleanName TEXT, Format TEXT, Matched TEXT)') # Matched is a temporary value used to see if there was a match found in alltracks - c.execute('CREATE TABLE IF NOT EXISTS lastfmcloud (ArtistName TEXT, ArtistID TEXT, Count INTEGER)') - c.execute('CREATE TABLE IF NOT EXISTS descriptions (ArtistID TEXT, ReleaseGroupID TEXT, ReleaseID TEXT, Summary TEXT, Content TEXT, LastUpdated TEXT)') + conn = sqlite3.connect(DB_FILE) + c = conn.cursor() + c.execute( + 'CREATE TABLE IF NOT EXISTS artists (ArtistID TEXT UNIQUE, ArtistName TEXT, ArtistSortName TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, LatestAlbum TEXT, ReleaseDate TEXT, AlbumID TEXT, HaveTracks INTEGER, TotalTracks INTEGER, LastUpdated TEXT, ArtworkURL TEXT, ThumbURL TEXT, Extras TEXT)') + # ReleaseFormat here means CD,Digital,Vinyl, etc. If using the default + # Headphones hybrid release, ReleaseID will equal AlbumID (AlbumID is + # releasegroup id) + c.execute( + 'CREATE TABLE IF NOT EXISTS albums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, DateAdded TEXT, AlbumID TEXT UNIQUE, Status TEXT, Type TEXT, ArtworkURL TEXT, ThumbURL TEXT, ReleaseID TEXT, ReleaseCountry TEXT, ReleaseFormat TEXT, SearchTerm TEXT)') + # Format here means mp3, flac, etc. + c.execute( + 'CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT, Format TEXT, ReleaseID TEXT)') + c.execute( + 'CREATE TABLE IF NOT EXISTS allalbums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, AlbumID TEXT, Type TEXT, ReleaseID TEXT, ReleaseCountry TEXT, ReleaseFormat TEXT)') + c.execute( + 'CREATE TABLE IF NOT EXISTS alltracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT, Format TEXT, ReleaseID TEXT)') + c.execute( + 'CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT, Kind TEXT)') + # Matched is a temporary value used to see if there was a match found in + # alltracks + c.execute( + 'CREATE TABLE IF NOT EXISTS have (ArtistName TEXT, AlbumTitle TEXT, TrackNumber TEXT, TrackTitle TEXT, TrackLength TEXT, BitRate TEXT, Genre TEXT, Date TEXT, TrackID TEXT, Location TEXT, CleanName TEXT, Format TEXT, Matched TEXT)') + c.execute( + 'CREATE TABLE IF NOT EXISTS lastfmcloud (ArtistName TEXT, ArtistID TEXT, Count INTEGER)') + c.execute( + 'CREATE TABLE IF NOT EXISTS descriptions (ArtistID TEXT, ReleaseGroupID TEXT, ReleaseID TEXT, Summary TEXT, Content TEXT, LastUpdated TEXT)') c.execute('CREATE TABLE IF NOT EXISTS blacklist (ArtistID TEXT UNIQUE)') c.execute('CREATE TABLE IF NOT EXISTS newartists (ArtistName TEXT UNIQUE)') - c.execute('CREATE TABLE IF NOT EXISTS releases (ReleaseID TEXT, ReleaseGroupID TEXT, UNIQUE(ReleaseID, ReleaseGroupID))') - c.execute('CREATE INDEX IF NOT EXISTS tracks_albumid ON tracks(AlbumID ASC)') - c.execute('CREATE INDEX IF NOT EXISTS album_artistid_reldate ON albums(ArtistID ASC, ReleaseDate DESC)') - #Below creates indices to speed up Active Artist updating - c.execute('CREATE INDEX IF NOT EXISTS alltracks_relid ON alltracks(ReleaseID ASC, TrackID ASC)') - c.execute('CREATE INDEX IF NOT EXISTS allalbums_relid ON allalbums(ReleaseID ASC)') + c.execute( + 'CREATE TABLE IF NOT EXISTS releases (ReleaseID TEXT, ReleaseGroupID TEXT, UNIQUE(ReleaseID, ReleaseGroupID))') + c.execute( + 'CREATE INDEX IF NOT EXISTS tracks_albumid ON tracks(AlbumID ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS album_artistid_reldate ON albums(ArtistID ASC, ReleaseDate DESC)') + # Below creates indices to speed up Active Artist updating + c.execute( + 'CREATE INDEX IF NOT EXISTS alltracks_relid ON alltracks(ReleaseID ASC, TrackID ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS allalbums_relid ON allalbums(ReleaseID ASC)') c.execute('CREATE INDEX IF NOT EXISTS have_location ON have(Location ASC)') - #Below creates indices to speed up library scanning & matching - c.execute('CREATE INDEX IF NOT EXISTS have_Metadata ON have(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)') - c.execute('CREATE INDEX IF NOT EXISTS have_CleanName ON have(CleanName ASC)') - c.execute('CREATE INDEX IF NOT EXISTS tracks_Metadata ON tracks(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)') - c.execute('CREATE INDEX IF NOT EXISTS tracks_CleanName ON tracks(CleanName ASC)') - c.execute('CREATE INDEX IF NOT EXISTS alltracks_Metadata ON alltracks(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)') - c.execute('CREATE INDEX IF NOT EXISTS alltracks_CleanName ON alltracks(CleanName ASC)') - c.execute('CREATE INDEX IF NOT EXISTS tracks_Location ON tracks(Location ASC)') - c.execute('CREATE INDEX IF NOT EXISTS alltracks_Location ON alltracks(Location ASC)') + # Below creates indices to speed up library scanning & matching + c.execute( + 'CREATE INDEX IF NOT EXISTS have_Metadata ON have(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS have_CleanName ON have(CleanName ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS tracks_Metadata ON tracks(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS tracks_CleanName ON tracks(CleanName ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS alltracks_Metadata ON alltracks(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS alltracks_CleanName ON alltracks(CleanName ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS tracks_Location ON tracks(Location ASC)') + c.execute( + 'CREATE INDEX IF NOT EXISTS alltracks_Location ON alltracks(Location ASC)') try: c.execute('SELECT IncludeExtras from artists') except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN IncludeExtras INTEGER DEFAULT 0') + c.execute( + 'ALTER TABLE artists ADD COLUMN IncludeExtras INTEGER DEFAULT 0') try: c.execute('SELECT LatestAlbum from artists') @@ -1229,12 +367,14 @@ def dbcheck(): try: c.execute('SELECT HaveTracks from artists') except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN HaveTracks INTEGER DEFAULT 0') + c.execute( + 'ALTER TABLE artists ADD COLUMN HaveTracks INTEGER DEFAULT 0') try: c.execute('SELECT TotalTracks from artists') except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN TotalTracks INTEGER DEFAULT 0') + c.execute( + 'ALTER TABLE artists ADD COLUMN TotalTracks INTEGER DEFAULT 0') try: c.execute('SELECT Type from albums') @@ -1290,12 +430,14 @@ def dbcheck(): try: c.execute('SELECT LastUpdated from artists') except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN LastUpdated TEXT DEFAULT NULL') + c.execute( + 'ALTER TABLE artists ADD COLUMN LastUpdated TEXT DEFAULT NULL') try: c.execute('SELECT ArtworkURL from artists') except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN ArtworkURL TEXT DEFAULT NULL') + c.execute( + 'ALTER TABLE artists ADD COLUMN ArtworkURL TEXT DEFAULT NULL') try: c.execute('SELECT ArtworkURL from albums') @@ -1315,12 +457,14 @@ def dbcheck(): try: c.execute('SELECT ArtistID from descriptions') except sqlite3.OperationalError: - c.execute('ALTER TABLE descriptions ADD COLUMN ArtistID TEXT DEFAULT NULL') + c.execute( + 'ALTER TABLE descriptions ADD COLUMN ArtistID TEXT DEFAULT NULL') try: c.execute('SELECT LastUpdated from descriptions') except sqlite3.OperationalError: - c.execute('ALTER TABLE descriptions ADD COLUMN LastUpdated TEXT DEFAULT NULL') + c.execute( + 'ALTER TABLE descriptions ADD COLUMN LastUpdated TEXT DEFAULT NULL') try: c.execute('SELECT ReleaseID from albums') @@ -1330,12 +474,14 @@ def dbcheck(): try: c.execute('SELECT ReleaseFormat from albums') except sqlite3.OperationalError: - c.execute('ALTER TABLE albums ADD COLUMN ReleaseFormat TEXT DEFAULT NULL') + c.execute( + 'ALTER TABLE albums ADD COLUMN ReleaseFormat TEXT DEFAULT NULL') try: c.execute('SELECT ReleaseCountry from albums') except sqlite3.OperationalError: - c.execute('ALTER TABLE albums ADD COLUMN ReleaseCountry TEXT DEFAULT NULL') + c.execute( + 'ALTER TABLE albums ADD COLUMN ReleaseCountry TEXT DEFAULT NULL') try: c.execute('SELECT ReleaseID from tracks') @@ -1351,14 +497,17 @@ def dbcheck(): c.execute('SELECT Extras from artists') except sqlite3.OperationalError: c.execute('ALTER TABLE artists ADD COLUMN Extras TEXT DEFAULT NULL') - # Need to update some stuff when people are upgrading and have 'include extras' set globally/for an artist - if INCLUDE_EXTRAS: - EXTRAS = "1,2,3,4,5,6,7,8" + # Need to update some stuff when people are upgrading and have 'include + # extras' set globally/for an artist + if CONFIG.INCLUDE_EXTRAS: + CONFIG.EXTRAS = "1,2,3,4,5,6,7,8" logger.info("Copying over current artist IncludeExtras information") - artists = c.execute('SELECT ArtistID, IncludeExtras from artists').fetchall() + artists = c.execute( + 'SELECT ArtistID, IncludeExtras from artists').fetchall() for artist in artists: if artist[1]: - c.execute('UPDATE artists SET Extras=? WHERE ArtistID=?', ("1,2,3,4,5,6,7,8", artist[0])) + c.execute( + 'UPDATE artists SET Extras=? WHERE ArtistID=?', ("1,2,3,4,5,6,7,8", artist[0])) try: c.execute('SELECT Kind from snatched') @@ -1370,7 +519,6 @@ def dbcheck(): except sqlite3.OperationalError: c.execute('ALTER TABLE albums ADD COLUMN SearchTerm TEXT DEFAULT NULL') - conn.commit() c.close() @@ -1380,7 +528,7 @@ def shutdown(restart=False, update=False): cherrypy.engine.exit() SCHED.shutdown(wait=False) - config_write() + CONFIG.write() if not restart and not update: logger.info('Headphones is shutting down...') @@ -1389,11 +537,11 @@ def shutdown(restart=False, update=False): logger.info('Headphones is updating...') try: versioncheck.update() - except Exception, e: + except Exception as e: logger.warn('Headphones failed to update: %s. Restarting.', e) - if CREATEPID : - logger.info ('Removing pidfile %s', PIDFILE) + if CREATEPID: + logger.info('Removing pidfile %s', PIDFILE) os.remove(PIDFILE) if restart: diff --git a/headphones/albumart.py b/headphones/albumart.py index daf137b0..8bbbd425 100644 --- a/headphones/albumart.py +++ b/headphones/albumart.py @@ -13,15 +13,18 @@ # You should have received a copy of the GNU General Public License # along with Headphones. If not, see . -from headphones import request, db +from headphones import request, db, logger + def getAlbumArt(albumid): myDB = db.DBConnection() - asin = myDB.action('SELECT AlbumASIN from albums WHERE AlbumID=?', [albumid]).fetchone()[0] + asin = myDB.action( + 'SELECT AlbumASIN from albums WHERE AlbumID=?', [albumid]).fetchone()[0] if asin: return 'http://ec1.images-amazon.com/images/P/%s.01.LZZZZZZZ.jpg' % asin + def getCachedArt(albumid): from headphones import cache diff --git a/headphones/albumswitcher.py b/headphones/albumswitcher.py index 34ab7b99..1edf1937 100644 --- a/headphones/albumswitcher.py +++ b/headphones/albumswitcher.py @@ -14,69 +14,84 @@ # along with Headphones. If not, see . import headphones -from headphones import db, logger +from headphones import db, logger, cache + def switch(AlbumID, ReleaseID): - ''' + """ Takes the contents from allalbums & alltracks (based on ReleaseID) and switches them into the albums & tracks table. - ''' + """ + logger.debug('Switching allalbums and alltracks') myDB = db.DBConnection() - oldalbumdata = myDB.action('SELECT * from albums WHERE AlbumID=?', [AlbumID]).fetchone() - newalbumdata = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [ReleaseID]).fetchone() - newtrackdata = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [ReleaseID]).fetchall() + oldalbumdata = myDB.action( + 'SELECT * from albums WHERE AlbumID=?', [AlbumID]).fetchone() + newalbumdata = myDB.action( + 'SELECT * from allalbums WHERE ReleaseID=?', [ReleaseID]).fetchone() + newtrackdata = myDB.action( + 'SELECT * from alltracks WHERE ReleaseID=?', [ReleaseID]).fetchall() myDB.action('DELETE from tracks WHERE AlbumID=?', [AlbumID]) - controlValueDict = {"AlbumID": AlbumID} + controlValueDict = {"AlbumID": AlbumID} - newValueDict = {"ArtistID": newalbumdata['ArtistID'], - "ArtistName": newalbumdata['ArtistName'], - "AlbumTitle": newalbumdata['AlbumTitle'], - "ReleaseID": newalbumdata['ReleaseID'], - "AlbumASIN": newalbumdata['AlbumASIN'], - "ReleaseDate": newalbumdata['ReleaseDate'], - "Type": newalbumdata['Type'], - "ReleaseCountry": newalbumdata['ReleaseCountry'], - "ReleaseFormat": newalbumdata['ReleaseFormat'] - } + newValueDict = {"ArtistID": newalbumdata['ArtistID'], + "ArtistName": newalbumdata['ArtistName'], + "AlbumTitle": newalbumdata['AlbumTitle'], + "ReleaseID": newalbumdata['ReleaseID'], + "AlbumASIN": newalbumdata['AlbumASIN'], + "ReleaseDate": newalbumdata['ReleaseDate'], + "Type": newalbumdata['Type'], + "ReleaseCountry": newalbumdata['ReleaseCountry'], + "ReleaseFormat": newalbumdata['ReleaseFormat'] + } myDB.upsert("albums", newValueDict, controlValueDict) + # Update cache + c = cache.Cache() + c.remove_from_cache(AlbumID=AlbumID) + c.get_artwork_from_cache(AlbumID=AlbumID) + for track in newtrackdata: - controlValueDict = {"TrackID": track['TrackID'], - "AlbumID": AlbumID} + controlValueDict = {"TrackID": track['TrackID'], + "AlbumID": AlbumID} - newValueDict = {"ArtistID": track['ArtistID'], - "ArtistName": track['ArtistName'], - "AlbumTitle": track['AlbumTitle'], - "AlbumASIN": track['AlbumASIN'], - "ReleaseID": track['ReleaseID'], - "TrackTitle": track['TrackTitle'], - "TrackDuration": track['TrackDuration'], - "TrackNumber": track['TrackNumber'], - "CleanName": track['CleanName'], - "Location": track['Location'], - "Format": track['Format'], - "BitRate": track['BitRate'] - } + newValueDict = {"ArtistID": track['ArtistID'], + "ArtistName": track['ArtistName'], + "AlbumTitle": track['AlbumTitle'], + "AlbumASIN": track['AlbumASIN'], + "ReleaseID": track['ReleaseID'], + "TrackTitle": track['TrackTitle'], + "TrackDuration": track['TrackDuration'], + "TrackNumber": track['TrackNumber'], + "CleanName": track['CleanName'], + "Location": track['Location'], + "Format": track['Format'], + "BitRate": track['BitRate'] + } myDB.upsert("tracks", newValueDict, controlValueDict) - # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album + # Mark albums as downloaded if they have at least 80% (by default, + # configurable) of the album total_track_count = len(newtrackdata) - have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [AlbumID])) + have_track_count = len(myDB.select( + 'SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [AlbumID])) - if oldalbumdata['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): - myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', AlbumID]) + if oldalbumdata['Status'] == 'Skipped' and ((have_track_count / float(total_track_count)) >= (headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)): + myDB.action( + 'UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', AlbumID]) # Update have track counts on index - totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND AlbumID IN (SELECT AlbumID FROM albums WHERE Status != "Ignored")', [newalbumdata['ArtistID']])) - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [newalbumdata['ArtistID']])) + totaltracks = len(myDB.select( + 'SELECT TrackTitle from tracks WHERE ArtistID=? AND AlbumID IN (SELECT AlbumID FROM albums WHERE Status != "Ignored")', [newalbumdata['ArtistID']])) + havetracks = len(myDB.select( + 'SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [newalbumdata['ArtistID']])) - controlValueDict = {"ArtistID": newalbumdata['ArtistID']} + controlValueDict = {"ArtistID": newalbumdata['ArtistID']} - newValueDict = { "TotalTracks": totaltracks, - "HaveTracks": havetracks} + newValueDict = {"TotalTracks": totaltracks, + "HaveTracks": havetracks} myDB.upsert("artists", newValueDict, controlValueDict) diff --git a/headphones/api.py b/headphones/api.py index d101b7e2..cb429ea6 100644 --- a/headphones/api.py +++ b/headphones/api.py @@ -15,18 +15,16 @@ from headphones import db, mb, importer, searcher, cache, postprocessor, versioncheck, logger -from xml.dom.minidom import Document - import headphones -import copy import json -cmd_list = [ 'getIndex', 'getArtist', 'getAlbum', 'getUpcoming', 'getWanted', 'getSimilar', 'getHistory', 'getLogs', +cmd_list = ['getIndex', 'getArtist', 'getAlbum', 'getUpcoming', 'getWanted', 'getSimilar', 'getHistory', 'getLogs', 'findArtist', 'findAlbum', 'addArtist', 'delArtist', 'pauseArtist', 'resumeArtist', 'refreshArtist', 'addAlbum', 'queueAlbum', 'unqueueAlbum', 'forceSearch', 'forceProcess', 'getVersion', 'checkGithub', 'shutdown', 'restart', 'update', 'getArtistArt', 'getAlbumArt', 'getArtistInfo', 'getAlbumInfo', 'getArtistThumb', 'getAlbumThumb', 'choose_specific_download', 'download_specific_release'] + class Api(object): def __init__(self): @@ -41,16 +39,15 @@ class Api(object): self.callback = None + def checkParams(self, *args, **kwargs): - def checkParams(self,*args,**kwargs): - - if not headphones.API_ENABLED: + if not headphones.CONFIG.API_ENABLED: self.data = 'API not enabled' return - if not headphones.API_KEY: + if not headphones.CONFIG.API_KEY: self.data = 'API key not generated' return - if len(headphones.API_KEY) != 32: + if len(headphones.CONFIG.API_KEY) != 32: self.data = 'API key not generated correctly' return @@ -58,7 +55,7 @@ class Api(object): self.data = 'Missing api key' return - if kwargs['apikey'] != headphones.API_KEY: + if kwargs['apikey'] != headphones.CONFIG.API_KEY: self.data = 'Incorrect API key' return else: @@ -82,9 +79,9 @@ class Api(object): if self.data == 'OK': logger.info('Recieved API command: %s', self.cmd) methodToCall = getattr(self, "_" + self.cmd) - result = methodToCall(**self.kwargs) + methodToCall(**self.kwargs) if 'callback' not in self.kwargs: - if type(self.data) == type(''): + if isinstance(self.data, basestring): return self.data else: return json.dumps(self.data) @@ -96,7 +93,7 @@ class Api(object): else: return self.data - def _dic_from_query(self,query): + def _dic_from_query(self, query): myDB = db.DBConnection() rows = myDB.select(query) @@ -111,7 +108,8 @@ class Api(object): def _getIndex(self, **kwargs): - self.data = self._dic_from_query('SELECT * from artists order by ArtistSortName COLLATE NOCASE') + self.data = self._dic_from_query( + 'SELECT * from artists order by ArtistSortName COLLATE NOCASE') return def _getArtist(self, **kwargs): @@ -122,11 +120,15 @@ class Api(object): else: self.id = kwargs['id'] - artist = self._dic_from_query('SELECT * from artists WHERE ArtistID="' + self.id + '"') - albums = self._dic_from_query('SELECT * from albums WHERE ArtistID="' + self.id + '" order by ReleaseDate DESC') - description = self._dic_from_query('SELECT * from descriptions WHERE ArtistID="' + self.id + '"') + artist = self._dic_from_query( + 'SELECT * from artists WHERE ArtistID="' + self.id + '"') + albums = self._dic_from_query( + 'SELECT * from albums WHERE ArtistID="' + self.id + '" order by ReleaseDate DESC') + description = self._dic_from_query( + 'SELECT * from descriptions WHERE ArtistID="' + self.id + '"') - self.data = { 'artist': artist, 'albums': albums, 'description' : description } + self.data = { + 'artist': artist, 'albums': albums, 'description': description} return def _getAlbum(self, **kwargs): @@ -137,23 +139,30 @@ class Api(object): else: self.id = kwargs['id'] - album = self._dic_from_query('SELECT * from albums WHERE AlbumID="' + self.id + '"') - tracks = self._dic_from_query('SELECT * from tracks WHERE AlbumID="' + self.id + '"') - description = self._dic_from_query('SELECT * from descriptions WHERE ReleaseGroupID="' + self.id + '"') + album = self._dic_from_query( + 'SELECT * from albums WHERE AlbumID="' + self.id + '"') + tracks = self._dic_from_query( + 'SELECT * from tracks WHERE AlbumID="' + self.id + '"') + description = self._dic_from_query( + 'SELECT * from descriptions WHERE ReleaseGroupID="' + self.id + '"') - self.data = { 'album' : album, 'tracks' : tracks, 'description' : description } + self.data = { + 'album': album, 'tracks': tracks, 'description': description} return def _getHistory(self, **kwargs): - self.data = self._dic_from_query('SELECT * from snatched WHERE status NOT LIKE "Seed%" order by DateAdded DESC') + self.data = self._dic_from_query( + 'SELECT * from snatched WHERE status NOT LIKE "Seed%" order by DateAdded DESC') return def _getUpcoming(self, **kwargs): - self.data = self._dic_from_query("SELECT * from albums WHERE ReleaseDate > date('now') order by ReleaseDate DESC") + self.data = self._dic_from_query( + "SELECT * from albums WHERE ReleaseDate > date('now') order by ReleaseDate DESC") return def _getWanted(self, **kwargs): - self.data = self._dic_from_query("SELECT * from albums WHERE Status='Wanted'") + self.data = self._dic_from_query( + "SELECT * from albums WHERE Status='Wanted'") return def _getSimilar(self, **kwargs): @@ -170,7 +179,7 @@ class Api(object): if 'limit' in kwargs: limit = kwargs['limit'] else: - limit=50 + limit = 50 self.data = mb.findArtist(kwargs['name'], limit) @@ -181,7 +190,7 @@ class Api(object): if 'limit' in kwargs: limit = kwargs['limit'] else: - limit=50 + limit = 50 self.data = mb.findRelease(kwargs['name'], limit) @@ -194,7 +203,7 @@ class Api(object): try: importer.addArtisttoDB(self.id) - except Exception, e: + except Exception as e: self.data = e return @@ -244,7 +253,7 @@ class Api(object): try: importer.addArtisttoDB(self.id) - except Exception, e: + except Exception as e: self.data = e return @@ -258,7 +267,7 @@ class Api(object): try: importer.addReleaseById(self.id) - except Exception, e: + except Exception as e: self.data = e return @@ -314,11 +323,11 @@ class Api(object): def _getVersion(self, **kwargs): self.data = { - 'git_path' : headphones.GIT_PATH, - 'install_type' : headphones.INSTALL_TYPE, - 'current_version' : headphones.CURRENT_VERSION, - 'latest_version' : headphones.LATEST_VERSION, - 'commits_behind' : headphones.COMMITS_BEHIND, + 'git_path': headphones.CONFIG.GIT_PATH, + 'install_type': headphones.INSTALL_TYPE, + 'current_version': headphones.CURRENT_VERSION, + 'latest_version': headphones.LATEST_VERSION, + 'commits_behind': headphones.COMMITS_BEHIND, } def _checkGithub(self, **kwargs): @@ -402,18 +411,19 @@ class Api(object): else: self.id = kwargs['id'] - results = searcher.searchforalbum(self.id, choose_specific_download=True) + results = searcher.searchforalbum( + self.id, choose_specific_download=True) results_as_dicts = [] for result in results: result_dict = { - 'title':result[0], - 'size':result[1], - 'url':result[2], - 'provider':result[3], - 'kind':result[4] + 'title': result[0], + 'size': result[1], + 'url': result[2], + 'provider': result[3], + 'kind': result[4] } results_as_dicts.append(result_dict) @@ -421,7 +431,7 @@ class Api(object): def _download_specific_release(self, **kwargs): - expected_kwargs =['id', 'title','size','url','provider','kind'] + expected_kwargs = ['id', 'title', 'size', 'url', 'provider', 'kind'] for kwarg in expected_kwargs: if kwarg not in kwargs: @@ -438,20 +448,24 @@ class Api(object): for kwarg in expected_kwargs: del kwargs[kwarg] - # Handle situations where the torrent url contains arguments that are parsed + # Handle situations where the torrent url contains arguments that are + # parsed if kwargs: - import urllib, urllib2 - url = urllib2.quote(url, safe=":?/=&") + '&' + urllib.urlencode(kwargs) + import urllib + import urllib2 + url = urllib2.quote( + url, safe=":?/=&") + '&' + urllib.urlencode(kwargs) try: - result = [(title,int(size),url,provider,kind)] + result = [(title, int(size), url, provider, kind)] except ValueError: - result = [(title,float(size),url,provider,kind)] + result = [(title, float(size), url, provider, kind)] logger.info(u"Making sure we can download the chosen result") (data, bestqual) = searcher.preprocess(result) if data and bestqual: - myDB = db.DBConnection() - album = myDB.action('SELECT * from albums WHERE AlbumID=?', [id]).fetchone() - searcher.send_to_downloader(data, bestqual, album) + myDB = db.DBConnection() + album = myDB.action( + 'SELECT * from albums WHERE AlbumID=?', [id]).fetchone() + searcher.send_to_downloader(data, bestqual, album) diff --git a/headphones/cache.py b/headphones/cache.py index 70d851e2..615b3e2c 100644 --- a/headphones/cache.py +++ b/headphones/cache.py @@ -14,14 +14,13 @@ # along with Headphones. If not, see . import os -import glob -import urllib import headphones from headphones import db, helpers, logger, lastfm, request LASTFM_API_KEY = "690e1ed3bc00bc91804cd8f7fe5ed6d4" + class Cache(object): """ This class deals with getting, storing and serving up artwork (album @@ -40,7 +39,7 @@ class Cache(object): and for info it is ..txt """ - path_to_art_cache = os.path.join(headphones.CACHE_DIR, 'artwork') + path_to_art_cache = os.path.join(headphones.CONFIG.CACHE_DIR, 'artwork') def __init__(self): self.id = None @@ -59,12 +58,12 @@ class Cache(object): self.info_summary = None self.info_content = None - def _findfilesstartingwith(self,pattern,folder): + def _findfilesstartingwith(self, pattern, folder): files = [] if os.path.exists(folder): for fname in os.listdir(folder): if fname.startswith(pattern): - files.append(os.path.join(folder,fname)) + files.append(os.path.join(folder, fname)) return files def _exists(self, type): @@ -72,14 +71,14 @@ class Cache(object): self.thumb_files = [] if type == 'artwork': - self.artwork_files = self._findfilesstartingwith(self.id,self.path_to_art_cache) + self.artwork_files = self._findfilesstartingwith(self.id, self.path_to_art_cache) if self.artwork_files: return True else: return False elif type == 'thumb': - self.thumb_files = self._findfilesstartingwith("T_" + self.id,self.path_to_art_cache) + self.thumb_files = self._findfilesstartingwith("T_" + self.id, self.path_to_art_cache) if self.thumb_files: return True else: @@ -88,11 +87,10 @@ class Cache(object): def _get_age(self, date): # There's probably a better way to do this split_date = date.split('-') - days_old = int(split_date[0])*365 + int(split_date[1])*30 + int(split_date[2]) + days_old = int(split_date[0]) * 365 + int(split_date[1]) * 30 + int(split_date[2]) return days_old - def _is_current(self, filename=None, date=None): if filename: @@ -191,11 +189,11 @@ class Cache(object): if not db_info or not db_info['LastUpdated'] or not self._is_current(date=db_info['LastUpdated']): self._update_cache() - info_dict = { 'Summary' : self.info_summary, 'Content' : self.info_content } + info_dict = {'Summary': self.info_summary, 'Content': self.info_content} return info_dict else: - info_dict = { 'Summary' : db_info['Summary'], 'Content' : db_info['Content'] } + info_dict = {'Summary': db_info['Summary'], 'Content': db_info['Content']} return info_dict def get_image_links(self, ArtistID=None, AlbumID=None): @@ -240,7 +238,37 @@ class Cache(object): if not thumb_url: logger.debug('No album thumbnail image found on last.fm') - return {'artwork' : image_url, 'thumbnail' : thumb_url } + return {'artwork': image_url, 'thumbnail': thumb_url} + + def remove_from_cache(self, ArtistID=None, AlbumID=None): + """ + Pass a musicbrainz id to this function (either ArtistID or AlbumID) + """ + + if ArtistID: + self.id = ArtistID + self.id_type = 'artist' + else: + self.id = AlbumID + self.id_type = 'album' + + self.query_type = 'artwork' + + if self._exists('artwork'): + for artwork_file in self.artwork_files: + try: + os.remove(artwork_file) + except: + logger.warn('Error deleting file from the cache: %s', artwork_file) + + self.query_type = 'thumb' + + if self._exists('thumb'): + for thumb_file in self.thumb_files: + try: + os.remove(thumb_file) + except Exception: + logger.warn('Error deleting file from the cache: %s', thumb_file) def _update_cache(self): ''' @@ -249,6 +277,7 @@ class Cache(object): myDB = db.DBConnection() # Since lastfm uses release ids rather than release group ids for albums, we have to do a artist + album search for albums + # Exception is when adding albums manually, then we should use release id if self.id_type == 'artist': data = lastfm.request_lastfm("artist.getinfo", mbid=self.id, api_key=LASTFM_API_KEY) @@ -278,8 +307,13 @@ class Cache(object): else: - dbartist = myDB.action('SELECT ArtistName, AlbumTitle FROM albums WHERE AlbumID=?', [self.id]).fetchone() - data = lastfm.request_lastfm("album.getinfo", artist=dbartist['ArtistName'], album=dbartist['AlbumTitle'], api_key=LASTFM_API_KEY) + dbalbum = myDB.action('SELECT ArtistName, AlbumTitle, ReleaseID FROM albums WHERE AlbumID=?', [self.id]).fetchone() + if dbalbum['ReleaseID'] != self.id: + data = lastfm.request_lastfm("album.getinfo", mbid=dbalbum['ReleaseID'], api_key=LASTFM_API_KEY) + if not data: + data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'], album=dbalbum['AlbumTitle'], api_key=LASTFM_API_KEY) + else: + data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'], album=dbalbum['AlbumTitle'], api_key=LASTFM_API_KEY) if not data: return @@ -307,13 +341,13 @@ class Cache(object): #Save the content & summary to the database no matter what if we've opened up the url if self.id_type == 'artist': - controlValueDict = {"ArtistID": self.id} + controlValueDict = {"ArtistID": self.id} else: - controlValueDict = {"ReleaseGroupID": self.id} + controlValueDict = {"ReleaseGroupID": self.id} - newValueDict = {"Summary": self.info_summary, - "Content": self.info_content, - "LastUpdated": helpers.today()} + newValueDict = {"Summary": self.info_summary, + "Content": self.info_content, + "LastUpdated": helpers.today()} myDB.upsert("descriptions", newValueDict, controlValueDict) @@ -340,7 +374,7 @@ class Cache(object): if not os.path.isdir(self.path_to_art_cache): try: os.makedirs(self.path_to_art_cache) - except Exception, e: + except Exception as e: logger.error('Unable to create artwork cache dir. Error: %s', e) self.artwork_errors = True self.artwork_url = image_url @@ -364,7 +398,7 @@ class Cache(object): self.artwork_url = image_url # Grab the thumbnail as well if we're getting the full artwork (as long as it's missing/outdated - if thumb_url and self.query_type in ['thumb','artwork'] and not (self.thumb_files and self._is_current(self.thumb_files[0])): + if thumb_url and self.query_type in ['thumb', 'artwork'] and not (self.thumb_files and self._is_current(self.thumb_files[0])): artwork = request.request_content(thumb_url, timeout=20) if artwork: @@ -395,6 +429,7 @@ class Cache(object): self.thumb_errors = True self.thumb_url = image_url + def getArtwork(ArtistID=None, AlbumID=None): c = Cache() @@ -409,6 +444,7 @@ def getArtwork(ArtistID=None, AlbumID=None): artwork_file = os.path.basename(artwork_path) return "cache/artwork/" + artwork_file + def getThumb(ArtistID=None, AlbumID=None): c = Cache() @@ -423,6 +459,7 @@ def getThumb(ArtistID=None, AlbumID=None): thumbnail_file = os.path.basename(artwork_path) return "cache/artwork/" + thumbnail_file + def getInfo(ArtistID=None, AlbumID=None): c = Cache() @@ -431,6 +468,7 @@ def getInfo(ArtistID=None, AlbumID=None): return info_dict + def getImageLinks(ArtistID=None, AlbumID=None): c = Cache() diff --git a/headphones/classes.py b/headphones/classes.py index c6b14055..96315ba7 100644 --- a/headphones/classes.py +++ b/headphones/classes.py @@ -17,31 +17,32 @@ ## Stolen from Sick-Beard's classes.py ## ######################################### -import headphones import urllib -import datetime from common import USER_AGENT + class HeadphonesURLopener(urllib.FancyURLopener): version = USER_AGENT + class AuthURLOpener(HeadphonesURLopener): """ URLOpener class that supports http auth without needing interactive password entry. If the provided username/password don't work it simply fails. - + user: username to use for HTTP auth pw: password to use for HTTP auth """ + def __init__(self, user, pw): self.username = user self.password = pw # remember if we've tried the username/password before self.numTries = 0 - + # call the base class urllib.FancyURLopener.__init__(self) @@ -55,7 +56,7 @@ class AuthURLOpener(HeadphonesURLopener): if self.numTries == 0: self.numTries = 1 return (self.username, self.password) - + # if we've tried before then return blank which cancels the request else: return ('', '') @@ -65,6 +66,7 @@ class AuthURLOpener(HeadphonesURLopener): self.numTries = 0 return HeadphonesURLopener.open(self, url) + class SearchResult: """ Represents a search result from an indexer. @@ -87,7 +89,7 @@ class SearchResult: def __str__(self): - if self.provider == None: + if self.provider is None: return "Invalid provider, unable to print self" myString = self.provider.name + " @ " + self.url + "\n" @@ -96,24 +98,28 @@ class SearchResult: myString += " " + extra + "\n" return myString + class NZBSearchResult(SearchResult): """ Regular NZB result with an URL to the NZB """ resultType = "nzb" + class NZBDataSearchResult(SearchResult): """ NZB result where the actual NZB XML data is stored in the extraInfo """ resultType = "nzbdata" + class TorrentSearchResult(SearchResult): """ Torrent result with an URL to the torrent """ resultType = "torrent" + class Proper: def __init__(self, name, url, date): self.name = name @@ -127,4 +133,4 @@ class Proper: self.episode = -1 def __str__(self): - return str(self.date)+" "+self.name+" "+str(self.season)+"x"+str(self.episode)+" of "+str(self.tvdbid) + return str(self.date) + " " + self.name + " " + str(self.season) + "x" + str(self.episode) + " of " + str(self.tvdbid) diff --git a/headphones/common.py b/headphones/common.py index 74aa1db8..02bfc3a2 100644 --- a/headphones/common.py +++ b/headphones/common.py @@ -18,12 +18,15 @@ Created on Aug 1, 2011 @author: Michael ''' -import platform, operator, os, re +import platform +import operator +import os +import re from headphones import version #Identify Our Application -USER_AGENT = 'Headphones/-'+version.HEADPHONES_VERSION+' ('+platform.system()+' '+platform.release()+')' +USER_AGENT = 'Headphones/-' + version.HEADPHONES_VERSION + ' (' + platform.system() + ' ' + platform.release() + ')' ### Notification Types NOTIFY_SNATCH = 1 @@ -44,17 +47,18 @@ ARCHIVED = 6 # releases that you don't have locally (counts toward download comp IGNORED = 7 # releases that you don't want included in your download stats SNATCHED_PROPER = 9 # qualified with quality + class Quality: NONE = 0 - B192 = 1<<1 # 2 - VBR = 1<<2 # 4 - B256 = 1<<3 # 8 - B320 = 1<<4 #16 - FLAC = 1<<5 #32 + B192 = 1 << 1 # 2 + VBR = 1 << 2 # 4 + B256 = 1 << 3 # 8 + B320 = 1 << 4 #16 + FLAC = 1 << 5 #32 # put these bits at the other end of the spectrum, far enough out that they shouldn't interfere - UNKNOWN = 1<<15 + UNKNOWN = 1 << 15 qualityStrings = {NONE: "N/A", UNKNOWN: "Unknown", @@ -71,7 +75,7 @@ class Quality: def _getStatusStrings(status): toReturn = {} for x in Quality.qualityStrings.keys(): - toReturn[Quality.compositeStatus(status, x)] = Quality.statusPrefixes[status]+" ("+Quality.qualityStrings[x]+")" + toReturn[Quality.compositeStatus(status, x)] = Quality.statusPrefixes[status] + " (" + Quality.qualityStrings[x] + ")" return toReturn @staticmethod @@ -82,7 +86,7 @@ class Quality: anyQuality = reduce(operator.or_, anyQualities) if bestQualities: bestQuality = reduce(operator.or_, bestQualities) - return anyQuality | (bestQuality<<16) + return anyQuality | (bestQuality << 16) @staticmethod def splitQuality(quality): @@ -91,7 +95,7 @@ class Quality: for curQual in Quality.qualityStrings.keys(): if curQual & quality: anyQualities.append(curQual) - if curQual<<16 & quality: + if curQual << 16 & quality: bestQualities.append(curQual) return (anyQualities, bestQualities) @@ -106,7 +110,7 @@ class Quality: if x == Quality.UNKNOWN: continue - regex = '\W'+Quality.qualityStrings[x].replace(' ','\W')+'\W' + regex = '\W' + Quality.qualityStrings[x].replace(' ', '\W') + '\W' regex_match = re.search(regex, name, re.I) if regex_match: return x @@ -147,8 +151,8 @@ class Quality: def splitCompositeStatus(status): """Returns a tuple containing (status, quality)""" for x in sorted(Quality.qualityStrings.keys(), reverse=True): - if status > x*100: - return (status-x*100, x) + if status > x * 100: + return (status - x * 100, x) return (Quality.NONE, status) diff --git a/headphones/config.py b/headphones/config.py new file mode 100644 index 00000000..7cae3861 --- /dev/null +++ b/headphones/config.py @@ -0,0 +1,434 @@ +import headphones.logger +import itertools +import os +import re +from configobj import ConfigObj + + +def bool_int(value): + """ + Casts a config value into a 0 or 1 + """ + if isinstance(value, basestring): + if value.lower() in ('', '0', 'false', 'f', 'no', 'n', 'off'): + value = 0 + return int(bool(value)) + +_CONFIG_DEFINITIONS = { + 'ADD_ALBUM_ART': (int, 'General', 0), + 'ADVANCEDENCODER': (str, 'General', ''), + 'ALBUM_ART_FORMAT': (str, 'General', 'folder'), + # This is used in importer.py to determine how complete an album needs to + # be - to be considered "downloaded". Percentage from 0-100 + 'ALBUM_COMPLETION_PCT': (int, 'Advanced', 80), + 'API_ENABLED': (int, 'General', 0), + 'API_KEY': (str, 'General', ''), + 'AUTOWANT_ALL': (int, 'General', 0), + 'AUTOWANT_MANUALLY_ADDED': (int, 'General', 1), + 'AUTOWANT_UPCOMING': (int, 'General', 1), + 'AUTO_ADD_ARTISTS': (int, 'General', 1), + 'BITRATE': (int, 'General', 192), + 'BLACKHOLE': (int, 'General', 0), + 'BLACKHOLE_DIR': (str, 'General', ''), + 'BOXCAR_ENABLED': (int, 'Boxcar', 0), + 'BOXCAR_ONSNATCH': (int, 'Boxcar', 0), + 'BOXCAR_TOKEN': (str, 'Boxcar', ''), + 'CACHE_DIR': (str, 'General', ''), + 'CACHE_SIZEMB': (int, 'Advanced', 32), + 'CHECK_GITHUB': (int, 'General', 1), + 'CHECK_GITHUB_INTERVAL': (int, 'General', 360), + 'CHECK_GITHUB_ON_STARTUP': (int, 'General', 1), + 'CLEANUP_FILES': (int, 'General', 0), + 'CONFIG_VERSION': (str, 'General', '0'), + 'CORRECT_METADATA': (int, 'General', 0), + 'CUE_SPLIT': (int, 'General', 1), + 'CUSTOMHOST': (str, 'General', 'localhost'), + 'CUSTOMPORT': (int, 'General', 5000), + 'CUSTOMSLEEP': (int, 'General', 1), + 'DELETE_LOSSLESS_FILES': (int, 'General', 1), + 'DESTINATION_DIR': (str, 'General', ''), + 'DETECT_BITRATE': (int, 'General', 0), + 'DOWNLOAD_DIR': (str, 'General', ''), + 'DOWNLOAD_SCAN_INTERVAL': (int, 'General', 5), + 'DOWNLOAD_TORRENT_DIR': (str, 'General', ''), + 'DO_NOT_OVERRIDE_GIT_BRANCH': (int, 'General', 0), + 'EMBED_ALBUM_ART': (int, 'General', 0), + 'EMBED_LYRICS': (int, 'General', 0), + 'ENABLE_HTTPS': (int, 'General', 0), + 'ENCODER': (str, 'General', 'ffmpeg'), + 'ENCODERFOLDER': (str, 'General', ''), + 'ENCODERLOSSLESS': (int, 'General', 1), + 'ENCODEROUTPUTFORMAT': (str, 'General', 'mp3'), + 'ENCODERQUALITY': (int, 'General', 2), + 'ENCODERVBRCBR': (str, 'General', 'cbr'), + 'ENCODER_MULTICORE': (int, 'General', 0), + 'ENCODER_MULTICORE_COUNT': (int, 'General', 0), + 'ENCODER_PATH': (str, 'General', ''), + 'EXTRAS': (str, 'General', ''), + 'EXTRA_NEWZNABS': (list, 'Newznab', ''), + 'FILE_FORMAT': (str, 'General', 'Track Artist - Album [Year] - Title'), + 'FILE_PERMISSIONS': (str, 'General', '0644'), + 'FILE_UNDERSCORES': (int, 'General', 0), + 'FOLDER_FORMAT': (str, 'General', 'Artist/Album [Year]'), + 'FOLDER_PERMISSIONS': (str, 'General', '0755'), + 'FREEZE_DB': (int, 'General', 0), + 'GIT_BRANCH': (str, 'General', 'master'), + 'GIT_PATH': (str, 'General', ''), + 'GIT_USER': (str, 'General', 'rembo10'), + 'GROWL_ENABLED': (int, 'Growl', 0), + 'GROWL_HOST': (str, 'Growl', ''), + 'GROWL_ONSNATCH': (int, 'Growl', 0), + 'GROWL_PASSWORD': (str, 'Growl', ''), + 'HEADPHONES_INDEXER': (bool_int, 'General', False), + 'HPPASS': (str, 'General', ''), + 'HPUSER': (str, 'General', ''), + 'HTTPS_CERT': (str, 'General', ''), + 'HTTPS_KEY': (str, 'General', ''), + 'HTTP_HOST': (str, 'General', '0.0.0.0'), + 'HTTP_PASSWORD': (str, 'General', ''), + 'HTTP_PORT': (int, 'General', 8181), + 'HTTP_PROXY': (int, 'General', 0), + 'HTTP_ROOT': (str, 'General', '/'), + 'HTTP_USERNAME': (str, 'General', ''), + 'IGNORED_WORDS': (str, 'General', ''), + 'INCLUDE_EXTRAS': (int, 'General', 0), + 'INTERFACE': (str, 'General', 'default'), + 'JOURNAL_MODE': (str, 'Advanced', 'wal'), + 'KAT': (int, 'Kat', 0), + 'KAT_PROXY_URL': (str, 'Kat', ''), + 'KAT_RATIO': (str, 'Kat', ''), + 'KEEP_NFO': (int, 'General', 0), + 'KEEP_TORRENT_FILES': (int, 'General', 0), + 'LASTFM_USERNAME': (str, 'General', ''), + 'LAUNCH_BROWSER': (int, 'General', 1), + 'LIBRARYSCAN': (int, 'General', 1), + 'LIBRARYSCAN_INTERVAL': (int, 'General', 300), + 'LMS_ENABLED': (int, 'LMS', 0), + 'LMS_HOST': (str, 'LMS', ''), + 'LOG_DIR': (str, 'General', ''), + 'LOSSLESS_BITRATE_FROM': (int, 'General', 0), + 'LOSSLESS_BITRATE_TO': (int, 'General', 0), + 'LOSSLESS_DESTINATION_DIR': (str, 'General', ''), + 'MB_IGNORE_AGE': (int, 'General', 365), + 'MININOVA': (int, 'Mininova', 0), + 'MININOVA_RATIO': (str, 'Mininova', ''), + 'MIRROR': (str, 'General', 'musicbrainz.org'), + 'MOVE_FILES': (int, 'General', 0), + 'MPC_ENABLED': (bool_int, 'MPC', False), + 'MUSIC_DIR': (str, 'General', ''), + 'MUSIC_ENCODER': (int, 'General', 0), + 'NEWZNAB': (int, 'Newznab', 0), + 'NEWZNAB_APIKEY': (str, 'Newznab', ''), + 'NEWZNAB_ENABLED': (int, 'Newznab', 1), + 'NEWZNAB_HOST': (str, 'Newznab', ''), + 'NMA_APIKEY': (str, 'NMA', ''), + 'NMA_ENABLED': (int, 'NMA', 0), + 'NMA_ONSNATCH': (int, 'NMA', 0), + 'NMA_PRIORITY': (int, 'NMA', 0), + 'NUMBEROFSEEDERS': (str, 'General', '10'), + 'NZBGET_CATEGORY': (str, 'NZBget', ''), + 'NZBGET_HOST': (str, 'NZBget', ''), + 'NZBGET_PASSWORD': (str, 'NZBget', ''), + 'NZBGET_PRIORITY': (int, 'NZBget', 0), + 'NZBGET_USERNAME': (str, 'NZBget', 'nzbget'), + 'NZBSORG': (int, 'NZBsorg', 0), + 'NZBSORG_HASH': (str, 'NZBsorg', ''), + 'NZBSORG_UID': (str, 'NZBsorg', ''), + 'NZB_DOWNLOADER': (int, 'General', 0), + 'OMGWTFNZBS': (int, 'omgwtfnzbs', 0), + 'OMGWTFNZBS_APIKEY': (str, 'omgwtfnzbs', ''), + 'OMGWTFNZBS_UID': (str, 'omgwtfnzbs', ''), + 'OPEN_MAGNET_LINKS': (int, 'General', 0), # 0: Ignore, 1: Open, 2: Convert + 'MAGNET_LINKS': (int, 'General', 0), + 'OSX_NOTIFY_APP': (str, 'OSX_Notify', '/Applications/Headphones'), + 'OSX_NOTIFY_ENABLED': (int, 'OSX_Notify', 0), + 'OSX_NOTIFY_ONSNATCH': (int, 'OSX_Notify', 0), + 'PIRATEBAY': (int, 'Piratebay', 0), + 'PIRATEBAY_PROXY_URL': (str, 'Piratebay', ''), + 'PIRATEBAY_RATIO': (str, 'Piratebay', ''), + 'PLEX_CLIENT_HOST': (str, 'Plex', ''), + 'PLEX_ENABLED': (int, 'Plex', 0), + 'PLEX_NOTIFY': (int, 'Plex', 0), + 'PLEX_PASSWORD': (str, 'Plex', ''), + 'PLEX_SERVER_HOST': (str, 'Plex', ''), + 'PLEX_UPDATE': (int, 'Plex', 0), + 'PLEX_USERNAME': (str, 'Plex', ''), + 'PREFERRED_BITRATE': (str, 'General', ''), + 'PREFERRED_BITRATE_ALLOW_LOSSLESS': (int, 'General', 0), + 'PREFERRED_BITRATE_HIGH_BUFFER': (int, 'General', 0), + 'PREFERRED_BITRATE_LOW_BUFFER': (int, 'General', 0), + 'PREFERRED_QUALITY': (int, 'General', 0), + 'PREFERRED_WORDS': (str, 'General', ''), + 'PREFER_TORRENTS': (int, 'General', 0), + 'PROWL_ENABLED': (int, 'Prowl', 0), + 'PROWL_KEYS': (str, 'Prowl', ''), + 'PROWL_ONSNATCH': (int, 'Prowl', 0), + 'PROWL_PRIORITY': (int, 'Prowl', 0), + 'PUSHALOT_APIKEY': (str, 'Pushalot', ''), + 'PUSHALOT_ENABLED': (int, 'Pushalot', 0), + 'PUSHALOT_ONSNATCH': (int, 'Pushalot', 0), + 'PUSHBULLET_APIKEY': (str, 'PushBullet', ''), + 'PUSHBULLET_DEVICEID': (str, 'PushBullet', ''), + 'PUSHBULLET_ENABLED': (int, 'PushBullet', 0), + 'PUSHBULLET_ONSNATCH': (int, 'PushBullet', 0), + 'PUSHOVER_APITOKEN': (str, 'Pushover', ''), + 'PUSHOVER_ENABLED': (int, 'Pushover', 0), + 'PUSHOVER_KEYS': (str, 'Pushover', ''), + 'PUSHOVER_ONSNATCH': (int, 'Pushover', 0), + 'PUSHOVER_PRIORITY': (int, 'Pushover', 0), + 'RENAME_FILES': (int, 'General', 0), + 'REPLACE_EXISTING_FOLDERS': (int, 'General', 0), + 'REQUIRED_WORDS': (str, 'General', ''), + 'RUTRACKER': (int, 'Rutracker', 0), + 'RUTRACKER_PASSWORD': (str, 'Rutracker', ''), + 'RUTRACKER_RATIO': (str, 'Rutracker', ''), + 'RUTRACKER_USER': (str, 'Rutracker', ''), + 'SAB_APIKEY': (str, 'SABnzbd', ''), + 'SAB_CATEGORY': (str, 'SABnzbd', ''), + 'SAB_HOST': (str, 'SABnzbd', ''), + 'SAB_PASSWORD': (str, 'SABnzbd', ''), + 'SAB_USERNAME': (str, 'SABnzbd', ''), + 'SAMPLINGFREQUENCY': (int, 'General', 44100), + 'SEARCH_INTERVAL': (int, 'General', 1440), + 'SONGKICK_APIKEY': (str, 'Songkick', 'nd1We7dFW2RqxPw8'), + 'SONGKICK_ENABLED': (int, 'Songkick', 1), + 'SONGKICK_FILTER_ENABLED': (int, 'Songkick', 0), + 'SONGKICK_LOCATION': (str, 'Songkick', ''), + 'SUBSONIC_ENABLED': (int, 'Subsonic', 0), + 'SUBSONIC_HOST': (str, 'Subsonic', ''), + 'SUBSONIC_PASSWORD': (str, 'Subsonic', ''), + 'SUBSONIC_USERNAME': (str, 'Subsonic', ''), + 'SYNOINDEX_ENABLED': (int, 'Synoindex', 0), + 'TORRENTBLACKHOLE_DIR': (str, 'General', ''), + 'TORRENT_DOWNLOADER': (int, 'General', 0), + 'TORRENT_REMOVAL_INTERVAL': (int, 'General', 720), + 'TRANSMISSION_HOST': (str, 'Transmission', ''), + 'TRANSMISSION_PASSWORD': (str, 'Transmission', ''), + 'TRANSMISSION_USERNAME': (str, 'Transmission', ''), + 'TWITTER_ENABLED': (int, 'Twitter', 0), + 'TWITTER_ONSNATCH': (int, 'Twitter', 0), + 'TWITTER_PASSWORD': (str, 'Twitter', ''), + 'TWITTER_PREFIX': (str, 'Twitter', 'Headphones'), + 'TWITTER_USERNAME': (str, 'Twitter', ''), + 'UPDATE_DB_INTERVAL': (int, 'General', 24), + 'USENET_RETENTION': (int, 'General', '1500'), + 'UTORRENT_HOST': (str, 'uTorrent', ''), + 'UTORRENT_LABEL': (str, 'uTorrent', ''), + 'UTORRENT_PASSWORD': (str, 'uTorrent', ''), + 'UTORRENT_USERNAME': (str, 'uTorrent', ''), + 'VERIFY_SSL_CERT': (bool_int, 'Advanced', 1), + 'WAFFLES': (int, 'Waffles', 0), + 'WAFFLES_PASSKEY': (str, 'Waffles', ''), + 'WAFFLES_RATIO': (str, 'Waffles', ''), + 'WAFFLES_UID': (str, 'Waffles', ''), + 'WHATCD': (int, 'What.cd', 0), + 'WHATCD_PASSWORD': (str, 'What.cd', ''), + 'WHATCD_RATIO': (str, 'What.cd', ''), + 'WHATCD_USERNAME': (str, 'What.cd', ''), + 'XBMC_ENABLED': (int, 'XBMC', 0), + 'XBMC_HOST': (str, 'XBMC', ''), + 'XBMC_NOTIFY': (int, 'XBMC', 0), + 'XBMC_PASSWORD': (str, 'XBMC', ''), + 'XBMC_UPDATE': (int, 'XBMC', 0), + 'XBMC_USERNAME': (str, 'XBMC', ''), + 'XLDPROFILE': (str, 'General', '') +} + +# pylint:disable=R0902 +# it might be nice to refactor for fewer instance variables +class Config(object): + """ Wraps access to particular values in a config file """ + + def __init__(self, config_file): + """ Initialize the config with values from a file """ + self._config_file = config_file + self._config = ConfigObj(self._config_file, encoding='utf-8') + for key in _CONFIG_DEFINITIONS.keys(): + self.check_setting(key) + self.ENCODER_MULTICORE_COUNT = max(0, self.ENCODER_MULTICORE_COUNT) + self._upgrade() + + def _define(self, name): + key = name.upper() + ini_key = name.lower() + definition = _CONFIG_DEFINITIONS[key] + if len(definition) == 3: + definition_type, section, default = definition + else: + definition_type, section, _, default = definition + return key, definition_type, section, ini_key, default + + def check_section(self, section): + """ Check if INI section exists, if not create it """ + if section not in self._config: + self._config[section] = {} + return True + else: + return False + + def check_setting(self, key): + """ Cast any value in the config to the right type or use the default """ + key, definition_type, section, ini_key, default = self._define(key) + self.check_section(section) + try: + my_val = definition_type(self._config[section][ini_key]) + except Exception: + my_val = definition_type(default) + self._config[section][ini_key] = my_val + return my_val + + def write(self): + """ Make a copy of the stored config and write it to the configured file """ + new_config = ConfigObj(encoding="UTF-8") + new_config.filename = self._config_file + + # first copy over everything from the old config, even if it is not + # correctly defined to keep from losing data + for key, subkeys in self._config.items(): + if key not in new_config: + new_config[key] = {} + for subkey, value in subkeys.items(): + new_config[key][subkey] = value + + # next make sure that everything we expect to have defined is so + for key in _CONFIG_DEFINITIONS.keys(): + key, definition_type, section, ini_key, default = self._define(key) + self.check_setting(key) + if section not in new_config: + new_config[section] = {} + new_config[section][ini_key] = self._config[section][ini_key] + + # Write it to file + headphones.logger.info("Writing configuration to file") + + try: + new_config.write() + except IOError as e: + headphones.logger.error("Error writing configuration file: %s", e) + + def get_extra_newznabs(self): + """ Return the extra newznab tuples """ + extra_newznabs = list( + itertools.izip(*[itertools.islice(self.EXTRA_NEWZNABS, i, None, 3) + for i in range(3)]) + ) + return extra_newznabs + + def clear_extra_newznabs(self): + """ Forget about the configured extra newznabs """ + self.EXTRA_NEWZNABS = [] + + def add_extra_newznab(self, newznab): + """ Add a new extra newznab """ + extra_newznabs = self.EXTRA_NEWZNABS + for item in newznab: + extra_newznabs.append(item) + self.EXTRA_NEWZNABS = extra_newznabs + + def __getattr__(self, name): + """ + Returns something from the ini unless it is a real property + of the configuration object or is not all caps. + """ + if not re.match(r'[A-Z_]+$', name): + return super(Config, self).__getattr__(name) + else: + return self.check_setting(name) + + def __setattr__(self, name, value): + """ + Maps all-caps properties to ini values unless they exist on the + configuration object. + """ + if not re.match(r'[A-Z_]+$', name): + super(Config, self).__setattr__(name, value) + return value + else: + key, definition_type, section, ini_key, default = self._define(name) + self._config[section][ini_key] = definition_type(value) + return self._config[section][ini_key] + + def process_kwargs(self, kwargs): + """ + Given a big bunch of key value pairs, apply them to the ini. + """ + for name, value in kwargs.items(): + key, definition_type, section, ini_key, default = self._define(name) + self._config[section][ini_key] = definition_type(value) + + def _upgrade(self): + """ Update folder formats in the config & bump up config version """ + if self.CONFIG_VERSION == '0': + from headphones.helpers import replace_all + file_values = { + 'tracknumber': 'Track', + 'title': 'Title', + 'artist': 'Artist', + 'album': 'Album', + 'year': 'Year' + } + folder_values = { + 'artist': 'Artist', + 'album': 'Album', + 'year': 'Year', + 'releasetype': 'Type', + 'first': 'First', + 'lowerfirst': 'first' + } + self.FILE_FORMAT = replace_all(self.FILE_FORMAT, file_values) + self.FOLDER_FORMAT = replace_all(self.FOLDER_FORMAT, folder_values) + + self.CONFIG_VERSION = '1' + + if self.CONFIG_VERSION == '1': + from headphones.helpers import replace_all + file_values = { + 'Track': '$Track', + 'Title': '$Title', + 'Artist': '$Artist', + 'Album': '$Album', + 'Year': '$Year', + 'track': '$track', + 'title': '$title', + 'artist': '$artist', + 'album': '$album', + 'year': '$year' + } + folder_values = { + 'Artist': '$Artist', + 'Album': '$Album', + 'Year': '$Year', + 'Type': '$Type', + 'First': '$First', + 'artist': '$artist', + 'album': '$album', + 'year': '$year', + 'type': '$type', + 'first': '$first' + } + self.FILE_FORMAT = replace_all(self.FILE_FORMAT, file_values) + self.FOLDER_FORMAT = replace_all(self.FOLDER_FORMAT, folder_values) + self.CONFIG_VERSION = '2' + + if self.CONFIG_VERSION == '2': + # Update the config to use direct path to the encoder rather than the encoder folder + if self.ENCODERFOLDER: + self.ENCODER_PATH = os.path.join(self.ENCODERFOLDER, self.ENCODER) + self.CONFIG_VERSION = '3' + + if self.CONFIG_VERSION == '3': + # Update the BLACKHOLE option to the NZB_DOWNLOADER format + if self.BLACKHOLE: + self.NZB_DOWNLOADER = 2 + self.CONFIG_VERSION = '4' + + # Enable Headphones Indexer if they have a VIP account + if self.CONFIG_VERSION == '4': + if self.HPUSER and self.HPPASS: + self.HEADPHONES_INDEXER = True + self.CONFIG_VERSION = '5' + + if self.CONFIG_VERSION == '5': + if self.OPEN_MAGNET_LINKS: + self.MAGNET_LINKS = 2 + self.CONFIG_VERSION = '5' diff --git a/headphones/cuesplit.py b/headphones/cuesplit.py new file mode 100755 index 00000000..467c93df --- /dev/null +++ b/headphones/cuesplit.py @@ -0,0 +1,661 @@ +# This file is part of Headphones. +# +# Headphones is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Headphones is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Headphones. If not, see . + +# Most of this lifted from here: https://github.com/SzieberthAdam/gneposis-cdgrab + +import os +import sys +import re +import subprocess +import copy +import glob + +import headphones +from headphones import logger +from mutagen.flac import FLAC + +CUE_HEADER = { + 'genre': '^REM GENRE (.+?)$', + 'date': '^REM DATE (.+?)$', + 'discid': '^REM DISCID (.+?)$', + 'comment': '^REM COMMENT (.+?)$', + 'catalog': '^CATALOG (.+?)$', + 'artist': '^PERFORMER (.+?)$', + 'title': '^TITLE (.+?)$', + 'file': '^FILE (.+?) (WAVE|FLAC)$', + 'accurateripid': '^REM ACCURATERIPID (.+?)$' +} + +CUE_TRACK = 'TRACK (\d\d) AUDIO$' + +CUE_TRACK_INFO = { + 'artist': 'PERFORMER (.+?)$', + 'title': 'TITLE (.+?)$', + 'isrc': 'ISRC (.+?)$', + 'index': 'INDEX (\d\d) (.+?)$' +} + +ALBUM_META_FILE_NAME = 'album.dat' +SPLIT_FILE_NAME = 'split.dat' + +ALBUM_META_ALBUM_BY_CUE = ('artist', 'title', 'date', 'genre') + +HTOA_LENGTH_TRIGGER = 3 + +WAVE_FILE_TYPE_BY_EXTENSION = { + '.wav': 'Waveform Audio', + '.wv': 'WavPack', + '.ape': "Monkey's Audio", + '.m4a': 'Apple Lossless', + '.flac': 'Free Lossless Audio Codec' +} + +# TODO: Only alow flac for now +#SHNTOOL_COMPATIBLE = ('Waveform Audio', 'WavPack', 'Free Lossless Audio Codec') +SHNTOOL_COMPATIBLE = ('Free Lossless Audio Codec') + +# TODO: Make this better! +# this module-level variable is bad. :( +CUE_META = None + + +def check_splitter(command): + '''Check xld or shntools installed''' + try: + env = os.environ.copy() + if 'xld' in command: + env['PATH'] += os.pathsep + '/Applications' + devnull = open(os.devnull) + subprocess.Popen([command], stdout=devnull, stderr=devnull, env=env).communicate() + except OSError as e: + if e.errno == os.errno.ENOENT: + return False + return True + + +def split_baby(split_file, split_cmd): + '''Let's split baby''' + logger.info('Splitting %s...', split_file.decode(headphones.SYS_ENCODING, 'replace')) + logger.debug(subprocess.list2cmdline(split_cmd)) + + # Prevent Windows from opening a terminal window + startupinfo = None + + if headphones.SYS_PLATFORM == "win32": + startupinfo = subprocess.STARTUPINFO() + try: + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + except AttributeError: + startupinfo.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW + + env = os.environ.copy() + if 'xld' in split_cmd: + env['PATH'] += os.pathsep + '/Applications' + + process = subprocess.Popen(split_cmd, startupinfo=startupinfo, + + stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=env) + stdout, stderr = process.communicate() + if process.returncode: + logger.error('Split failed for %s', split_file.decode(headphones.SYS_ENCODING, 'replace')) + out = stdout if stdout else stderr + logger.error('Error details: %s', out.decode(headphones.SYS_ENCODING, 'replace')) + return False + else: + logger.info('Split success %s', split_file.decode(headphones.SYS_ENCODING, 'replace')) + return True + + +def check_list(list, ignore=0): + '''Checks a list for None elements. If list have None (after ignore index) then it should pass only if all elements + are None threreafter. Returns a tuple without the None entries.''' + + if ignore: + try: + list[int(ignore)] + except: + raise ValueError('non-integer ignore index or ignore index not in list') + + list1 = list[:ignore] + list2 = list[ignore:] + + try: + first_none = list2.index(None) + except: + return tuple(list1 + list2) + + for i in range(first_none, len(list2)): + if list2[i]: + raise ValueError('non-None entry after None entry in list at index {0}'.format(i)) + + while True: + list2.remove(None) + try: + list2.index(None) + except: + break + + return tuple(list1 + list2) + + +def trim_cue_entry(string): + '''Removes leading and trailing "s.''' + if string[0] == '"' and string[-1] == '"': + string = string[1:-1] + return string + + +def int_to_str(value, length=2): + '''Converts integer to string eg 3 to "03"''' + try: + int(value) + except: + raise ValueError('expected an integer value') + + content = str(value) + while len(content) < length: + content = '0' + content + return content + + +class Directory: + def __init__(self, path): + self.path = path + self.name = os.path.split(self.path)[-1] + self.content = [] + self.update() + + def filter(self, classname): + content = [] + for c in self.content: + if c.__class__.__name__ == classname: + content.append(c) + return content + + def tracks(self, ext=None, split=False): + content = [] + for c in self.content: + ext_match = False + if c.__class__.__name__ == 'WaveFile': + if not ext or (ext and ext == c.name_ext): + ext_match = True + if ext_match and c.track_nr: + if not split or (split and c.split_file): + content.append(c) + return content + + def update(self): + def check_match(filename): + for i in self.content: + if i.name == filename: + return True + return False + + def identify_track_number(filename): + if 'split-track' in filename: + search = re.search('split-track(\d\d)', filename) + if search: + n = int(search.group(1)) + if n: + return n + for n in range(0, 100): + search = re.search(int_to_str(n), filename) + if search: + # TODO: not part of other value such as year + return n + + list_dir = glob.glob1(self.path, '*') + + # TODO: for some reason removes only one file + rem_list = [] + for i in self.content: + if i.name not in list_dir: + rem_list.append(i) + for i in rem_list: + self.content.remove(i) + + for i in list_dir: + if not check_match(i): + # music file + if os.path.splitext(i)[-1] in WAVE_FILE_TYPE_BY_EXTENSION.keys(): + track_nr = identify_track_number(i) + if track_nr: + self.content.append(WaveFile(self.path + os.sep + i, track_nr=track_nr)) + else: + self.content.append(WaveFile(self.path + os.sep + i)) + + # cue file + elif os.path.splitext(i)[-1] == '.cue': + self.content.append(CueFile(self.path + os.sep + i)) + + # meta file + elif i == ALBUM_META_FILE_NAME: + self.content.append(MetaFile(self.path + os.sep + i)) + + # directory + elif os.path.isdir(i): + self.content.append(Directory(self.path + os.sep + i)) + + else: + self.content.append(File(self.path + os.sep + i)) + + +class File(object): + def __init__(self, path): + self.path = path + self.name = os.path.split(self.path)[-1] + + self.name_name = ''.join(os.path.splitext(self.name)[:-1]) + self.name_ext = os.path.splitext(self.name)[-1] + self.split_file = True if self.name_name[:11] == 'split-track' else False + + def get_name(self, ext=True, cmd=False): + + if ext is True: + content = self.name + elif ext is False: + content = self.name_name + elif ext[0] == '.': + content = self.name_name + ext + else: + raise ValueError('ext parameter error') + + if cmd: + content = content.replace(' ', '\ ') + + return content + + +class CueFile(File): + def __init__(self, path): + + def header_parser(): + global line_content + c = self.content.splitlines() + header_dict = {} + #remaining_headers = CUE_HEADER + remaining_headers = copy.copy(CUE_HEADER) + line_index = 0 + match = True + while match: + match = False + saved_match = None + line_content = c[line_index] + for e in remaining_headers: + search_result = re.search(remaining_headers[e], line_content, re.I) + if search_result: + search_content = trim_cue_entry(search_result.group(1)) + header_dict[e] = search_content + saved_match = e + match = True + line_index += 1 + if saved_match: + del remaining_headers[saved_match] + return header_dict, line_index + + def track_parser(start_line): + c = self.content.splitlines() + line_index = start_line + line_content = c[line_index] + search_result = re.search(CUE_TRACK, line_content, re.I) + if not search_result: + raise ValueError('inconsistent CUE sheet, TRACK expected at line {0}'.format(line_index + 1)) + track_nr = int(search_result.group(1)) + line_index += 1 + next_track = False + track_meta = {} + # we make room for future indexes + track_meta['index'] = [None for m in range(100)] + + while not next_track: + if line_index < len(c): + line_content = c[line_index] + + artist_search = re.search(CUE_TRACK_INFO['artist'], line_content, re.I) + title_search = re.search(CUE_TRACK_INFO['title'], line_content, re.I) + isrc_search = re.search(CUE_TRACK_INFO['isrc'], line_content, re.I) + index_search = re.search(CUE_TRACK_INFO['index'], line_content, re.I) + + if artist_search: + if trim_cue_entry(artist_search.group(1)) != self.header['artist']: + track_meta['artist'] = trim_cue_entry(artist_search.group(1)) + line_index += 1 + elif title_search: + track_meta['title'] = trim_cue_entry(title_search.group(1)) + line_index += 1 + elif isrc_search: + track_meta['isrc'] = trim_cue_entry(isrc_search.group(1)) + line_index += 1 + elif index_search: + track_meta['index'][int(index_search.group(1))] = index_search.group(2) + line_index += 1 + elif re.search(CUE_TRACK, line_content, re.I): + next_track = True + elif line_index == len(c) - 1 and not line_content: + # last line is empty + line_index += 1 + elif re.search('FLAGS DCP$', line_content, re.I): + track_meta['dcpflag'] = True + line_index += 1 + else: + raise ValueError('unknown entry in track error, line {0}'.format(line_index + 1)) + else: + next_track = True + + track_meta['index'] = check_list(track_meta['index'], ignore=1) + + return track_nr, track_meta, line_index + + super(CueFile, self).__init__(path) + + try: + with open(self.name) as cue_file: + self.content = cue_file.read() + except: + self.content = None + + if not self.content: + try: + with open(self.name, encoding="cp1252") as cue_file: + self.content = cue_file.read() + except: + raise ValueError('Cant encode CUE Sheet.') + + if self.content[0] == u'\ufeff': + self.content = self.content[1:] + + header = header_parser() + + self.header = header[0] + + line_index = header[1] + + # we make room for tracks + tracks = [None for m in range(100)] + + while line_index < len(self.content.splitlines()): + parsed_track = track_parser(line_index) + line_index = parsed_track[2] + tracks[parsed_track[0]] = parsed_track[1] + + self.tracks = check_list(tracks, ignore=1) + + def get_meta(self): + content = '' + for i in ALBUM_META_ALBUM_BY_CUE: + if self.header.get(i): + content += i + '\t' + self.header[i] + '\n' + else: + content += i + '\t' + '\n' + + for i in range(len(self.tracks)): + if self.tracks[i]: + if self.tracks[i].get('artist'): + content += 'track' + int_to_str(i) + 'artist' + '\t' + self.tracks[i].get('artist') + '\n' + if self.tracks[i].get('title'): + content += 'track' + int_to_str(i) + 'title' + '\t' + self.tracks[i].get('title') + '\n' + return content + + def htoa(self): + '''Returns true if Hidden Track exists.''' + if int(self.tracks[1]['index'][1][-5:-3]) >= HTOA_LENGTH_TRIGGER: + return True + return False + + def breakpoints(self): + '''Returns track break points. Identical as CUETools' cuebreakpoints, with the exception of my standards for HTOA.''' + content = '' + for t in range(len(self.tracks)): + if t == 1 and not self.htoa(): + content += '' + elif t >= 1: + t_index = self.tracks[t]['index'] + content += t_index[1] + if (t < len(self.tracks) - 1): + content += '\n' + return content + + +class MetaFile(File): + def __init__(self, path): + super(MetaFile, self).__init__(path) + with open(self.path) as meta_file: + self.rawcontent = meta_file.read() + + content = {} + content['tracks'] = [None for m in range(100)] + + for l in self.rawcontent.splitlines(): + parsed_line = re.search('^(.+?)\t(.+?)$', l) + if parsed_line: + if parsed_line.group(1)[:5] == 'track': + parsed_track = re.search('^track(\d\d)(.+?)$', parsed_line.group(1)) + if not parsed_track: + raise ValueError('Syntax error in album meta file') + if not content['tracks'][int(parsed_track.group(1))]: + content['tracks'][int(parsed_track.group(1))] = dict() + content['tracks'][int(parsed_track.group(1))][parsed_track.group(2)] = parsed_line.group(2) + else: + content[parsed_line.group(1)] = parsed_line.group(2) + + content['tracks'] = check_list(content['tracks'], ignore=1) + + self.content = content + + def flac_tags(self, track_nr): + common_tags = dict() + freeform_tags = dict() + + # common flac tags + common_tags['artist'] = self.content['artist'] + common_tags['album'] = self.content['title'] + common_tags['title'] = self.content['tracks'][track_nr]['title'] + common_tags['tracknumber'] = str(track_nr) + common_tags['tracktotal'] = str(len(self.content['tracks']) - 1) + if 'date' in self.content: + common_tags['date'] = self.content['date'] + if 'genre' in CUE_META.content: + common_tags['genre'] = CUE_META.content['genre'] + + #freeform tags + #freeform_tags['country'] = self.content['country'] + #freeform_tags['releasedate'] = self.content['releasedate'] + + return common_tags, freeform_tags + + def folders(self): + artist = self.content['artist'] + album = self.content['date'] + ' - ' + self.content['title'] + ' (' + self.content['label'] + ' - ' + self.content['catalog'] + ')' + return artist, album + + def complete(self): + '''Check MetaFile for containing all data''' + self.__init__(self.path) + for l in self.rawcontent.splitlines(): + if re.search('^[0-9A-Za-z]+?\t$', l): + return False + return True + + def count_tracks(self): + '''Returns tracks count''' + return len(self.content['tracks']) - self.content['tracks'].count(None) + + +class WaveFile(File): + def __init__(self, path, track_nr=None): + super(WaveFile, self).__init__(path) + + self.track_nr = track_nr + self.type = WAVE_FILE_TYPE_BY_EXTENSION[self.name_ext] + + def filename(self, ext=None, cmd=False): + title = CUE_META.content['tracks'][self.track_nr]['title'] + + if ext: + if ext[0] != '.': + ext = '.' + ext + else: + ext = self.name_ext + + f_name = int_to_str(self.track_nr) + ' - ' + title + ext + + if cmd: + f_name = f_name.replace(' ', '\ ') + + f_name = f_name.replace('!', '') + f_name = f_name.replace('?', '') + f_name = f_name.replace('/', ';') + + return f_name + + def tag(self): + if self.type == 'Free Lossless Audio Codec': + f = FLAC(self.name) + tags = CUE_META.flac_tags(self.track_nr) + for t in tags[0]: + f[t] = tags[0][t] + f.save() + + def mutagen(self): + if self.type == 'Free Lossless Audio Codec': + return FLAC(self.name) + +def split(albumpath): + global CUE_META + os.chdir(albumpath) + base_dir = Directory(os.getcwd()) + + cue = None + wave = None + + # determining correct cue file + # if perfect match found + for _cue in base_dir.filter('CueFile'): + for _wave in base_dir.filter('WaveFile'): + if _cue.header['file'] == _wave.name: + logger.info('CUE Sheet found: {0}'.format(_cue.name)) + logger.info('Music file found: {0}'.format(_wave.name)) + cue = _cue + wave = _wave + # if no perfect match found then try without extensions + if not cue and not wave: + logger.info('No match for music files, trying to match without extensions...') + for _cue in base_dir.filter('CueFile'): + for _wave in base_dir.filter('WaveFile'): + if ''.join(os.path.splitext(_cue.header['file'])[:-1]) == _wave.name_name: + logger.info('Possible CUE Sheet found: {0}'.format(_cue.name)) + logger.info('CUE Sheet refers music file: {0}'.format(_cue.header['file'])) + logger.info('Possible Music file found: {0}'.format(_wave.name)) + cue = _cue + wave = _wave + cue.header['file'] = wave.name + # if still no match then raise an exception + if not cue and not wave: + raise ValueError('No music file match found!') + + # Split with xld or shntool + splitter = 'shntool' + xldprofile = None + + # use xld profile to split cue + if headphones.CONFIG.ENCODER == 'xld' and headphones.CONFIG.MUSIC_ENCODER and headphones.CONFIG.XLDPROFILE: + import getXldProfile + xldprofile, xldformat, _ = getXldProfile.getXldProfile(headphones.CONFIG.XLDPROFILE) + if not xldformat: + raise ValueError('Details for xld profile "%s" not found, cannot split cue' % (xldprofile)) + else: + if headphones.CONFIG.ENCODERFOLDER: + splitter = os.path.join(headphones.CONFIG.ENCODERFOLDER, 'xld') + else: + splitter = 'xld' + # use standard xld command to split cue + elif sys.platform == 'darwin': + splitter = 'xld' + if not check_splitter(splitter): + splitter = 'shntool' + + if splitter == 'shntool' and not check_splitter(splitter): + raise ValueError('Command not found, ensure shntools with FLAC or xld (OS X) installed') + + # Determine if file can be split (only flac allowed for shntools) + if 'xld' in splitter and wave.name_ext not in WAVE_FILE_TYPE_BY_EXTENSION.keys() or \ + wave.type not in SHNTOOL_COMPATIBLE: + raise ValueError('Cannot split, audio file has unsupported extension') + + # Split with xld + if 'xld' in splitter: + cmd = [splitter] + cmd.extend([wave.name]) + cmd.extend(['-c']) + cmd.extend([cue.name]) + if xldprofile: + cmd.extend(['--profile']) + cmd.extend([xldprofile]) + else: + cmd.extend(['-f']) + cmd.extend(['flac']) + cmd.extend(['-o']) + cmd.extend([base_dir.path]) + split = split_baby(wave.name, cmd) + else: + + # Split with shntool + + # generate temporary metafile describing the cue + with open(ALBUM_META_FILE_NAME, mode='w') as meta_file: + meta_file.write(cue.get_meta()) + base_dir.content.append(MetaFile(os.path.abspath(ALBUM_META_FILE_NAME))) + # check metafile for completeness + if not base_dir.filter('MetaFile'): + raise ValueError('Cue Meta file {0} missing!'.format(ALBUM_META_FILE_NAME)) + else: + CUE_META = base_dir.filter('MetaFile')[0] + + with open(SPLIT_FILE_NAME, mode='w') as split_file: + split_file.write(cue.breakpoints()) + + cmd = ['shntool'] + cmd.extend(['split']) + cmd.extend(['-f']) + cmd.extend([SPLIT_FILE_NAME]) + cmd.extend(['-o']) + cmd.extend(['flac']) + cmd.extend([wave.name]) + split = split_baby(wave.name, cmd) + os.remove(SPLIT_FILE_NAME) + base_dir.update() + + # tag FLAC files + if split and CUE_META.count_tracks() == len(base_dir.tracks(ext='.flac', split=True)): + for t in base_dir.tracks(ext='.flac', split=True): + logger.info('Tagging {0}...'.format(t.name)) + t.tag() + + # rename FLAC files + if split and CUE_META.count_tracks() == len(base_dir.tracks(ext='.flac', split=True)): + for t in base_dir.tracks(ext='.flac', split=True): + if t.name != t.filename(): + logger.info('Renaming {0} to {1}...'.format(t.name, t.filename())) + os.rename(t.name, t.filename()) + + os.remove(ALBUM_META_FILE_NAME) + + if not split: + raise ValueError('Failed to split, check logs') + else: + # Rename original file + os.rename(wave.name, wave.name + '.original') + return True diff --git a/headphones/db.py b/headphones/db.py index 060b91b1..cf4cdeae 100644 --- a/headphones/db.py +++ b/headphones/db.py @@ -21,23 +21,24 @@ from __future__ import with_statement import os import sqlite3 -import threading -import time import headphones from headphones import logger + def dbFilename(filename="headphones.db"): return os.path.join(headphones.DATA_DIR, filename) + def getCacheSize(): #this will protect against typecasting problems produced by empty string and None settings - if not headphones.CACHE_SIZEMB: + if not headphones.CONFIG.CACHE_SIZEMB: #sqlite will work with this (very slowly) return 0 - return int(headphones.CACHE_SIZEMB) + return int(headphones.CONFIG.CACHE_SIZEMB) + class DBConnection: @@ -48,25 +49,25 @@ class DBConnection: #don't wait for the disk to finish writing self.connection.execute("PRAGMA synchronous = OFF") #journal disabled since we never do rollbacks - self.connection.execute("PRAGMA journal_mode = %s" % headphones.JOURNAL_MODE) + self.connection.execute("PRAGMA journal_mode = %s" % headphones.CONFIG.JOURNAL_MODE) #64mb of cache memory,probably need to make it user configurable - self.connection.execute("PRAGMA cache_size=-%s" % (getCacheSize()*1024)) + self.connection.execute("PRAGMA cache_size=-%s" % (getCacheSize() * 1024)) self.connection.row_factory = sqlite3.Row def action(self, query, args=None): - if query == None: + if query is None: return sqlResult = None - + try: with self.connection as c: - if args == None: + if args is None: sqlResult = c.execute(query) else: sqlResult = c.execute(query, args) - + except sqlite3.OperationalError, e: if "unable to open database file" in e.message or "database is locked" in e.message: logger.warn('Database Error: %s', e) @@ -77,14 +78,14 @@ class DBConnection: except sqlite3.DatabaseError, e: logger.error('Fatal Error executing %s :: %s', query, e) raise - + return sqlResult def select(self, query, args=None): sqlResults = self.action(query, args).fetchall() - - if sqlResults == None or sqlResults == [None]: + + if sqlResults is None or sqlResults == [None]: return [] return sqlResults @@ -93,13 +94,13 @@ class DBConnection: changesBefore = self.connection.total_changes - genParams = lambda myDict : [x + " = ?" for x in myDict.keys()] + genParams = lambda myDict: [x + " = ?" for x in myDict.keys()] - query = "UPDATE "+tableName+" SET " + ", ".join(genParams(valueDict)) + " WHERE " + " AND ".join(genParams(keyDict)) + query = "UPDATE " + tableName + " SET " + ", ".join(genParams(valueDict)) + " WHERE " + " AND ".join(genParams(keyDict)) self.action(query, valueDict.values() + keyDict.values()) if self.connection.total_changes == changesBefore: - query = "INSERT INTO "+tableName+" (" + ", ".join(valueDict.keys() + keyDict.keys()) + ")" + \ + query = "INSERT INTO " + tableName + " (" + ", ".join(valueDict.keys() + keyDict.keys()) + ")" + \ " VALUES (" + ", ".join(["?"] * len(valueDict.keys() + keyDict.keys())) + ")" self.action(query, valueDict.values() + keyDict.values()) diff --git a/headphones/exceptions.py b/headphones/exceptions.py index a1e62f1a..5d0ddf52 100644 --- a/headphones/exceptions.py +++ b/headphones/exceptions.py @@ -13,11 +13,13 @@ # You should have received a copy of the GNU General Public License # along with Headphones. If not, see . + class HeadphonesException(Exception): """ Generic Headphones Exception - should never be thrown, only subclassed """ + class NewzbinAPIThrottled(HeadphonesException): """ Newzbin has throttled us, deal with it diff --git a/headphones/getXldProfile.py b/headphones/getXldProfile.py index e083d1b6..b143cfa1 100755 --- a/headphones/getXldProfile.py +++ b/headphones/getXldProfile.py @@ -2,26 +2,26 @@ import os.path import plistlib import sys import xml.parsers.expat as expat -import commands from headphones import logger + def getXldProfile(xldProfile): xldProfileNotFound = xldProfile expandedPath = os.path.expanduser('~/Library/Preferences/jp.tmkk.XLD.plist') try: preferences = plistlib.Plist.fromFile(expandedPath) except (expat.ExpatError): - os.system("/usr/bin/plutil -convert xml1 %s" % expandedPath ) + os.system("/usr/bin/plutil -convert xml1 %s" % expandedPath) try: preferences = plistlib.Plist.fromFile(expandedPath) except (ImportError): - os.system("/usr/bin/plutil -convert binary1 %s" % expandedPath ) - logger.info('The plist at "%s" has a date in it, and therefore is not useable.' % expandedPath) + os.system("/usr/bin/plutil -convert binary1 %s" % expandedPath) + logger.info('The plist at "%s" has a date in it, and therefore is not useable.', expandedPath) return(xldProfileNotFound, None, None) except (ImportError): - logger.info('The plist at "%s" has a date in it, and therefore is not useable.' % expandedPath) + logger.info('The plist at "%s" has a date in it, and therefore is not useable.', expandedPath) except: - logger.info('Unexpected error:', sys.exc_info()[0]) + logger.info('Unexpected error: %s', sys.exc_info()[0]) return(xldProfileNotFound, None, None) xldProfile = xldProfile.lower() @@ -178,4 +178,4 @@ def getXldProfile(xldProfile): return(xldProfileForCmd, xldFormat, xldBitrate) - return(xldProfileNotFound, None, None) \ No newline at end of file + return(xldProfileNotFound, None, None) diff --git a/headphones/helpers.py b/headphones/helpers.py index 6437e2d1..29f1fe23 100644 --- a/headphones/helpers.py +++ b/headphones/helpers.py @@ -31,8 +31,9 @@ RE_FEATURING = re.compile(r"[fF]t\.|[fF]eaturing|[fF]eat\.|\b[wW]ith\b|&|vs\.") RE_CD_ALBUM = re.compile(r"\(?((CD|disc)\s*[0-9]+)\)?", re.I) RE_CD = re.compile(r"^(CD|dics)\s*[0-9]+$", re.I) + def multikeysort(items, columns): - comparers = [ ((itemgetter(col[1:].strip()), -1) if col.startswith('-') else (itemgetter(col.strip()), 1)) for col in columns] + comparers = [((itemgetter(col[1:].strip()), -1) if col.startswith('-') else (itemgetter(col.strip()), 1)) for col in columns] def comparer(left, right): for fn, mult in comparers: @@ -44,12 +45,14 @@ def multikeysort(items, columns): return sorted(items, cmp=comparer) + def checked(variable): if variable: return 'Checked' else: return '' + def radio(variable, pos): if variable == pos: @@ -57,40 +60,41 @@ def radio(variable, pos): else: return '' + def latinToAscii(unicrap): """ From couch potato """ - xlate = {0xc0:'A', 0xc1:'A', 0xc2:'A', 0xc3:'A', 0xc4:'A', 0xc5:'A', - 0xc6:'Ae', 0xc7:'C', - 0xc8:'E', 0xc9:'E', 0xca:'E', 0xcb:'E', 0x86:'e', - 0xcc:'I', 0xcd:'I', 0xce:'I', 0xcf:'I', - 0xd0:'Th', 0xd1:'N', - 0xd2:'O', 0xd3:'O', 0xd4:'O', 0xd5:'O', 0xd6:'O', 0xd8:'O', - 0xd9:'U', 0xda:'U', 0xdb:'U', 0xdc:'U', - 0xdd:'Y', 0xde:'th', 0xdf:'ss', - 0xe0:'a', 0xe1:'a', 0xe2:'a', 0xe3:'a', 0xe4:'a', 0xe5:'a', - 0xe6:'ae', 0xe7:'c', - 0xe8:'e', 0xe9:'e', 0xea:'e', 0xeb:'e', 0x0259:'e', - 0xec:'i', 0xed:'i', 0xee:'i', 0xef:'i', - 0xf0:'th', 0xf1:'n', - 0xf2:'o', 0xf3:'o', 0xf4:'o', 0xf5:'o', 0xf6:'o', 0xf8:'o', - 0xf9:'u', 0xfa:'u', 0xfb:'u', 0xfc:'u', - 0xfd:'y', 0xfe:'th', 0xff:'y', - 0xa1:'!', 0xa2:'{cent}', 0xa3:'{pound}', 0xa4:'{currency}', - 0xa5:'{yen}', 0xa6:'|', 0xa7:'{section}', 0xa8:'{umlaut}', - 0xa9:'{C}', 0xaa:'{^a}', 0xab:'<<', 0xac:'{not}', - 0xad:'-', 0xae:'{R}', 0xaf:'_', 0xb0:'{degrees}', - 0xb1:'{+/-}', 0xb2:'{^2}', 0xb3:'{^3}', 0xb4:"'", - 0xb5:'{micro}', 0xb6:'{paragraph}', 0xb7:'*', 0xb8:'{cedilla}', - 0xb9:'{^1}', 0xba:'{^o}', 0xbb:'>>', - 0xbc:'{1/4}', 0xbd:'{1/2}', 0xbe:'{3/4}', 0xbf:'?', - 0xd7:'*', 0xf7:'/' + xlate = {0xc0: 'A', 0xc1: 'A', 0xc2: 'A', 0xc3: 'A', 0xc4: 'A', 0xc5: 'A', + 0xc6: 'Ae', 0xc7: 'C', + 0xc8: 'E', 0xc9: 'E', 0xca: 'E', 0xcb: 'E', 0x86: 'e', + 0xcc: 'I', 0xcd: 'I', 0xce: 'I', 0xcf: 'I', + 0xd0: 'Th', 0xd1: 'N', + 0xd2: 'O', 0xd3: 'O', 0xd4: 'O', 0xd5: 'O', 0xd6: 'O', 0xd8: 'O', + 0xd9: 'U', 0xda: 'U', 0xdb: 'U', 0xdc: 'U', + 0xdd: 'Y', 0xde: 'th', 0xdf: 'ss', + 0xe0: 'a', 0xe1: 'a', 0xe2: 'a', 0xe3: 'a', 0xe4: 'a', 0xe5: 'a', + 0xe6: 'ae', 0xe7: 'c', + 0xe8: 'e', 0xe9: 'e', 0xea: 'e', 0xeb: 'e', 0x0259: 'e', + 0xec: 'i', 0xed: 'i', 0xee: 'i', 0xef: 'i', + 0xf0: 'th', 0xf1: 'n', + 0xf2: 'o', 0xf3: 'o', 0xf4: 'o', 0xf5: 'o', 0xf6: 'o', 0xf8: 'o', + 0xf9: 'u', 0xfa: 'u', 0xfb: 'u', 0xfc: 'u', + 0xfd: 'y', 0xfe: 'th', 0xff: 'y', + 0xa1: '!', 0xa2: '{cent}', 0xa3: '{pound}', 0xa4: '{currency}', + 0xa5: '{yen}', 0xa6: '|', 0xa7: '{section}', 0xa8: '{umlaut}', + 0xa9: '{C}', 0xaa: '{^a}', 0xab: '<<', 0xac: '{not}', + 0xad: '-', 0xae: '{R}', 0xaf: '_', 0xb0: '{degrees}', + 0xb1: '{+/-}', 0xb2: '{^2}', 0xb3: '{^3}', 0xb4: "'", + 0xb5: '{micro}', 0xb6: '{paragraph}', 0xb7: '*', 0xb8: '{cedilla}', + 0xb9: '{^1}', 0xba: '{^o}', 0xbb: '>>', + 0xbc: '{1/4}', 0xbd: '{1/2}', 0xbe: '{3/4}', 0xbf: '?', + 0xd7: '*', 0xf7: '/' } r = '' for i in unicrap: - if xlate.has_key(ord(i)): + if ord(i) in xlate: r += xlate[ord(i)] elif ord(i) >= 0x80: pass @@ -98,9 +102,10 @@ def latinToAscii(unicrap): r += str(i) return r + def convert_milliseconds(ms): - seconds = ms/1000 + seconds = ms / 1000 gmtime = time.gmtime(seconds) if seconds > 3600: minutes = time.strftime("%H:%M:%S", gmtime) @@ -109,6 +114,7 @@ def convert_milliseconds(ms): return minutes + def convert_seconds(s): gmtime = time.gmtime(s) @@ -119,15 +125,18 @@ def convert_seconds(s): return minutes + def today(): today = datetime.date.today() yyyymmdd = datetime.date.isoformat(today) return yyyymmdd + def now(): now = datetime.datetime.now() return now.strftime("%Y-%m-%d %H:%M:%S") + def get_age(date): try: @@ -136,22 +145,25 @@ def get_age(date): return False try: - days_old = int(split_date[0])*365 + int(split_date[1])*30 + int(split_date[2]) + days_old = int(split_date[0]) * 365 + int(split_date[1]) * 30 + int(split_date[2]) except IndexError: days_old = False return days_old + def bytes_to_mb(bytes): - mb = int(bytes)/1048576 + mb = int(bytes) / 1048576 size = '%.1f MB' % mb return size + def mb_to_bytes(mb_str): result = re.search('^(\d+(?:\.\d+)?)\s?(?:mb)?', mb_str, flags=re.I) if result: - return int(float(result.group(1))*1048576) + return int(float(result.group(1)) * 1048576) + def piratesize(size): split = size.split(" ") @@ -170,6 +182,7 @@ def piratesize(size): return size + def replace_all(text, dic, normalize=False): if not text: @@ -187,6 +200,7 @@ def replace_all(text, dic, normalize=False): text = text.replace(i, j) return text + def replace_illegal_chars(string, type="file"): if type == "file": string = re.sub('[\?"*:|<>/]', '_', string) @@ -195,6 +209,7 @@ def replace_illegal_chars(string, type="file"): return string + def cleanName(string): pass1 = latinToAscii(string).lower() @@ -202,6 +217,7 @@ def cleanName(string): return out_string + def cleanTitle(title): title = re.sub('[\.\-\/\_]', ' ', title).lower() @@ -213,6 +229,7 @@ def cleanTitle(title): return title + def split_path(f): """ Split a path into components, starting with the drive letter (if any). Given @@ -222,7 +239,7 @@ def split_path(f): components = [] drive, path = os.path.splitdrive(f) - # Stip the folder from the path, iterate until nothing is left + # Strip the folder from the path, iterate until nothing is left while True: path, folder = os.path.split(path) @@ -244,6 +261,7 @@ def split_path(f): # Done return components + def expand_subfolders(f): """ Try to expand a given folder and search for subfolders containing media @@ -272,7 +290,7 @@ def expand_subfolders(f): return # Split into path components - media_folders = [ split_path(media_folder) for media_folder in media_folders ] + media_folders = [split_path(media_folder) for media_folder in media_folders] # Correct folder endings such as CD1 etc. for index, media_folder in enumerate(media_folders): @@ -280,7 +298,7 @@ def expand_subfolders(f): media_folders[index] = media_folders[index][:-1] # Verify the result by computing path depth relative to root. - path_depths = [ len(media_folder) for media_folder in media_folders ] + path_depths = [len(media_folder) for media_folder in media_folders] difference = max(path_depths) - min(path_depths) if difference > 0: @@ -290,15 +308,15 @@ def expand_subfolders(f): # directory may contain separate CD's and maybe some extra's. The # structure may look like X albums at same depth, and (one or more) # extra folders with a higher depth. - extra_media_folders = [ media_folder[:min(path_depths)] for media_folder in media_folders if len(media_folder) > min(path_depths) ] - extra_media_folders = list(set([ os.path.join(*media_folder) for media_folder in extra_media_folders ])) + extra_media_folders = [media_folder[:min(path_depths)] for media_folder in media_folders if len(media_folder) > min(path_depths)] + extra_media_folders = list(set([os.path.join(*media_folder) for media_folder in extra_media_folders])) logger.info("Please look at the following folder(s), since they cause the depth difference: %s", extra_media_folders) return # Convert back to paths and remove duplicates, which may be there after # correcting the paths - media_folders = list(set([ os.path.join(*media_folder) for media_folder in media_folders ])) + media_folders = list(set([os.path.join(*media_folder) for media_folder in media_folders])) # Don't return a result if the number of subfolders is one. In this case, # this algorithm will not improve processing and will likely interfere @@ -310,23 +328,15 @@ def expand_subfolders(f): logger.debug("Expanded subfolders in folder: %s", media_folders) return media_folders + def extract_data(s): s = s.replace('_', ' ') #headphones default format - pattern = re.compile(r'(?P.*?)\s\-\s(?P.*?)\s\[(?P.*?)\]', re.VERBOSE) + pattern = re.compile(r'(?P.*?)\s\-\s(?P.*?)\s[\[\(](?P.*?)[\]\)]', re.VERBOSE) match = pattern.match(s) - if match: - name = match.group("name") - album = match.group("album") - year = match.group("year") - return (name, album, year) - - #newzbin default format - pattern = re.compile(r'(?P.*?)\s\-\s(?P.*?)\s\((?P\d+?\))', re.VERBOSE) - match = pattern.match(s) if match: name = match.group("name") album = match.group("album") @@ -346,6 +356,7 @@ def extract_data(s): else: return (None, None, None) + def extract_metadata(f): """ Scan all files in the given directory and decide on an artist, album and @@ -395,9 +406,9 @@ def extract_metadata(f): return (None, None, None) # Count distinct values - artists = list(set([ x[0] for x in results ])) - albums = list(set([ x[1] for x in results ])) - years = list(set([ x[2] for x in results ])) + artists = list(set([x[0] for x in results])) + albums = list(set([x[1] for x in results])) + years = list(set([x[2] for x in results])) # Remove things such as CD2 from album names if len(albums) > 1: @@ -425,8 +436,8 @@ def extract_metadata(f): # (Lots of) different artists. Could be a featuring album, so test for this. if len(artists) > 1 and len(albums) == 1: - split_artists = [ RE_FEATURING.split(artist) for artist in artists ] - featurings = [ len(split_artist) - 1 for split_artist in split_artists ] + split_artists = [RE_FEATURING.split(x) for x in artists] + featurings = [len(split_artist) - 1 for split_artist in split_artists] logger.info("Album seem to feature %d different artists", sum(featurings)) if sum(featurings) > 0: @@ -444,6 +455,79 @@ def extract_metadata(f): return (None, None, None) + +def get_downloaded_track_list(albumpath): + """ + Return a list of audio files for the given directory. + """ + downloaded_track_list = [] + + for root, dirs, files in os.walk(albumpath): + for _file in files: + extension = os.path.splitext(_file)[1].lower()[1:] + if extension in headphones.MEDIA_FORMATS: + downloaded_track_list.append(os.path.join(root, _file)) + + return downloaded_track_list + + +def preserve_torrent_direcory(albumpath): + """ + Copy torrent directory to headphones-modified to keep files for seeding. + """ + from headphones import logger + new_folder = os.path.join(albumpath, 'headphones-modified'.encode(headphones.SYS_ENCODING, 'replace')) + logger.info("Copying files to 'headphones-modified' subfolder to preserve downloaded files for seeding") + try: + shutil.copytree(albumpath, new_folder) + return new_folder + except Exception as e: + logger.warn("Cannot copy/move files to temp folder: " + \ + new_folder.decode(headphones.SYS_ENCODING, 'replace') + \ + ". Not continuing. Error: " + str(e)) + return None + + +def cue_split(albumpath): + """ + Attempts to check and split audio files by a cue for the given directory. + """ + # Walk directory and scan all media files + count = 0 + cue_count = 0 + cue_dirs = [] + + for root, dirs, files in os.walk(albumpath): + for _file in files: + extension = os.path.splitext(_file)[1].lower()[1:] + if extension in headphones.MEDIA_FORMATS: + count += 1 + elif extension == 'cue': + cue_count += 1 + if root not in cue_dirs: + cue_dirs.append(root) + + # Split cue + if cue_count and cue_count >= count and cue_dirs: + + from headphones import logger, cuesplit + logger.info("Attempting to split audio files by cue") + + cwd = os.getcwd() + for cue_dir in cue_dirs: + try: + cuesplit.split(cue_dir) + except Exception as e: + os.chdir(cwd) + logger.warn("Cue not split: " + str(e)) + return False + + os.chdir(cwd) + return True + + return False + + def extract_logline(s): # Default log format pattern = re.compile(r'(?P.*?)\s\-\s(?P.*?)\s*\:\:\s(?P.*?)\s\:\s(?P.*)', re.VERBOSE) @@ -457,14 +541,11 @@ def extract_logline(s): else: return None + def extract_song_data(s): + from headphones import logger #headphones default format - music_dir = headphones.MUSIC_DIR - folder_format = headphones.FOLDER_FORMAT - file_format = headphones.FILE_FORMAT - - full_format = os.path.join(headphones.MUSIC_DIR) pattern = re.compile(r'(?P.*?)\s\-\s(?P.*?)\s\[(?P.*?)\]', re.VERBOSE) match = pattern.match(s) @@ -488,6 +569,7 @@ def extract_song_data(s): logger.info("Couldn't parse %s into a valid Newbin format", s) return (name, album, year) + def smartMove(src, dest, delete=True): from headphones import logger @@ -509,7 +591,7 @@ def smartMove(src, dest, delete=True): try: os.rename(src, os.path.join(source_dir, newfile)) filename = newfile - except Exception, e: + except Exception as e: logger.warn('Error renaming %s: %s', src.decode(headphones.SYS_ENCODING, 'replace'), e) break @@ -519,7 +601,7 @@ def smartMove(src, dest, delete=True): else: shutil.copy(os.path.join(source_dir, filename), os.path.join(dest, filename)) return True - except Exception, e: + except Exception as e: logger.warn('Error moving file %s: %s', filename.decode(headphones.SYS_ENCODING, 'replace'), e) ######################### @@ -528,32 +610,36 @@ def smartMove(src, dest, delete=True): # TODO: Grab config values from sab to know when these options are checked. For now we'll just iterate through all combinations + def sab_replace_dots(name): - return name.replace('.',' ') + return name.replace('.', ' ') + + def sab_replace_spaces(name): - return name.replace(' ','_') + return name.replace(' ', '_') + def sab_sanitize_foldername(name): """ Return foldername with dodgy chars converted to safe ones Remove any leading and trailing dot and space characters """ CH_ILLEGAL = r'\/<>?*|"' - CH_LEGAL = r'++{}!@#`' + CH_LEGAL = r'++{}!@#`' FL_ILLEGAL = CH_ILLEGAL + ':\x92"' - FL_LEGAL = CH_LEGAL + "-''" + FL_LEGAL = CH_LEGAL + "-''" uFL_ILLEGAL = FL_ILLEGAL.decode('latin-1') - uFL_LEGAL = FL_LEGAL.decode('latin-1') + uFL_LEGAL = FL_LEGAL.decode('latin-1') if not name: return name if isinstance(name, unicode): illegal = uFL_ILLEGAL - legal = uFL_LEGAL + legal = uFL_LEGAL else: illegal = FL_ILLEGAL - legal = FL_LEGAL + legal = FL_LEGAL lst = [] for ch in name.strip(): @@ -574,12 +660,14 @@ def sab_sanitize_foldername(name): return name + def split_string(mystring, splitvar=','): mylist = [] for each_word in mystring.split(splitvar): mylist.append(each_word.strip()) return mylist + def create_https_certificates(ssl_cert, ssl_key): """ Stolen from SickBeard (http://github.com/midgetspy/Sick-Beard): @@ -589,7 +677,7 @@ def create_https_certificates(ssl_cert, ssl_key): try: from OpenSSL import crypto - from lib.certgen import createKeyPair, createCertRequest, createCertificate, TYPE_RSA, serial + from certgen import createKeyPair, createCertRequest, createCertificate, TYPE_RSA, serial except: logger.warn("pyOpenSSL module missing, please install to enable HTTPS") return False @@ -597,12 +685,12 @@ def create_https_certificates(ssl_cert, ssl_key): # Create the CA Certificate cakey = createKeyPair(TYPE_RSA, 1024) careq = createCertRequest(cakey, CN='Certificate Authority') - cacert = createCertificate(careq, (careq, cakey), serial, (0, 60*60*24*365*10)) # ten years + cacert = createCertificate(careq, (careq, cakey), serial, (0, 60 * 60 * 24 * 365 * 10)) # ten years cname = 'Headphones' pkey = createKeyPair(TYPE_RSA, 1024) req = createCertRequest(pkey, CN=cname) - cert = createCertificate(req, (cacert, cakey), serial, (0, 60*60*24*365*10)) # ten years + cert = createCertificate(req, (cacert, cakey), serial, (0, 60 * 60 * 24 * 365 * 10)) # ten years # Save the key and certificate to disk try: diff --git a/headphones/importer.py b/headphones/importer.py index 440c7a3c..758470ad 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -17,13 +17,11 @@ from headphones import logger, helpers, db, mb, lastfm from beets.mediafile import MediaFile -import os import time -import threading import headphones blacklisted_special_artist_names = ['[anonymous]', '[data]', '[no artist]', - '[traditional]','[unknown]','Various Artists'] + '[traditional]', '[unknown]', 'Various Artists'] blacklisted_special_artists = ['f731ccc4-e22a-43af-a747-64213329e088', '33cf029c-63b0-41a0-9855-be2a3665fb3b', '314e1c25-dde7-4e4d-b2f4-0a7b9f7c56dc', @@ -32,6 +30,7 @@ blacklisted_special_artists = ['f731ccc4-e22a-43af-a747-64213329e088', '125ec42a-7229-4250-afc5-e057484327fe', '89ad4ac3-39f7-470e-963a-56509c546377'] + def is_exists(artistid): myDB = db.DBConnection() @@ -52,7 +51,6 @@ def artistlist_to_mbids(artistlist, forced=False): if not artist and not (artist == ' '): continue - # If adding artists through Manage New Artists, they're coming through as non-unicode (utf-8?) # and screwing everything up if not isinstance(artist, unicode): @@ -105,12 +103,14 @@ def artistlist_to_mbids(artistlist, forced=False): except Exception as e: logger.warn('Failed to update arist information from Last.fm: %s' % e) + def addArtistIDListToDB(artistidlist): # Used to add a list of artist IDs to the database in a single thread logger.debug("Importer: Adding artist ids %s" % artistidlist) for artistid in artistidlist: addArtisttoDB(artistid) + def addArtisttoDB(artistid, extrasonly=False, forcefull=False): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums @@ -131,19 +131,19 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): # We need the current minimal info in the database instantly # so we don't throw a 500 error when we redirect to the artistPage - controlValueDict = {"ArtistID": artistid} + controlValueDict = {"ArtistID": artistid} # Don't replace a known artist name with an "Artist ID" placeholder dbartist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [artistid]).fetchone() # Only modify the Include Extras stuff if it's a new artist. We need it early so we know what to fetch if not dbartist: - newValueDict = {"ArtistName": "Artist ID: %s" % (artistid), - "Status": "Loading", - "IncludeExtras": headphones.INCLUDE_EXTRAS, - "Extras": headphones.EXTRAS } + newValueDict = {"ArtistName": "Artist ID: %s" % (artistid), + "Status": "Loading", + "IncludeExtras": headphones.CONFIG.INCLUDE_EXTRAS, + "Extras": headphones.CONFIG.EXTRAS} else: - newValueDict = {"Status": "Loading"} + newValueDict = {"Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) @@ -160,10 +160,10 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): if not artist: logger.warn("Error fetching artist info. ID: " + artistid) if dbartist is None: - newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), - "Status": "Active"} + newValueDict = {"ArtistName": "Fetch failed, try refreshing. (%s)" % (artistid), + "Status": "Active"} else: - newValueDict = {"Status": "Active"} + newValueDict = {"Status": "Active"} myDB.upsert("artists", newValueDict, controlValueDict) return @@ -172,13 +172,12 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): else: sortname = artist['artist_name'] - logger.info(u"Now adding/updating: " + artist['artist_name']) - controlValueDict = {"ArtistID": artistid} - newValueDict = {"ArtistName": artist['artist_name'], - "ArtistSortName": sortname, - "DateAdded": helpers.today(), - "Status": "Loading"} + controlValueDict = {"ArtistID": artistid} + newValueDict = {"ArtistName": artist['artist_name'], + "ArtistSortName": sortname, + "DateAdded": helpers.today(), + "Status": "Loading"} myDB.upsert("artists", newValueDict, controlValueDict) @@ -227,7 +226,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): rgid = rg['id'] skip_log = 0 #Make a user configurable variable to skip update of albums with release dates older than this date (in days) - pause_delta = headphones.MB_IGNORE_AGE + pause_delta = headphones.CONFIG.MB_IGNORE_AGE rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() @@ -240,27 +239,26 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): check_release_date = None new_release_group = True - if new_release_group: logger.info("[%s] Now adding: %s (New Release Group)" % (artist['artist_name'], rg['title'])) - new_releases = mb.get_new_releases(rgid,includeExtras) + new_releases = mb.get_new_releases(rgid, includeExtras) else: if check_release_date is None or check_release_date == u"None": logger.info("[%s] Now updating: %s (No Release Date)" % (artist['artist_name'], rg['title'])) - new_releases = mb.get_new_releases(rgid,includeExtras,True) + new_releases = mb.get_new_releases(rgid, includeExtras, True) else: if len(check_release_date) == 10: release_date = check_release_date elif len(check_release_date) == 7: - release_date = check_release_date+"-31" + release_date = check_release_date + "-31" elif len(check_release_date) == 4: - release_date = check_release_date+"-12-31" + release_date = check_release_date + "-12-31" else: release_date = today if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta: logger.info("[%s] Now updating: %s (Release Date <%s Days)", artist['artist_name'], rg['title'], pause_delta) - new_releases = mb.get_new_releases(rgid,includeExtras,True) + new_releases = mb.get_new_releases(rgid, includeExtras, True) else: logger.info("[%s] Skipping: %s (Release Date >%s Days)", artist['artist_name'], rg['title'], pause_delta) skip_log = 1 @@ -273,7 +271,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): new_releases = new_releases else: logger.info("[%s] Now adding/updating: %s (Comprehensive Force)", artist['artist_name'], rg['title']) - new_releases = mb.get_new_releases(rgid,includeExtras,forcefull) + new_releases = mb.get_new_releases(rgid, includeExtras, forcefull) if new_releases != 0: # Dump existing hybrid release since we're repackaging/replacing it @@ -292,26 +290,26 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): for items in find_hybrid_releases: if items['ReleaseID'] != rg['id']: #don't include hybrid information, since that's what we're replacing hybrid_release_id = items['ReleaseID'] - newValueDict = {"ArtistID": items['ArtistID'], - "ArtistName": items['ArtistName'], - "AlbumTitle": items['AlbumTitle'], - "AlbumID": items['AlbumID'], - "AlbumASIN": items['AlbumASIN'], - "ReleaseDate": items['ReleaseDate'], - "Type": items['Type'], - "ReleaseCountry": items['ReleaseCountry'], - "ReleaseFormat": items['ReleaseFormat'] + newValueDict = {"ArtistID": items['ArtistID'], + "ArtistName": items['ArtistName'], + "AlbumTitle": items['AlbumTitle'], + "AlbumID": items['AlbumID'], + "AlbumASIN": items['AlbumASIN'], + "ReleaseDate": items['ReleaseDate'], + "Type": items['Type'], + "ReleaseCountry": items['ReleaseCountry'], + "ReleaseFormat": items['ReleaseFormat'] } find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id]) totalTracks = 1 hybrid_track_array = [] for hybrid_tracks in find_hybrid_tracks: hybrid_track_array.append({ - 'number': hybrid_tracks['TrackNumber'], - 'title': hybrid_tracks['TrackTitle'], - 'id': hybrid_tracks['TrackID'], + 'number': hybrid_tracks['TrackNumber'], + 'title': hybrid_tracks['TrackTitle'], + 'id': hybrid_tracks['TrackID'], #'url': hybrid_tracks['TrackURL'], - 'duration': hybrid_tracks['TrackDuration'] + 'duration': hybrid_tracks['TrackDuration'] }) totalTracks += 1 newValueDict['ReleaseID'] = hybrid_release_id @@ -325,21 +323,21 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): logger.info('[%s] Packaging %s releases into hybrid title' % (artist['artist_name'], rg['title'])) except Exception as e: errors = True - logger.warn('[%s] Unable to get hybrid release information for %s: %s' % (artist['artist_name'],rg['title'],e)) + logger.warn('[%s] Unable to get hybrid release information for %s: %s' % (artist['artist_name'], rg['title'], e)) continue # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it # The hybrid won't have a country or a format - controlValueDict = {"ReleaseID": rg['id']} + controlValueDict = {"ReleaseID": rg['id']} - newValueDict = {"ArtistID": artistid, - "ArtistName": artist['artist_name'], - "AlbumTitle": rg['title'], - "AlbumID": rg['id'], - "AlbumASIN": hybridrelease['AlbumASIN'], - "ReleaseDate": hybridrelease['ReleaseDate'], - "Type": rg['type'] + newValueDict = {"ArtistID": artistid, + "ArtistName": artist['artist_name'], + "AlbumTitle": rg['title'], + "AlbumID": rg['id'], + "AlbumASIN": hybridrelease['AlbumASIN'], + "ReleaseDate": hybridrelease['ReleaseDate'], + "Type": rg['type'] } myDB.upsert("allalbums", newValueDict, controlValueDict) @@ -348,18 +346,18 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) - controlValueDict = {"TrackID": track['id'], - "ReleaseID": rg['id']} + controlValueDict = {"TrackID": track['id'], + "ReleaseID": rg['id']} - newValueDict = {"ArtistID": artistid, - "ArtistName": artist['artist_name'], - "AlbumTitle": rg['title'], - "AlbumASIN": hybridrelease['AlbumASIN'], - "AlbumID": rg['id'], - "TrackTitle": track['title'], - "TrackDuration": track['duration'], - "TrackNumber": track['number'], - "CleanName": cleanname + newValueDict = {"ArtistID": artistid, + "ArtistName": artist['artist_name'], + "AlbumTitle": rg['title'], + "AlbumASIN": hybridrelease['AlbumASIN'], + "AlbumID": rg['id'], + "TrackTitle": track['title'], + "TrackDuration": track['duration'], + "TrackNumber": track['number'], + "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() @@ -392,35 +390,35 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() - controlValueDict = {"AlbumID": rg['id']} + controlValueDict = {"AlbumID": rg['id']} - newValueDict = {"ArtistID": album['ArtistID'], - "ArtistName": album['ArtistName'], - "AlbumTitle": album['AlbumTitle'], - "ReleaseID": album['ReleaseID'], - "AlbumASIN": album['AlbumASIN'], - "ReleaseDate": album['ReleaseDate'], - "Type": album['Type'], - "ReleaseCountry": album['ReleaseCountry'], - "ReleaseFormat": album['ReleaseFormat'] + newValueDict = {"ArtistID": album['ArtistID'], + "ArtistName": album['ArtistName'], + "AlbumTitle": album['AlbumTitle'], + "ReleaseID": album['ReleaseID'], + "AlbumASIN": album['AlbumASIN'], + "ReleaseDate": album['ReleaseDate'], + "Type": album['Type'], + "ReleaseCountry": album['ReleaseCountry'], + "ReleaseFormat": album['ReleaseFormat'] } if rg_exists: newValueDict['DateAdded'] = rg_exists['DateAdded'] - newValueDict['Status'] = rg_exists['Status'] + newValueDict['Status'] = rg_exists['Status'] else: today = helpers.today() newValueDict['DateAdded'] = today - if headphones.AUTOWANT_ALL: + if headphones.CONFIG.AUTOWANT_ALL: newValueDict['Status'] = "Wanted" - elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING: + elif album['ReleaseDate'] > today and headphones.CONFIG.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these # The first test just makes sure we have year-month-day - elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING: + elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.CONFIG.AUTOWANT_UPCOMING: newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" @@ -440,21 +438,21 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): continue for track in tracks: - controlValueDict = {"TrackID": track['TrackID'], - "AlbumID": rg['id']} + controlValueDict = {"TrackID": track['TrackID'], + "AlbumID": rg['id']} - newValueDict = {"ArtistID": track['ArtistID'], - "ArtistName": track['ArtistName'], - "AlbumTitle": track['AlbumTitle'], - "AlbumASIN": track['AlbumASIN'], - "ReleaseID": track['ReleaseID'], - "TrackTitle": track['TrackTitle'], - "TrackDuration": track['TrackDuration'], - "TrackNumber": track['TrackNumber'], - "CleanName": track['CleanName'], - "Location": track['Location'], - "Format": track['Format'], - "BitRate": track['BitRate'] + newValueDict = {"ArtistID": track['ArtistID'], + "ArtistName": track['ArtistName'], + "AlbumTitle": track['AlbumTitle'], + "AlbumASIN": track['AlbumASIN'], + "ReleaseID": track['ReleaseID'], + "TrackTitle": track['TrackTitle'], + "TrackDuration": track['TrackDuration'], + "TrackNumber": track['TrackNumber'], + "CleanName": track['CleanName'], + "Location": track['Location'], + "Format": track['Format'], + "BitRate": track['BitRate'] } myDB.upsert("tracks", newValueDict, controlValueDict) @@ -464,11 +462,11 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): marked_as_downloaded = False if rg_exists: - if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): + if rg_exists['Status'] == 'Skipped' and ((have_track_count / float(total_track_count)) >= (headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True else: - if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): + if ((have_track_count / float(total_track_count)) >= (headphones.CONFIG.ALBUM_COMPLETION_PCT / 100.0)): myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) marked_as_downloaded = True @@ -478,7 +476,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): # Start a search for the album if it's new, hasn't been marked as # downloaded and autowant_all is selected. This search is deferred, # in case the search failes and the rest of the import will halt. - if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL: + if not rg_exists and not marked_as_downloaded and headphones.CONFIG.AUTOWANT_ALL: album_searches.append(rg['id']) else: if skip_log == 0: @@ -504,6 +502,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): for album_search in album_searches: searcher.searchforalbum(albumid=album_search) + def finalize_update(artistid, artistname, errors=False): # Moving this little bit to it's own function so we can update have tracks & latest album when deleting extras @@ -514,25 +513,26 @@ def finalize_update(artistid, artistname, errors=False): #havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']])) havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artistname])) - controlValueDict = {"ArtistID": artistid} + controlValueDict = {"ArtistID": artistid} if latestalbum: - newValueDict = {"Status": "Active", - "LatestAlbum": latestalbum['AlbumTitle'], - "ReleaseDate": latestalbum['ReleaseDate'], - "AlbumID": latestalbum['AlbumID'], - "TotalTracks": totaltracks, - "HaveTracks": havetracks} + newValueDict = {"Status": "Active", + "LatestAlbum": latestalbum['AlbumTitle'], + "ReleaseDate": latestalbum['ReleaseDate'], + "AlbumID": latestalbum['AlbumID'], + "TotalTracks": totaltracks, + "HaveTracks": havetracks} else: - newValueDict = {"Status": "Active", - "TotalTracks": totaltracks, - "HaveTracks": havetracks} + newValueDict = {"Status": "Active", + "TotalTracks": totaltracks, + "HaveTracks": havetracks} if not errors: newValueDict['LastUpdated'] = helpers.now() myDB.upsert("artists", newValueDict, controlValueDict) + def addReleaseById(rid, rgid=None): myDB = db.DBConnection() @@ -543,10 +543,10 @@ def addReleaseById(rid, rgid=None): dbalbum = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid]) if not dbalbum: status = 'Loading' - controlValueDict = {"AlbumID": rgid} - newValueDict = {"AlbumTitle": rgid, - "ArtistName": status, - "Status": status} + controlValueDict = {"AlbumID": rgid} + newValueDict = {"AlbumTitle": rgid, + "ArtistName": status, + "Status": status} myDB.upsert("albums", newValueDict, controlValueDict) time.sleep(1) @@ -590,15 +590,15 @@ def addReleaseById(rid, rgid=None): sortname = release_dict['artist_name'] logger.info(u"Now manually adding: " + release_dict['artist_name'] + " - with status Paused") - controlValueDict = {"ArtistID": release_dict['artist_id']} - newValueDict = {"ArtistName": release_dict['artist_name'], - "ArtistSortName": sortname, - "DateAdded": helpers.today(), - "Status": "Paused"} + controlValueDict = {"ArtistID": release_dict['artist_id']} + newValueDict = {"ArtistName": release_dict['artist_name'], + "ArtistSortName": sortname, + "DateAdded": helpers.today(), + "Status": "Paused"} - if headphones.INCLUDE_EXTRAS: + if headphones.CONFIG.INCLUDE_EXTRAS: newValueDict['IncludeExtras'] = 1 - newValueDict['Extras'] = headphones.EXTRAS + newValueDict['Extras'] = headphones.CONFIG.EXTRAS myDB.upsert("artists", newValueDict, controlValueDict) @@ -611,20 +611,20 @@ def addReleaseById(rid, rgid=None): if not rg_exists and release_dict or status == 'Loading' and release_dict: #it should never be the case that we have an rg and not the artist #but if it is this will fail logger.info(u"Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid) - controlValueDict = {"AlbumID": rgid} + controlValueDict = {"AlbumID": rgid} if status != 'Loading': status = 'Wanted' - newValueDict = {"ArtistID": release_dict['artist_id'], - "ReleaseID": rgid, - "ArtistName": release_dict['artist_name'], - "AlbumTitle": release_dict['rg_title'], - "AlbumASIN": release_dict['asin'], - "ReleaseDate": release_dict['date'], - "DateAdded": helpers.today(), - "Status": status, - "Type": release_dict['rg_type'], - "ReleaseID": rid + newValueDict = {"ArtistID": release_dict['artist_id'], + "ReleaseID": rgid, + "ArtistName": release_dict['artist_name'], + "AlbumTitle": release_dict['title'] if 'title' in release_dict else release_dict['rg_title'], + "AlbumASIN": release_dict['asin'], + "ReleaseDate": release_dict['date'], + "DateAdded": helpers.today(), + "Status": status, + "Type": release_dict['rg_type'], + "ReleaseID": rid } myDB.upsert("albums", newValueDict, controlValueDict) @@ -635,16 +635,16 @@ def addReleaseById(rid, rgid=None): for track in release_dict['tracks']: cleanname = helpers.cleanName(release_dict['artist_name'] + ' ' + release_dict['rg_title'] + ' ' + track['title']) - controlValueDict = {"TrackID": track['id'], - "AlbumID": rgid} - newValueDict = {"ArtistID": release_dict['artist_id'], - "ArtistName": release_dict['artist_name'], - "AlbumTitle": release_dict['rg_title'], - "AlbumASIN": release_dict['asin'], - "TrackTitle": track['title'], - "TrackDuration": track['duration'], - "TrackNumber": track['number'], - "CleanName": cleanname + controlValueDict = {"TrackID": track['id'], + "AlbumID": rgid} + newValueDict = {"ArtistID": release_dict['artist_id'], + "ArtistName": release_dict['artist_name'], + "AlbumTitle": release_dict['rg_title'], + "AlbumASIN": release_dict['asin'], + "TrackTitle": track['title'], + "TrackDuration": track['duration'], + "TrackNumber": track['number'], + "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format, Matched from have WHERE CleanName=?', [cleanname]).fetchone() @@ -669,15 +669,15 @@ def addReleaseById(rid, rgid=None): # Reset status if status == 'Loading': - controlValueDict = {"AlbumID": rgid} - if headphones.AUTOWANT_MANUALLY_ADDED: - newValueDict = {"Status": "Wanted"} + controlValueDict = {"AlbumID": rgid} + if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED: + newValueDict = {"Status": "Wanted"} else: - newValueDict = {"Status": "Skipped"} + newValueDict = {"Status": "Skipped"} myDB.upsert("albums", newValueDict, controlValueDict) # Start a search for the album - if headphones.AUTOWANT_MANUALLY_ADDED: + if headphones.CONFIG.AUTOWANT_MANUALLY_ADDED: import searcher searcher.searchforalbum(rgid, False) @@ -689,6 +689,7 @@ def addReleaseById(rid, rgid=None): else: logger.info('Release ' + str(rid) + " already exists in the database!") + def updateFormat(): myDB = db.DBConnection() tracks = myDB.select('SELECT * from tracks WHERE Location IS NOT NULL and Format IS NULL') @@ -697,10 +698,10 @@ def updateFormat(): for track in tracks: try: f = MediaFile(track['Location']) - except Exception, e: + except Exception as e: logger.info("Exception from MediaFile for: " + track['Location'] + " : " + str(e)) continue - controlValueDict = {"TrackID": track['TrackID']} + controlValueDict = {"TrackID": track['TrackID']} newValueDict = {"Format": f.format} myDB.upsert("tracks", newValueDict, controlValueDict) logger.info('Finished finding media format for %s files' % len(tracks)) @@ -710,14 +711,15 @@ def updateFormat(): for track in havetracks: try: f = MediaFile(track['Location']) - except Exception, e: + except Exception as e: logger.info("Exception from MediaFile for: " + track['Location'] + " : " + str(e)) continue - controlValueDict = {"TrackID": track['TrackID']} + controlValueDict = {"TrackID": track['TrackID']} newValueDict = {"Format": f.format} myDB.upsert("have", newValueDict, controlValueDict) logger.info('Finished finding media format for %s files' % len(havetracks)) + def getHybridRelease(fullreleaselist): """ Returns a dictionary of best group of tracks from the list of releases and @@ -730,18 +732,18 @@ def getHybridRelease(fullreleaselist): sortable_release_list = [] formats = { - '2xVinyl': '2', - 'Vinyl': '2', - 'CD': '0', - 'Cassette': '3', - '2xCD': '1', - 'Digital Media': '0' + '2xVinyl': '2', + 'Vinyl': '2', + 'CD': '0', + 'Cassette': '3', + '2xCD': '1', + 'Digital Media': '0' } countries = { - 'US': '0', - 'GB': '1', - 'JP': '2', + 'US': '0', + 'GB': '1', + 'JP': '2', } for release in fullreleaselist: @@ -758,14 +760,14 @@ def getHybridRelease(fullreleaselist): # Create record release_dict = { - 'hasasin': bool(release['AlbumASIN']), - 'asin': release['AlbumASIN'], - 'trackscount': len(release['Tracks']), - 'releaseid': release['ReleaseID'], - 'releasedate': release['ReleaseDate'], - 'format': format, - 'country': country, - 'tracks': release['Tracks'] + 'hasasin': bool(release['AlbumASIN']), + 'asin': release['AlbumASIN'], + 'trackscount': len(release['Tracks']), + 'releaseid': release['ReleaseID'], + 'releasedate': release['ReleaseDate'], + 'format': format, + 'country': country, + 'tracks': release['Tracks'] } sortable_release_list.append(release_dict) @@ -776,8 +778,8 @@ def getHybridRelease(fullreleaselist): # Change this value to change the sorting behaviour of none, returning # 'None' will put it at the top which was normal behaviour for pre-ngs # versions - if releaseDate == None: - return 'None'; + if releaseDate is None: + return 'None' if releaseDate.count('-') == 2: return releaseDate @@ -786,7 +788,7 @@ def getHybridRelease(fullreleaselist): else: return releaseDate + '13-32' - sortable_release_list.sort(key=lambda x:getSortableReleaseDate(x['releasedate'])) + sortable_release_list.sort(key=lambda x: getSortableReleaseDate(x['releasedate'])) average_tracks = sum(x['trackscount'] for x in sortable_release_list) / float(len(sortable_release_list)) for item in sortable_release_list: @@ -794,9 +796,9 @@ def getHybridRelease(fullreleaselist): a = helpers.multikeysort(sortable_release_list, ['-hasasin', 'country', 'format', 'trackscount_delta']) - release_dict = {'ReleaseDate' : sortable_release_list[0]['releasedate'], - 'Tracks' : a[0]['tracks'], - 'AlbumASIN' : a[0]['asin'] + release_dict = {'ReleaseDate': sortable_release_list[0]['releasedate'], + 'Tracks': a[0]['tracks'], + 'AlbumASIN': a[0]['asin'] } return release_dict diff --git a/headphones/lastfm.py b/headphones/lastfm.py index 0b72adf6..c97c019e 100644 --- a/headphones/lastfm.py +++ b/headphones/lastfm.py @@ -30,6 +30,7 @@ API_KEY = "395e6ec6bb557382fc41fde867bce66f" # Required for API request limit lock = threading.Lock() + def request_lastfm(method, **kwargs): """ Call a Last.FM API method. Automatically sets the method and API key. Method @@ -62,6 +63,7 @@ def request_lastfm(method, **kwargs): return data + def getSimilar(): myDB = db.DBConnection() results = myDB.select("SELECT ArtistID from artists ORDER BY HaveTracks DESC") @@ -107,16 +109,17 @@ def getSimilar(): logger.debug("Inserted %d artists into Last.FM tag cloud", len(top_list)) + def getArtists(): myDB = db.DBConnection() results = myDB.select("SELECT ArtistID from artists") - if not headphones.LASTFM_USERNAME: + if not headphones.CONFIG.LASTFM_USERNAME: logger.warn("Last.FM username not set, not importing artists.") return - logger.info("Fetching artists from Last.FM for username: %s", headphones.LASTFM_USERNAME) - data = request_lastfm("library.getartists", limit=10000, user=headphones.LASTFM_USERNAME) + logger.info("Fetching artists from Last.FM for username: %s", headphones.CONFIG.LASTFM_USERNAME) + data = request_lastfm("library.getartists", limit=10000, user=headphones.CONFIG.LASTFM_USERNAME) if data and "artists" in data: artistlist = [] @@ -136,6 +139,7 @@ def getArtists(): logger.info("Imported %d new artists from Last.FM", len(artistlist)) + def getTagTopArtists(tag, limit=50): myDB = db.DBConnection() results = myDB.select("SELECT ArtistID from artists") @@ -159,4 +163,4 @@ def getTagTopArtists(tag, limit=50): for artistid in artistlist: importer.addArtisttoDB(artistid) - logger.debug("Added %d new artists from Last.FM", len(artistlist)) \ No newline at end of file + logger.debug("Added %d new artists from Last.FM", len(artistlist)) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index e8a126cc..0b49f563 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -14,7 +14,6 @@ # along with Headphones. If not, see . import os -import glob import headphones from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError @@ -22,17 +21,18 @@ from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError from headphones import db, logger, helpers, importer, lastfm # You can scan a single directory and append it to the current library by specifying append=True, ArtistID & ArtistName + + def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=False): - - if cron and not headphones.LIBRARYSCAN: + if cron and not headphones.CONFIG.LIBRARYSCAN: return if not dir: - if not headphones.MUSIC_DIR: + if not headphones.CONFIG.MUSIC_DIR: return else: - dir = headphones.MUSIC_DIR + dir = headphones.CONFIG.MUSIC_DIR # If we're appending a dir, it's coming from the post processor which is # already bytestring @@ -78,9 +78,11 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal latest_subdirectory = [] - for r,d,f in os.walk(dir): - #need to abuse slicing to get a copy of the list, doing it directly will skip the element after a deleted one - #using a list comprehension will not work correctly for nested subdirectories (os.walk keeps its original list) + for r, d, f in os.walk(dir, followlinks=True): + # Need to abuse slicing to get a copy of the list, doing it directly + # will skip the element after a deleted one using a list comprehension + # will not work correctly for nested subdirectories (os.walk keeps its + # original list) for directory in d[:]: if directory.startswith("."): d.remove(directory) @@ -89,11 +91,11 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): - subdirectory = r.replace(dir,'') + subdirectory = r.replace(dir, '') latest_subdirectory.append(subdirectory) - if file_count == 0 and r.replace(dir,'') !='': + if file_count == 0 and r.replace(dir, '') != '': logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) - elif latest_subdirectory[file_count] != latest_subdirectory[file_count-1] and file_count !=0: + elif latest_subdirectory[file_count] != latest_subdirectory[file_count - 1] and file_count != 0: logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) song = os.path.join(r, files) @@ -107,7 +109,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal except (FileTypeError, UnreadableFileError): logger.warning("Cannot read media file '%s', skipping. It may be corrupted or not a media file.", unicode_song_path) continue - except IOError as e: + except IOError: logger.warning("Cannnot read media file '%s', skipping. Does the file exists?", unicode_song_path) continue @@ -127,24 +129,24 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) if f_artist and f.album and f.title: - CleanName = helpers.cleanName(f_artist +' '+ f.album +' '+ f.title) + CleanName = helpers.cleanName(f_artist + ' ' + f.album + ' ' + f.title) else: CleanName = None - controlValueDict = {'Location' : unicode_song_path} + controlValueDict = {'Location': unicode_song_path} - newValueDict = { 'TrackID' : f.mb_trackid, + newValueDict = {'TrackID': f.mb_trackid, #'ReleaseID' : f.mb_albumid, - 'ArtistName' : f_artist, - 'AlbumTitle' : f.album, + 'ArtistName': f_artist, + 'AlbumTitle': f.album, 'TrackNumber': f.track, 'TrackLength': f.length, - 'Genre' : f.genre, - 'Date' : f.date, - 'TrackTitle' : f.title, - 'BitRate' : f.bitrate, - 'Format' : f.format, - 'CleanName' : CleanName + 'Genre': f.genre, + 'Date': f.date, + 'TrackTitle': f.title, + 'BitRate': f.bitrate, + 'Format': f.format, + 'CleanName': CleanName } #song_list.append(song_dict) @@ -155,7 +157,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal if f_artist: new_artists.append(f_artist) myDB.upsert("have", newValueDict, controlValueDict) - new_song_count+=1 + new_song_count += 1 else: if check_exist_song['ArtistName'] != f_artist or check_exist_song['AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title: #Important track metadata has been modified, need to run matcher again @@ -170,19 +172,18 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal myDB.upsert("have", newValueDict, controlValueDict) myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) - new_song_count+=1 + new_song_count += 1 else: #This track information hasn't changed if f_artist and check_exist_song['Matched'] != "Ignored": new_artists.append(f_artist) - file_count+=1 - + file_count += 1 # Now we start track matching logger.info("%s new/modified songs found and added to the database" % new_song_count) - song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace')+"%"]) - total_number_of_songs = myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace')+"%"]).fetchone()[0] + song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]) + total_number_of_songs = myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir.decode(headphones.SYS_ENCODING, 'replace') + "%"]).fetchone()[0] logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) @@ -200,14 +201,13 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal latest_artist.append(song['ArtistName']) if song_count == 0: logger.info("Now matching songs by %s" % song['ArtistName']) - elif latest_artist[song_count] != latest_artist[song_count-1] and song_count !=0: + elif latest_artist[song_count] != latest_artist[song_count - 1] and song_count != 0: logger.info("Now matching songs by %s" % song['ArtistName']) - #print song['ArtistName']+' - '+song['AlbumTitle']+' - '+song['TrackTitle'] song_count += 1 - completion_percentage = float(song_count)/total_number_of_songs * 100 + completion_percentage = float(song_count) / total_number_of_songs * 100 - if completion_percentage%10 == 0: + if completion_percentage % 10 == 0: logger.info("Track matching is " + str(completion_percentage) + "% complete") #THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED. WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT @@ -220,79 +220,78 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() have_updated = False if track: - controlValueDict = { 'ArtistName' : track['ArtistName'], - 'AlbumTitle' : track['AlbumTitle'], - 'TrackTitle' : track['TrackTitle'] } - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } + controlValueDict = {'ArtistName': track['ArtistName'], + 'AlbumTitle': track['AlbumTitle'], + 'TrackTitle': track['TrackTitle']} + newValueDict = {'Location': song['Location'], + 'BitRate': song['BitRate'], + 'Format': song['Format']} myDB.upsert("tracks", newValueDict, controlValueDict) - controlValueDict2 = { 'Location' : song['Location']} - newValueDict2 = { 'Matched' : track['AlbumID']} + controlValueDict2 = {'Location': song['Location']} + newValueDict2 = {'Matched': track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) have_updated = True else: track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() if track: - controlValueDict = { 'CleanName' : track['CleanName']} - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } + controlValueDict = {'CleanName': track['CleanName']} + newValueDict = {'Location': song['Location'], + 'BitRate': song['BitRate'], + 'Format': song['Format']} myDB.upsert("tracks", newValueDict, controlValueDict) - controlValueDict2 = { 'Location' : song['Location']} - newValueDict2 = { 'Matched' : track['AlbumID']} + controlValueDict2 = {'Location': song['Location']} + newValueDict2 = {'Matched': track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) have_updated = True else: - controlValueDict2 = { 'Location' : song['Location']} - newValueDict2 = { 'Matched' : "Failed"} + controlValueDict2 = {'Location': song['Location']} + newValueDict2 = {'Matched': "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) have_updated = True alltrack = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if alltrack: - controlValueDict = { 'ArtistName' : alltrack['ArtistName'], - 'AlbumTitle' : alltrack['AlbumTitle'], - 'TrackTitle' : alltrack['TrackTitle'] } - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } + controlValueDict = {'ArtistName': alltrack['ArtistName'], + 'AlbumTitle': alltrack['AlbumTitle'], + 'TrackTitle': alltrack['TrackTitle']} + newValueDict = {'Location': song['Location'], + 'BitRate': song['BitRate'], + 'Format': song['Format']} myDB.upsert("alltracks", newValueDict, controlValueDict) - controlValueDict2 = { 'Location' : song['Location']} - newValueDict2 = { 'Matched' : alltrack['AlbumID']} + controlValueDict2 = {'Location': song['Location']} + newValueDict2 = {'Matched': alltrack['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: alltrack = myDB.action('SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() if alltrack: - controlValueDict = { 'CleanName' : alltrack['CleanName']} - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } + controlValueDict = {'CleanName': alltrack['CleanName']} + newValueDict = {'Location': song['Location'], + 'BitRate': song['BitRate'], + 'Format': song['Format']} myDB.upsert("alltracks", newValueDict, controlValueDict) - controlValueDict2 = { 'Location' : song['Location']} - newValueDict2 = { 'Matched' : alltrack['AlbumID']} + controlValueDict2 = {'Location': song['Location']} + newValueDict2 = {'Matched': alltrack['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: # alltracks may not exist if adding album manually, have should only be set to failed if not already updated in tracks if not have_updated: - controlValueDict2 = { 'Location' : song['Location']} - newValueDict2 = { 'Matched' : "Failed"} + controlValueDict2 = {'Location': song['Location']} + newValueDict2 = {'Matched': "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) else: - controlValueDict2 = { 'Location' : song['Location']} - newValueDict2 = { 'Matched' : "Failed"} + controlValueDict2 = {'Location': song['Location']} + newValueDict2 = {'Matched': "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) #######myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) - if not append: logger.info('Updating scanned artist track counts') @@ -301,15 +300,30 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') #There was a bug where artists with special characters (-,') would show up in new artists. - artist_list = [f for f in unique_artists if helpers.cleanName(f).lower() not in [helpers.cleanName(x[0]).lower() for x in current_artists]] - artists_checked = [f for f in unique_artists if helpers.cleanName(f).lower() in [helpers.cleanName(x[0]).lower() for x in current_artists]] + artist_list = [ + x for x in unique_artists + if helpers.cleanName(x).lower() not in [ + helpers.cleanName(y[0]).lower() + for y in current_artists + ] + ] + artists_checked = [ + x for x in unique_artists + if helpers.cleanName(x).lower() in [ + helpers.cleanName(y[0]).lower() + for y in current_artists + ] + ] # Update track counts for artist in artists_checked: # Have tracks are selected from tracks table and not all tracks because of duplicates # We update the track count upon an album switch to compliment this - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL', [artist])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artist])) + havetracks = ( + len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL', [artist])) + + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artist])) + ) #Note, some people complain about having "artist have tracks" > # of tracks total in artist official releases # (can fix by getting rid of second len statement) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist]) @@ -317,7 +331,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal logger.info('Found %i new artists' % len(artist_list)) if len(artist_list): - if headphones.ADD_ARTISTS: + if headphones.CONFIG.AUTO_ADD_ARTISTS: logger.info('Importing %i new artists' % len(artist_list)) importer.artistlist_to_mbids(artist_list) else: @@ -326,8 +340,8 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal for artist in artist_list: myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist]) - if headphones.DETECT_BITRATE: - headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000 + if headphones.CONFIG.DETECT_BITRATE: + headphones.CONFIG.PREFERRED_BITRATE = sum(bitrates) / len(bitrates) / 1000 else: # If we're appending a new album to the database, update the artists total track counts @@ -342,6 +356,8 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal logger.info('Library scan complete') #ADDED THIS SECTION TO MARK ALBUMS AS DOWNLOADED IF ARTISTS ARE ADDED EN MASSE BEFORE LIBRARY IS SCANNED + + def update_album_status(AlbumID=None): myDB = db.DBConnection() logger.info('Counting matched tracks to mark albums as skipped/downloaded') @@ -354,16 +370,16 @@ def update_album_status(AlbumID=None): total_tracks = 0 have_tracks = 0 for track in track_counter: - total_tracks+=1 + total_tracks += 1 if track['Location']: - have_tracks+=1 + have_tracks += 1 if total_tracks != 0: album_completion = float(float(have_tracks) / float(total_tracks)) * 100 else: album_completion = 0 logger.info('Album %s does not have any tracks in database' % album['AlbumTitle']) - if album_completion >= headphones.ALBUM_COMPLETION_PCT and album['Status'] == 'Skipped': + if album_completion >= headphones.CONFIG.ALBUM_COMPLETION_PCT and album['Status'] == 'Skipped': new_album_status = "Downloaded" # I don't think we want to change Downloaded->Skipped..... @@ -378,7 +394,7 @@ def update_album_status(AlbumID=None): else: new_album_status = album['Status'] - myDB.upsert("albums", {'Status' : new_album_status}, {'AlbumID' : album['AlbumID']}) + myDB.upsert("albums", {'Status': new_album_status}, {'AlbumID': album['AlbumID']}) if new_album_status != album['Status']: logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status)) logger.info('Album status update complete') diff --git a/headphones/logger.py b/headphones/logger.py index 5c22097d..781d5402 100644 --- a/headphones/logger.py +++ b/headphones/logger.py @@ -39,6 +39,7 @@ logger = logging.getLogger("headphones") # Global queue for multiprocessing logging queue = None + class LogListHandler(logging.Handler): """ Log handler for Web UI. @@ -50,6 +51,7 @@ class LogListHandler(logging.Handler): headphones.LOG_LIST.insert(0, (helpers.now(), message, record.levelname, record.threadName)) + @contextlib.contextmanager def listener(): """ @@ -85,6 +87,7 @@ def listener(): finally: queue_listener.stop() + def initMultiprocessing(): """ Remove all handlers and add QueueHandler on top. This should only be called @@ -108,6 +111,7 @@ def initMultiprocessing(): # Change current thread name for log record threading.current_thread().name = multiprocessing.current_process().name + def initLogger(console=False, verbose=False): """ Setup logging for Headphones. It uses the logger instance with the name @@ -136,7 +140,7 @@ def initLogger(console=False, verbose=False): logger.setLevel(logging.DEBUG if verbose else logging.INFO) # Setup file logger - filename = os.path.join(headphones.LOG_DIR, FILENAME) + filename = os.path.join(headphones.CONFIG.LOG_DIR, FILENAME) file_formatter = logging.Formatter('%(asctime)s - %(levelname)-7s :: %(threadName)s : %(message)s', '%d-%b-%Y %H:%M:%S') file_handler = handlers.RotatingFileHandler(filename, maxBytes=MAX_SIZE, backupCount=MAX_FILES) @@ -163,6 +167,7 @@ def initLogger(console=False, verbose=False): # Install exception hooks initHooks() + def initHooks(global_exceptions=True, thread_exceptions=True, pass_original=True): """ This method installs exception catching mechanisms. Any exception caught @@ -217,4 +222,4 @@ warn = logger.warn error = logger.error debug = logger.debug warning = logger.warning -exception = logger.exception \ No newline at end of file +exception = logger.exception diff --git a/headphones/lyrics.py b/headphones/lyrics.py index ea8458ee..066b97c5 100644 --- a/headphones/lyrics.py +++ b/headphones/lyrics.py @@ -18,9 +18,10 @@ import htmlentitydefs from headphones import logger, request + def getLyrics(artist, song): - params = { "artist": artist.encode('utf-8'), + params = {"artist": artist.encode('utf-8'), "song": song.encode('utf-8'), "fmt": 'xml' } @@ -60,6 +61,7 @@ def getLyrics(artist, song): return lyrics + def convert_html_entities(s): matches = re.findall("&#\d+;", s) if len(matches) > 0: @@ -79,7 +81,7 @@ def convert_html_entities(s): hits.remove(amp) for hit in hits: name = hit[1:-1] - if htmlentitydefs.name2codepoint.has_key(name): - s = s.replace(hit, unichr(htmlentitydefs.name2codepoint[name])) + if name in htmlentitydefs.name2codepoint: + s = s.replace(hit, unichr(htmlentitydefs.name2codepoint[name])) s = s.replace(amp, "&") return s diff --git a/headphones/mb.py b/headphones/mb.py index 2c308ec7..fc0bc8d0 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -15,7 +15,6 @@ from headphones import logger, db, helpers -from headphones.helpers import multikeysort, replace_all import time import threading @@ -23,7 +22,10 @@ import headphones import musicbrainzngs try: + # pylint:disable=E0611 + # ignore this error because we are catching the ImportError from collections import OrderedDict + # pylint:enable=E0611 except ImportError: # Python 2.6.x fallback, from libs from ordereddict import OrderedDict @@ -32,29 +34,31 @@ mb_lock = threading.Lock() # Quick fix to add mirror switching on the fly. Need to probably return the mbhost & mbport that's # being used, so we can send those values to the log + + def startmb(): mbuser = None mbpass = None - if headphones.MIRROR == "musicbrainz.org": + if headphones.CONFIG.MIRROR == "musicbrainz.org": mbhost = "musicbrainz.org" mbport = 80 sleepytime = 1 - elif headphones.MIRROR == "custom": - mbhost = headphones.CUSTOMHOST - mbport = int(headphones.CUSTOMPORT) - sleepytime = int(headphones.CUSTOMSLEEP) - elif headphones.MIRROR == "headphones": + elif headphones.CONFIG.MIRROR == "custom": + mbhost = headphones.CONFIG.CUSTOMHOST + mbport = int(headphones.CONFIG.CUSTOMPORT) + sleepytime = int(headphones.CONFIG.CUSTOMSLEEP) + elif headphones.CONFIG.MIRROR == "headphones": mbhost = "144.76.94.239" mbport = 8181 - mbuser = headphones.HPUSER - mbpass = headphones.HPPASS + mbuser = headphones.CONFIG.HPUSER + mbpass = headphones.CONFIG.HPPASS sleepytime = 0 else: return False - musicbrainzngs.set_useragent("headphones","0.0","https://github.com/rembo10/headphones") + musicbrainzngs.set_useragent("headphones", "0.0", "https://github.com/rembo10/headphones") musicbrainzngs.set_hostname(mbhost + ":" + str(mbport)) if sleepytime == 0: musicbrainzngs.set_rate_limit(False) @@ -63,16 +67,17 @@ def startmb(): musicbrainzngs.set_rate_limit(limit_or_interval=float(sleepytime)) # Add headphones credentials - if headphones.MIRROR == "headphones": + if headphones.CONFIG.MIRROR == "headphones": if not mbuser and mbpass: logger.warn("No username or password set for VIP server") else: - musicbrainzngs.hpauth(mbuser,mbpass) + musicbrainzngs.hpauth(mbuser, mbpass) logger.debug('Using the following server values: MBHost: %s, MBPort: %i, Sleep Interval: %i', mbhost, mbport, sleepytime) return True + def findArtist(name, limit=1): with mb_lock: @@ -81,7 +86,7 @@ def findArtist(name, limit=1): chars = set('!?*-') if any((c in chars) for c in name): - name = '"'+name+'"' + name = '"' + name + '"' criteria = {'artist': name.lower()} @@ -107,7 +112,7 @@ def findArtist(name, limit=1): # Just need the artist id if the limit is 1 # 'name': unicode(result['sort-name']), # 'uniquename': uniquename, - 'id': unicode(result['id']), + 'id': unicode(result['id']), # 'url': unicode("http://musicbrainz.org/artist/" + result['id']),#probably needs to be changed # 'score': int(result['ext:score']) }) @@ -115,14 +120,15 @@ def findArtist(name, limit=1): artistlist.append(artistdict) else: artistlist.append({ - 'name': unicode(result['sort-name']), - 'uniquename': uniquename, - 'id': unicode(result['id']), - 'url': unicode("http://musicbrainz.org/artist/" + result['id']),#probably needs to be changed - 'score': int(result['ext:score']) + 'name': unicode(result['sort-name']), + 'uniquename': uniquename, + 'id': unicode(result['id']), + 'url': unicode("http://musicbrainz.org/artist/" + result['id']),#probably needs to be changed + 'score': int(result['ext:score']) }) return artistlist + def findRelease(name, limit=1, artist=None): with mb_lock: @@ -131,16 +137,16 @@ def findRelease(name, limit=1, artist=None): # additional artist search if not artist and ':' in name: - name, artist = name.rsplit(":",1) + name, artist = name.rsplit(":", 1) chars = set('!?*-') if any((c in chars) for c in name): - name = '"'+name+'"' + name = '"' + name + '"' if artist and any((c in chars) for c in artist): - artist = '"'+artist+'"' + artist = '"' + artist + '"' try: - releaseResults = musicbrainzngs.search_releases(query=name,limit=limit,artist=artist)['release-list'] + releaseResults = musicbrainzngs.search_releases(query=name, limit=limit, artist=artist)['release-list'] except musicbrainzngs.WebServiceError as e: #need to update exceptions logger.warn('Attempt to query MusicBrainz for "%s" failed: %s' % (name, str(e))) time.sleep(5) @@ -185,22 +191,23 @@ def findRelease(name, limit=1, artist=None): rg_type = secondary_type releaselist.append({ - 'uniquename': unicode(result['artist-credit'][0]['artist']['name']), - 'title': unicode(title), - 'id': unicode(result['artist-credit'][0]['artist']['id']), - 'albumid': unicode(result['id']), - 'url': unicode("http://musicbrainz.org/artist/" + result['artist-credit'][0]['artist']['id']),#probably needs to be changed - 'albumurl': unicode("http://musicbrainz.org/release/" + result['id']),#probably needs to be changed - 'score': int(result['ext:score']), - 'date': unicode(result['date']) if 'date' in result else '', - 'country': unicode(result['country']) if 'country' in result else '', - 'formats': unicode(formats), - 'tracks': unicode(tracks), - 'rgid': unicode(result['release-group']['id']), - 'rgtype': unicode(rg_type) + 'uniquename': unicode(result['artist-credit'][0]['artist']['name']), + 'title': unicode(title), + 'id': unicode(result['artist-credit'][0]['artist']['id']), + 'albumid': unicode(result['id']), + 'url': unicode("http://musicbrainz.org/artist/" + result['artist-credit'][0]['artist']['id']),#probably needs to be changed + 'albumurl': unicode("http://musicbrainz.org/release/" + result['id']),#probably needs to be changed + 'score': int(result['ext:score']), + 'date': unicode(result['date']) if 'date' in result else '', + 'country': unicode(result['country']) if 'country' in result else '', + 'formats': unicode(formats), + 'tracks': unicode(tracks), + 'rgid': unicode(result['release-group']['id']), + 'rgtype': unicode(rg_type) }) return releaselist + def getArtist(artistid, extrasonly=False): with mb_lock: @@ -213,13 +220,13 @@ def getArtist(artistid, extrasonly=False): artist = musicbrainzngs.get_artist_by_id(artistid)['artist'] newRgs = None artist['release-group-list'] = [] - while newRgs == None or len(newRgs) >= limit: - newRgs = musicbrainzngs.browse_release_groups(artistid,release_type="album",offset=len(artist['release-group-list']),limit=limit)['release-group-list'] + while newRgs is None or len(newRgs) >= limit: + newRgs = musicbrainzngs.browse_release_groups(artistid, release_type="album", offset=len(artist['release-group-list']), limit=limit)['release-group-list'] artist['release-group-list'] += newRgs except musicbrainzngs.WebServiceError as e: logger.warn('Attempt to retrieve artist information from MusicBrainz failed for artistid: %s (%s)' % (artistid, str(e))) time.sleep(5) - except Exception,e: + except Exception as e: pass if not artist: @@ -247,7 +254,6 @@ def getArtist(artistid, extrasonly=False): # if 'end' in artist['life-span']: # artist_dict['artist_enddate'] = unicode(artist['life-span']['end']) - releasegroups = [] if not extrasonly: @@ -255,10 +261,10 @@ def getArtist(artistid, extrasonly=False): if "secondary-type-list" in rg.keys(): #only add releases without a secondary type continue releasegroups.append({ - 'title': unicode(rg['title']), - 'id': unicode(rg['id']), - 'url': u"http://musicbrainz.org/release-group/" + rg['id'], - 'type': unicode(rg['type']) + 'title': unicode(rg['title']), + 'id': unicode(rg['id']), + 'url': u"http://musicbrainz.org/release-group/" + rg['id'], + 'type': unicode(rg['type']) }) # See if we need to grab extras. Artist specific extras take precedence over global option @@ -278,7 +284,7 @@ def getArtist(artistid, extrasonly=False): extras = map(int, db_artist['Extras'].split(',')) else: extras = [] - extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "spokenword", "audiobook", "other", "dj-mix", "mixtape/street", "broadcast", "interview", "demo"] + extras_list = headphones.POSSIBLE_EXTRAS includes = [] @@ -295,8 +301,8 @@ def getArtist(artistid, extrasonly=False): try: limit = 200 newRgs = None - while newRgs == None or len(newRgs) >= limit: - newRgs = musicbrainzngs.browse_release_groups(artistid,release_type=include,offset=len(mb_extras_list),limit=limit)['release-group-list'] + while newRgs is None or len(newRgs) >= limit: + newRgs = musicbrainzngs.browse_release_groups(artistid, release_type=include, offset=len(mb_extras_list), limit=limit)['release-group-list'] mb_extras_list += newRgs except musicbrainzngs.WebServiceError as e: logger.warn('Attempt to retrieve artist information from MusicBrainz failed for artistid: %s (%s)' % (artistid, str(e))) @@ -311,28 +317,27 @@ def getArtist(artistid, extrasonly=False): rg_type = secondary_type releasegroups.append({ - 'title': unicode(rg['title']), - 'id': unicode(rg['id']), - 'url': u"http://musicbrainz.org/release-group/" + rg['id'], - 'type': unicode(rg_type) + 'title': unicode(rg['title']), + 'id': unicode(rg['id']), + 'url': u"http://musicbrainz.org/release-group/" + rg['id'], + 'type': unicode(rg_type) }) artist_dict['releasegroups'] = releasegroups return artist_dict + def getReleaseGroup(rgid): """ Returns a list of releases in a release group """ with mb_lock: - releaselist = [] - releaseGroup = None try: - releaseGroup = musicbrainzngs.get_release_group_by_id(rgid,["artists","releases","media","discids",])['release-group'] + releaseGroup = musicbrainzngs.get_release_group_by_id(rgid, ["artists", "releases", "media", "discids", ])['release-group'] except musicbrainzngs.WebServiceError as e: logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e))) time.sleep(5) @@ -342,6 +347,7 @@ def getReleaseGroup(rgid): else: return releaseGroup['release-list'] + def getRelease(releaseid, include_artist_info=True): """ Deep release search to get track info @@ -353,9 +359,9 @@ def getRelease(releaseid, include_artist_info=True): try: if include_artist_info: - results = musicbrainzngs.get_release_by_id(releaseid,["artists","release-groups","media","recordings"]).get('release') + results = musicbrainzngs.get_release_by_id(releaseid, ["artists", "release-groups", "media", "recordings"]).get('release') else: - results = musicbrainzngs.get_release_by_id(releaseid,["media","recordings"]).get('release') + results = musicbrainzngs.get_release_by_id(releaseid, ["media", "recordings"]).get('release') except musicbrainzngs.WebServiceError as e: logger.warn('Attempt to retrieve information from MusicBrainz for release "%s" failed (%s)' % (releaseid, str(e))) time.sleep(5) @@ -377,7 +383,6 @@ def getRelease(releaseid, include_artist_info=True): except: release['country'] = u'Unknown' - if include_artist_info: if 'release-group' in results: @@ -404,15 +409,16 @@ def getRelease(releaseid, include_artist_info=True): return release -def get_new_releases(rgid,includeExtras=False,forcefull=False): + +def get_new_releases(rgid, includeExtras=False, forcefull=False): myDB = db.DBConnection() results = [] try: limit = 100 newResults = None - while newResults == None or len(newResults) >= limit: - newResults = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media'],limit=limit,offset=len(results)) + while newResults is None or len(newResults) >= limit: + newResults = musicbrainzngs.browse_releases(release_group=rgid, includes=['artist-credits', 'labels', 'recordings', 'release-groups', 'media'], limit=limit, offset=len(results)) if 'release-list' not in newResults: break #may want to raise an exception here instead ? newResults = newResults['release-list'] @@ -457,8 +463,6 @@ def get_new_releases(rgid,includeExtras=False,forcefull=False): release = {} rel_id_check = releasedata['id'] - artistid = unicode(releasedata['artist-credit'][0]['artist']['id']) - album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone() if not album_checker or forcefull: #DELETE all references to this release since we're updating it anyway. @@ -486,21 +490,20 @@ def get_new_releases(rgid,includeExtras=False,forcefull=False): logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.') return False - release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown' #assuming that the list will contain media and that the format will be consistent try: - additional_medium='' + additional_medium = '' for position in releasedata['medium-list']: if position['format'] == releasedata['medium-list'][0]['format']: medium_count = int(position['position']) else: - additional_medium = additional_medium+' + '+position['format'] + additional_medium = additional_medium + ' + ' + position['format'] if medium_count == 1: disc_number = '' else: - disc_number = str(medium_count)+'x' - packaged_medium = disc_number+releasedata['medium-list'][0]['format']+additional_medium + disc_number = str(medium_count) + 'x' + packaged_medium = disc_number + releasedata['medium-list'][0]['format'] + additional_medium release['ReleaseFormat'] = unicode(packaged_medium) except: release['ReleaseFormat'] = u'Unknown' @@ -510,17 +513,17 @@ def get_new_releases(rgid,includeExtras=False,forcefull=False): # What we're doing here now is first updating the allalbums & alltracks table to the most # current info, then moving the appropriate release into the album table and its associated # tracks into the tracks table - controlValueDict = {"ReleaseID" : release['ReleaseID']} + controlValueDict = {"ReleaseID": release['ReleaseID']} - newValueDict = {"ArtistID": release['ArtistID'], - "ArtistName": release['ArtistName'], - "AlbumTitle": release['AlbumTitle'], - "AlbumID": release['AlbumID'], - "AlbumASIN": release['AlbumASIN'], - "ReleaseDate": release['ReleaseDate'], - "Type": release['Type'], - "ReleaseCountry": release['ReleaseCountry'], - "ReleaseFormat": release['ReleaseFormat'] + newValueDict = {"ArtistID": release['ArtistID'], + "ArtistName": release['ArtistName'], + "AlbumTitle": release['AlbumTitle'], + "AlbumID": release['AlbumID'], + "AlbumASIN": release['AlbumASIN'], + "ReleaseDate": release['ReleaseDate'], + "Type": release['Type'], + "ReleaseCountry": release['ReleaseCountry'], + "ReleaseFormat": release['ReleaseFormat'] } myDB.upsert("allalbums", newValueDict, controlValueDict) @@ -529,18 +532,18 @@ def get_new_releases(rgid,includeExtras=False,forcefull=False): cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title']) - controlValueDict = {"TrackID": track['id'], - "ReleaseID": release['ReleaseID']} + controlValueDict = {"TrackID": track['id'], + "ReleaseID": release['ReleaseID']} - newValueDict = {"ArtistID": release['ArtistID'], - "ArtistName": release['ArtistName'], - "AlbumTitle": release['AlbumTitle'], - "AlbumID": release['AlbumID'], - "AlbumASIN": release['AlbumASIN'], - "TrackTitle": track['title'], - "TrackDuration": track['duration'], - "TrackNumber": track['number'], - "CleanName": cleanname + newValueDict = {"ArtistID": release['ArtistID'], + "ArtistName": release['ArtistName'], + "AlbumTitle": release['AlbumTitle'], + "AlbumID": release['AlbumID'], + "AlbumASIN": release['AlbumASIN'], + "TrackTitle": track['title'], + "TrackDuration": track['duration'], + "TrackNumber": track['number'], + "CleanName": cleanname } match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() @@ -558,8 +561,6 @@ def get_new_releases(rgid,includeExtras=False,forcefull=False): myDB.upsert("alltracks", newValueDict, controlValueDict) num_new_releases = num_new_releases + 1 - #print releasedata['title'] - #print num_new_releases if album_checker: logger.info('[%s] Existing release %s (%s) updated' % (release['ArtistName'], release['AlbumTitle'], rel_id_check)) else: @@ -572,6 +573,7 @@ def get_new_releases(rgid,includeExtras=False,forcefull=False): return num_new_releases + def getTracksFromRelease(release): totalTracks = 1 tracks = [] @@ -582,16 +584,18 @@ def getTracksFromRelease(release): except: track_title = unicode(track['recording']['title']) tracks.append({ - 'number': totalTracks, - 'title': track_title, - 'id': unicode(track['recording']['id']), - 'url': u"http://musicbrainz.org/track/" + track['recording']['id'], - 'duration': int(track['length']) if 'length' in track else 0 + 'number': totalTracks, + 'title': track_title, + 'id': unicode(track['recording']['id']), + 'url': u"http://musicbrainz.org/track/" + track['recording']['id'], + 'duration': int(track['length']) if 'length' in track else 0 }) totalTracks += 1 return tracks # Used when there is a disambiguation + + def findArtistbyAlbum(name): myDB = db.DBConnection() @@ -605,7 +609,7 @@ def findArtistbyAlbum(name): if not artist['AlbumTitle']: return False - term = '"'+artist['AlbumTitle']+'" AND artist:"'+name+'"' + term = '"' + artist['AlbumTitle'] + '" AND artist:"' + name + '"' results = None @@ -615,7 +619,6 @@ def findArtistbyAlbum(name): logger.warn('Attempt to query MusicBrainz for %s failed (%s)' % (name, str(e))) time.sleep(5) - if not results: return False @@ -633,10 +636,9 @@ def findArtistbyAlbum(name): #artist_dict['url'] = u'http://musicbrainz.org/artist/' + newArtist['id'] #artist_dict['score'] = int(releaseGroup['ext:score']) - - return artist_dict + def findAlbumID(artist=None, album=None): results = None @@ -645,14 +647,14 @@ def findAlbumID(artist=None, album=None): try: if album and artist: if any((c in chars) for c in album): - album = '"'+album+'"' + album = '"' + album + '"' if any((c in chars) for c in artist): - artist = '"'+artist+'"' + artist = '"' + artist + '"' criteria = {'release': album.lower()} criteria['artist'] = artist.lower() else: if any((c in chars) for c in album): - album = '"'+album+'"' + album = '"' + album + '"' criteria = {'release': album.lower()} results = musicbrainzngs.search_release_groups(limit=1, **criteria).get('release-group-list') diff --git a/headphones/music_encoder.py b/headphones/music_encoder.py index 48c93a50..27f596ae 100644 --- a/headphones/music_encoder.py +++ b/headphones/music_encoder.py @@ -24,26 +24,23 @@ from headphones import logger from beets.mediafile import MediaFile # xld -if headphones.ENCODER == 'xld': - import getXldProfile - XLD = True -else: - XLD = False +import getXldProfile + def encode(albumPath): + use_xld = headphones.CONFIG.ENCODER == 'xld' # Return if xld details not found - if XLD: - global xldProfile - (xldProfile, xldFormat, xldBitrate) = getXldProfile.getXldProfile(headphones.XLDPROFILE) + if use_xld: + (xldProfile, xldFormat, xldBitrate) = getXldProfile.getXldProfile(headphones.CONFIG.XLDPROFILE) if not xldFormat: logger.error('Details for xld profile \'%s\' not found, files will not be re-encoded', xldProfile) return None - tempDirEncode=os.path.join(albumPath,"temp") - musicFiles=[] - musicFinalFiles=[] - musicTempFiles=[] + tempDirEncode = os.path.join(albumPath, "temp") + musicFiles = [] + musicFinalFiles = [] + musicTempFiles = [] encoder = "" # Create temporary directory, but remove the old one first. @@ -57,19 +54,19 @@ def encode(albumPath): logger.exception("Unable to create temporary directory") return None - for r,d,f in os.walk(albumPath): + for r, d, f in os.walk(albumPath): for music in f: if any(music.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): - if not XLD: - encoderFormat = headphones.ENCODEROUTPUTFORMAT.encode(headphones.SYS_ENCODING) + if not use_xld: + encoderFormat = headphones.CONFIG.ENCODEROUTPUTFORMAT.encode(headphones.SYS_ENCODING) else: xldMusicFile = os.path.join(r, music) xldInfoMusic = MediaFile(xldMusicFile) encoderFormat = xldFormat - if (headphones.ENCODERLOSSLESS): + if (headphones.CONFIG.ENCODERLOSSLESS): ext = os.path.normpath(os.path.splitext(music)[1].lstrip(".")).lower() - if not XLD and ext == 'flac' or XLD and (ext != xldFormat and (xldInfoMusic.bitrate / 1000 > 400)): + if not use_xld and ext == 'flac' or use_xld and (ext != xldFormat and (xldInfoMusic.bitrate / 1000 > 400)): musicFiles.append(os.path.join(r, music)) musicTemp = os.path.normpath(os.path.splitext(music)[0] + '.' + encoderFormat) musicTempFiles.append(os.path.join(tempDirEncode, musicTemp)) @@ -80,29 +77,29 @@ def encode(albumPath): musicTemp = os.path.normpath(os.path.splitext(music)[0] + '.' + encoderFormat) musicTempFiles.append(os.path.join(tempDirEncode, musicTemp)) - if headphones.ENCODER_PATH: - encoder = headphones.ENCODER_PATH.encode(headphones.SYS_ENCODING) + if headphones.CONFIG.ENCODER_PATH: + encoder = headphones.CONFIG.ENCODER_PATH.encode(headphones.SYS_ENCODING) else: - if XLD: + if use_xld: encoder = os.path.join('/Applications', 'xld') - elif headphones.ENCODER =='lame': + elif headphones.CONFIG.ENCODER == 'lame': if headphones.SYS_PLATFORM == "win32": ## NEED THE DEFAULT LAME INSTALL ON WIN! encoder = "C:/Program Files/lame/lame.exe" else: - encoder="lame" - elif headphones.ENCODER =='ffmpeg': + encoder = "lame" + elif headphones.CONFIG.ENCODER == 'ffmpeg': if headphones.SYS_PLATFORM == "win32": encoder = "C:/Program Files/ffmpeg/bin/ffmpeg.exe" else: - encoder="ffmpeg" - elif headphones.ENCODER == 'libav': + encoder = "ffmpeg" + elif headphones.CONFIG.ENCODER == 'libav': if headphones.SYS_PLATFORM == "win32": encoder = "C:/Program Files/libav/bin/avconv.exe" else: - encoder="avconv" + encoder = "avconv" - i=0 + i = 0 encoder_failed = False jobs = [] @@ -110,28 +107,28 @@ def encode(albumPath): infoMusic = MediaFile(music) encode = False - if XLD: + if use_xld: if xldBitrate and (infoMusic.bitrate / 1000 <= xldBitrate): logger.info('%s has bitrate <= %skb, will not be re-encoded', music.decode(headphones.SYS_ENCODING, 'replace'), xldBitrate) else: encode = True - elif headphones.ENCODER == 'lame': + elif headphones.CONFIG.ENCODER == 'lame': if not any(music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.' + x) for x in ["mp3", "wav"]): logger.warn('Lame cannot encode %s format for %s, use ffmpeg', os.path.splitext(music)[1], music) else: - if (music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.mp3') and (int(infoMusic.bitrate / 1000) <= headphones.BITRATE)): - logger.info('%s has bitrate <= %skb, will not be re-encoded', music, headphones.BITRATE) + if (music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.mp3') and (int(infoMusic.bitrate / 1000) <= headphones.CONFIG.BITRATE)): + logger.info('%s has bitrate <= %skb, will not be re-encoded', music, headphones.CONFIG.BITRATE) else: encode = True else: - if headphones.ENCODEROUTPUTFORMAT=='ogg': + if headphones.CONFIG.ENCODEROUTPUTFORMAT == 'ogg': if music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.ogg'): logger.warn('Cannot re-encode .ogg %s', music.decode(headphones.SYS_ENCODING, 'replace')) else: encode = True - elif (headphones.ENCODEROUTPUTFORMAT=='mp3' or headphones.ENCODEROUTPUTFORMAT=='m4a'): - if (music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.'+headphones.ENCODEROUTPUTFORMAT) and (int(infoMusic.bitrate / 1000 ) <= headphones.BITRATE)): - logger.info('%s has bitrate <= %skb, will not be re-encoded', music, headphones.BITRATE) + elif (headphones.CONFIG.ENCODEROUTPUTFORMAT == 'mp3' or headphones.CONFIG.ENCODEROUTPUTFORMAT == 'm4a'): + if (music.decode(headphones.SYS_ENCODING, 'replace').lower().endswith('.' + headphones.CONFIG.ENCODEROUTPUTFORMAT) and (int(infoMusic.bitrate / 1000) <= headphones.CONFIG.BITRATE)): + logger.info('%s has bitrate <= %skb, will not be re-encoded', music, headphones.CONFIG.BITRATE) else: encode = True # encode @@ -142,18 +139,18 @@ def encode(albumPath): musicFiles[i] = None musicTempFiles[i] = None - i=i+1 + i = i + 1 # Encode music files if len(jobs) > 0: processes = 1 # Use multicore if enabled - if headphones.ENCODER_MULTICORE: - if headphones.ENCODER_MULTICORE_COUNT == 0: + if headphones.CONFIG.ENCODER_MULTICORE: + if headphones.CONFIG.ENCODER_MULTICORE_COUNT == 0: processes = multiprocessing.cpu_count() else: - processes = headphones.ENCODER_MULTICORE_COUNT + processes = headphones.CONFIG.ENCODER_MULTICORE_COUNT logger.debug("Multi-core encoding enabled, spawning %d processes", processes) @@ -194,14 +191,14 @@ def encode(albumPath): for dest in musicTempFiles: if os.path.exists(dest): source = musicFiles[i] - if headphones.DELETE_LOSSLESS_FILES: + if headphones.CONFIG.DELETE_LOSSLESS_FILES: os.remove(source) check_dest = os.path.join(albumPath, os.path.split(dest)[1]) if os.path.exists(check_dest): os.remove(check_dest) try: shutil.move(dest, albumPath) - except Exception, e: + except Exception as e: logger.error('Could not move %s to %s: %s', dest, albumPath, e) encoder_failed = True break @@ -212,11 +209,11 @@ def encode(albumPath): # Return with error if any encoding errors if encoder_failed: - logger.error("One or more files failed to encode. Ensure you have the latest version of %s installed.", headphones.ENCODER) + logger.error("One or more files failed to encode. Ensure you have the latest version of %s installed.", headphones.CONFIG.ENCODER) return None time.sleep(1) - for r,d,f in os.walk(albumPath): + for r, d, f in os.walk(albumPath): for music in f: if any(music.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): musicFinalFiles.append(os.path.join(r, music)) @@ -226,6 +223,7 @@ def encode(albumPath): return musicFinalFiles + def command_map(args): """ Wrapper for the '[multiprocessing.]map()' method, to unpack the arguments @@ -239,21 +237,27 @@ def command_map(args): # Start encoding try: return command(*args) - except Exception as e: + except Exception: logger.exception("Encoder raised an exception.") return False + def command(encoder, musicSource, musicDest, albumPath): """ Encode a given music file with a certain encoder. Returns True on success, or False otherwise. """ + use_xld = headphones.CONFIG.ENCODER == 'xld' startMusicTime = time.time() cmd = [] - # XLD - if XLD: + # Return if xld details not found + if use_xld: + (xldProfile, xldFormat, xldBitrate) = getXldProfile.getXldProfile(headphones.CONFIG.XLDPROFILE) + if not xldFormat: + logger.error('Details for xld profile \'%s\' not found, files will not be re-encoded', xldProfile) + return None xldDestDir = os.path.split(musicDest)[0] cmd = [encoder] cmd.extend([musicSource]) @@ -263,17 +267,17 @@ def command(encoder, musicSource, musicDest, albumPath): cmd.extend([xldDestDir]) # Lame - elif headphones.ENCODER == 'lame': + elif headphones.CONFIG.ENCODER == 'lame': cmd = [encoder] opts = [] - if not headphones.ADVANCEDENCODER: + if not headphones.CONFIG.ADVANCEDENCODER: opts.extend(['-h']) - if headphones.ENCODERVBRCBR=='cbr': - opts.extend(['--resample', str(headphones.SAMPLINGFREQUENCY), '-b', str(headphones.BITRATE)]) - elif headphones.ENCODERVBRCBR=='vbr': - opts.extend(['-v', str(headphones.ENCODERQUALITY)]) + if headphones.CONFIG.ENCODERVBRCBR == 'cbr': + opts.extend(['--resample', str(headphones.CONFIG.SAMPLINGFREQUENCY), '-b', str(headphones.CONFIG.BITRATE)]) + elif headphones.CONFIG.ENCODERVBRCBR == 'vbr': + opts.extend(['-v', str(headphones.CONFIG.ENCODERQUALITY)]) else: - advanced = (headphones.ADVANCEDENCODER.split()) + advanced = (headphones.CONFIG.ADVANCEDENCODER.split()) for tok in advanced: opts.extend([tok.encode(headphones.SYS_ENCODING)]) opts.extend([musicSource]) @@ -281,42 +285,42 @@ def command(encoder, musicSource, musicDest, albumPath): cmd.extend(opts) # FFmpeg - elif headphones.ENCODER == 'ffmpeg': + elif headphones.CONFIG.ENCODER == 'ffmpeg': cmd = [encoder, '-i', musicSource] opts = [] - if not headphones.ADVANCEDENCODER: - if headphones.ENCODEROUTPUTFORMAT=='ogg': + if not headphones.CONFIG.ADVANCEDENCODER: + if headphones.CONFIG.ENCODEROUTPUTFORMAT == 'ogg': opts.extend(['-acodec', 'libvorbis']) - if headphones.ENCODEROUTPUTFORMAT=='m4a': + if headphones.CONFIG.ENCODEROUTPUTFORMAT == 'm4a': opts.extend(['-strict', 'experimental']) - if headphones.ENCODERVBRCBR=='cbr': - opts.extend(['-ar', str(headphones.SAMPLINGFREQUENCY), '-ab', str(headphones.BITRATE) + 'k']) - elif headphones.ENCODERVBRCBR=='vbr': - opts.extend(['-aq', str(headphones.ENCODERQUALITY)]) + if headphones.CONFIG.ENCODERVBRCBR == 'cbr': + opts.extend(['-ar', str(headphones.CONFIG.SAMPLINGFREQUENCY), '-ab', str(headphones.CONFIG.BITRATE) + 'k']) + elif headphones.CONFIG.ENCODERVBRCBR == 'vbr': + opts.extend(['-aq', str(headphones.CONFIG.ENCODERQUALITY)]) opts.extend(['-y', '-ac', '2', '-vn']) else: - advanced = (headphones.ADVANCEDENCODER.split()) + advanced = (headphones.CONFIG.ADVANCEDENCODER.split()) for tok in advanced: opts.extend([tok.encode(headphones.SYS_ENCODING)]) opts.extend([musicDest]) cmd.extend(opts) # Libav - elif headphones.ENCODER == "libav": + elif headphones.CONFIG.ENCODER == "libav": cmd = [encoder, '-i', musicSource] opts = [] - if not headphones.ADVANCEDENCODER: - if headphones.ENCODEROUTPUTFORMAT=='ogg': + if not headphones.CONFIG.ADVANCEDENCODER: + if headphones.CONFIG.ENCODEROUTPUTFORMAT == 'ogg': opts.extend(['-acodec', 'libvorbis']) - if headphones.ENCODEROUTPUTFORMAT=='m4a': + if headphones.CONFIG.ENCODEROUTPUTFORMAT == 'm4a': opts.extend(['-strict', 'experimental']) - if headphones.ENCODERVBRCBR=='cbr': - opts.extend(['-ar', str(headphones.SAMPLINGFREQUENCY), '-ab', str(headphones.BITRATE) + 'k']) - elif headphones.ENCODERVBRCBR=='vbr': - opts.extend(['-aq', str(headphones.ENCODERQUALITY)]) + if headphones.CONFIG.ENCODERVBRCBR == 'cbr': + opts.extend(['-ar', str(headphones.CONFIG.SAMPLINGFREQUENCY), '-ab', str(headphones.CONFIG.BITRATE) + 'k']) + elif headphones.CONFIG.ENCODERVBRCBR == 'vbr': + opts.extend(['-aq', str(headphones.CONFIG.ENCODERQUALITY)]) opts.extend(['-y', '-ac', '2', '-vn']) else: - advanced = (headphones.ADVANCEDENCODER.split()) + advanced = (headphones.CONFIG.ADVANCEDENCODER.split()) for tok in advanced: opts.extend([tok.encode(headphones.SYS_ENCODING)]) opts.extend([musicDest]) @@ -339,7 +343,7 @@ def command(encoder, musicSource, musicDest, albumPath): process = subprocess.Popen(cmd, startupinfo=startupinfo, stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = process.communicate(headphones.ENCODER) + stdout, stderr = process.communicate(headphones.CONFIG.ENCODER) # Error if return code not zero if process.returncode: @@ -347,7 +351,7 @@ def command(encoder, musicSource, musicDest, albumPath): out = stdout if stdout else stderr out = out.decode(headphones.SYS_ENCODING, 'replace') outlast2lines = '\n'.join(out.splitlines()[-2:]) - logger.error('%s error details: %s' % (headphones.ENCODER, outlast2lines)) + logger.error('%s error details: %s' % (headphones.CONFIG.ENCODER, outlast2lines)) out = out.rstrip("\n") logger.debug(out) encoded = False @@ -357,10 +361,11 @@ def command(encoder, musicSource, musicDest, albumPath): return encoded + def getTimeEncode(start): - seconds =int(time.time()-start) + seconds = int(time.time() - start) hours = seconds / 3600 - seconds -= 3600*hours + seconds -= 3600 * hours minutes = seconds / 60 - seconds -= 60*minutes - return "%02d:%02d:%02d" % (hours, minutes, seconds) \ No newline at end of file + seconds -= 60 * minutes + return "%02d:%02d:%02d" % (hours, minutes, seconds) diff --git a/headphones/notifiers.py b/headphones/notifiers.py index 517f35e2..953b7d5f 100644 --- a/headphones/notifiers.py +++ b/headphones/notifiers.py @@ -28,7 +28,6 @@ import headphones import os.path import subprocess import gntp.notifier -import time import json import oauth2 as oauth @@ -39,15 +38,16 @@ try: except ImportError: from cgi import parse_qsl + class GROWL(object): """ Growl notifications, for OS X. """ def __init__(self): - self.enabled = headphones.GROWL_ENABLED - self.host = headphones.GROWL_HOST - self.password = headphones.GROWL_PASSWORD + self.enabled = headphones.CONFIG.GROWL_ENABLED + self.host = headphones.CONFIG.GROWL_HOST + self.password = headphones.CONFIG.GROWL_PASSWORD def conf(self, options): return cherrypy.config['config'].get('Growl', options) @@ -124,35 +124,36 @@ class GROWL(object): self.notify('ZOMG Lazors Pewpewpew!', 'Test Message') + class PROWL(object): """ Prowl notifications. """ def __init__(self): - self.enabled = headphones.PROWL_ENABLED - self.keys = headphones.PROWL_KEYS - self.priority = headphones.PROWL_PRIORITY + self.enabled = headphones.CONFIG.PROWL_ENABLED + self.keys = headphones.CONFIG.PROWL_KEYS + self.priority = headphones.CONFIG.PROWL_PRIORITY def conf(self, options): return cherrypy.config['config'].get('Prowl', options) def notify(self, message, event): - if not headphones.PROWL_ENABLED: + if not headphones.CONFIG.PROWL_ENABLED: return http_handler = HTTPSConnection("api.prowlapp.com") - data = {'apikey': headphones.PROWL_KEYS, + data = {'apikey': headphones.CONFIG.PROWL_KEYS, 'application': 'Headphones', 'event': event, 'description': message.encode("utf-8"), - 'priority': headphones.PROWL_PRIORITY } + 'priority': headphones.CONFIG.PROWL_PRIORITY} http_handler.request("POST", "/publicapi/add", - headers = {'Content-type': "application/x-www-form-urlencoded"}, - body = urlencode(data)) + headers={'Content-type': "application/x-www-form-urlencoded"}, + body=urlencode(data)) response = http_handler.getresponse() request_status = response.status @@ -177,6 +178,7 @@ class PROWL(object): self.notify('ZOMG Lazors Pewpewpew!', 'Test Message') + class MPC(object): """ MPC library update @@ -186,8 +188,8 @@ class MPC(object): pass - def notify( self ): - subprocess.call( ["mpc", "update"] ) + def notify(self): + subprocess.call(["mpc", "update"]) class XBMC(object): @@ -197,9 +199,9 @@ class XBMC(object): def __init__(self): - self.hosts = headphones.XBMC_HOST - self.username = headphones.XBMC_USERNAME - self.password = headphones.XBMC_PASSWORD + self.hosts = headphones.CONFIG.XBMC_HOST + self.username = headphones.CONFIG.XBMC_USERNAME + self.password = headphones.CONFIG.XBMC_PASSWORD def _sendhttp(self, host, command): url_command = urllib.urlencode(command) @@ -230,7 +232,7 @@ class XBMC(object): hosts = [x.strip() for x in self.hosts.split(',')] for host in hosts: - logger.info('Sending library update command to XBMC @ '+host) + logger.info('Sending library update command to XBMC @ ' + host) request = self._sendjson(host, 'AudioLibrary.Scan') if not request: @@ -245,17 +247,17 @@ class XBMC(object): time = "3000" # in ms for host in hosts: - logger.info('Sending notification command to XMBC @ '+host) + logger.info('Sending notification command to XMBC @ ' + host) try: version = self._sendjson(host, 'Application.GetProperties', {'properties': ['version']})['version']['major'] if version < 12: #Eden notification = header + "," + message + "," + time + "," + albumartpath - notifycommand = {'command': 'ExecBuiltIn', 'parameter': 'Notification('+notification+')'} + notifycommand = {'command': 'ExecBuiltIn', 'parameter': 'Notification(' + notification + ')'} request = self._sendhttp(host, notifycommand) else: #Frodo - params = {'title':header, 'message': message, 'displaytime': int(time), 'image': albumartpath} + params = {'title': header, 'message': message, 'displaytime': int(time), 'image': albumartpath} request = self._sendjson(host, 'GUI.ShowNotification', params) if not request: @@ -264,25 +266,26 @@ class XBMC(object): except Exception: logger.error('Error sending notification request to XBMC') + class LMS(object): """ Class for updating a Logitech Media Server """ def __init__(self): - self.hosts = headphones.LMS_HOST + self.hosts = headphones.CONFIG.LMS_HOST def _sendjson(self, host): - data = {'id': 1, 'method': 'slim.request', 'params': ["",["rescan"]]} + data = {'id': 1, 'method': 'slim.request', 'params': ["", ["rescan"]]} data = json.JSONEncoder().encode(data) content = {'Content-Type': 'application/json'} - req = urllib2.Request(host+'/jsonrpc.js', data, content) + req = urllib2.Request(host + '/jsonrpc.js', data, content) try: handle = urllib2.urlopen(req) - except Exception, e: + except Exception as e: logger.warn('Error opening LMS url: %s' % e) return @@ -299,19 +302,20 @@ class LMS(object): hosts = [x.strip() for x in self.hosts.split(',')] for host in hosts: - logger.info('Sending library rescan command to LMS @ '+host) + logger.info('Sending library rescan command to LMS @ ' + host) request = self._sendjson(host) if not request: logger.warn('Error sending rescan request to LMS') + class Plex(object): def __init__(self): - self.server_hosts = headphones.PLEX_SERVER_HOST - self.client_hosts = headphones.PLEX_CLIENT_HOST - self.username = headphones.PLEX_USERNAME - self.password = headphones.PLEX_PASSWORD + self.server_hosts = headphones.CONFIG.PLEX_SERVER_HOST + self.client_hosts = headphones.CONFIG.PLEX_CLIENT_HOST + self.username = headphones.CONFIG.PLEX_USERNAME + self.password = headphones.CONFIG.PLEX_PASSWORD def _sendhttp(self, host, command): @@ -332,7 +336,7 @@ class Plex(object): try: handle = urllib2.urlopen(req) - except Exception, e: + except Exception as e: logger.warn('Error opening Plex url: %s' % e) return @@ -348,7 +352,7 @@ class Plex(object): hosts = [x.strip() for x in self.server_hosts.split(',')] for host in hosts: - logger.info('Sending library update command to Plex Media Server@ '+host) + logger.info('Sending library update command to Plex Media Server@ ' + host) url = "%s/library/sections" % host try: xml_sections = minidom.parse(urllib.urlopen(url)) @@ -366,7 +370,7 @@ class Plex(object): url = "%s/library/sections/%s/refresh" % (host, s.getAttribute('key')) try: urllib.urlopen(url) - except Exception, e: + except Exception as e: logger.warn("Error updating library section for Plex Media Server: %s" % e) return False @@ -379,10 +383,10 @@ class Plex(object): time = "3000" # in ms for host in hosts: - logger.info('Sending notification command to Plex Media Server @ '+host) + logger.info('Sending notification command to Plex Media Server @ ' + host) try: notification = header + "," + message + "," + time + "," + albumartpath - notifycommand = {'command': 'ExecBuiltIn', 'parameter': 'Notification('+notification+')'} + notifycommand = {'command': 'ExecBuiltIn', 'parameter': 'Notification(' + notification + ')'} request = self._sendhttp(host, notifycommand) if not request: @@ -391,11 +395,12 @@ class Plex(object): except: logger.warn('Error sending notification request to Plex Media Server') + class NMA(object): def notify(self, artist=None, album=None, snatched=None): title = 'Headphones' - api = headphones.NMA_APIKEY - nma_priority = headphones.NMA_PRIORITY + api = headphones.CONFIG.NMA_APIKEY + nma_priority = headphones.CONFIG.NMA_PRIORITY logger.debug(u"NMA title: " + title) logger.debug(u"NMA API: " + api) @@ -417,7 +422,8 @@ class NMA(object): keys = api.split(',') p.addkey(keys) - if len(keys) > 1: batch = True + if len(keys) > 1: + batch = True response = p.push(title, event, message, priority=nma_priority, batch_mode=batch) @@ -427,31 +433,32 @@ class NMA(object): else: return True + class PUSHBULLET(object): def __init__(self): - self.apikey = headphones.PUSHBULLET_APIKEY - self.deviceid = headphones.PUSHBULLET_DEVICEID + self.apikey = headphones.CONFIG.PUSHBULLET_APIKEY + self.deviceid = headphones.CONFIG.PUSHBULLET_DEVICEID def conf(self, options): return cherrypy.config['config'].get('PUSHBULLET', options) def notify(self, message, event): - if not headphones.PUSHBULLET_ENABLED: + if not headphones.CONFIG.PUSHBULLET_ENABLED: return http_handler = HTTPSConnection("api.pushbullet.com") - data = {'device_iden': headphones.PUSHBULLET_DEVICEID, + data = {'device_iden': headphones.CONFIG.PUSHBULLET_DEVICEID, 'type': "note", 'title': "Headphones", - 'body': message.encode("utf-8") } + 'body': message.encode("utf-8")} http_handler.request("POST", "/api/pushes", - headers = {'Content-type': "application/x-www-form-urlencoded", - 'Authorization' : 'Basic %s' % base64.b64encode(headphones.PUSHBULLET_APIKEY + ":") }, - body = urlencode(data)) + headers={'Content-type': "application/x-www-form-urlencoded", + 'Authorization': 'Basic %s' % base64.b64encode(headphones.CONFIG.PUSHBULLET_APIKEY + ":")}, + body=urlencode(data)) response = http_handler.getresponse() request_status = response.status logger.debug(u"PushBullet response status: %r" % request_status) @@ -480,13 +487,14 @@ class PUSHBULLET(object): self.notify('Main Screen Activate', 'Test Message') + class PUSHALOT(object): def notify(self, message, event): - if not headphones.PUSHALOT_ENABLED: + if not headphones.CONFIG.PUSHALOT_ENABLED: return - pushalot_authorizationtoken = headphones.PUSHALOT_APIKEY + pushalot_authorizationtoken = headphones.CONFIG.PUSHALOT_APIKEY logger.debug(u"Pushalot event: " + event) logger.debug(u"Pushalot message: " + message) @@ -496,12 +504,12 @@ class PUSHALOT(object): data = {'AuthorizationToken': pushalot_authorizationtoken, 'Title': event.encode('utf-8'), - 'Body': message.encode("utf-8") } + 'Body': message.encode("utf-8")} http_handler.request("POST", "/api/sendmessage", - headers = {'Content-type': "application/x-www-form-urlencoded"}, - body = urlencode(data)) + headers={'Content-type': "application/x-www-form-urlencoded"}, + body=urlencode(data)) response = http_handler.getresponse() request_status = response.status @@ -519,6 +527,7 @@ class PUSHALOT(object): logger.info(u"Pushalot notification failed.") return False + class Synoindex(object): def __init__(self, util_loc='/usr/syno/bin/synoindex'): self.util_loc = util_loc @@ -555,15 +564,16 @@ class Synoindex(object): for path in path_list: self.notify(path) + class PUSHOVER(object): def __init__(self): - self.enabled = headphones.PUSHOVER_ENABLED - self.keys = headphones.PUSHOVER_KEYS - self.priority = headphones.PUSHOVER_PRIORITY + self.enabled = headphones.CONFIG.PUSHOVER_ENABLED + self.keys = headphones.CONFIG.PUSHOVER_KEYS + self.priority = headphones.CONFIG.PUSHOVER_PRIORITY - if headphones.PUSHOVER_APITOKEN: - self.application_token = headphones.PUSHOVER_APITOKEN + if headphones.CONFIG.PUSHOVER_APITOKEN: + self.application_token = headphones.CONFIG.PUSHOVER_APITOKEN else: self.application_token = "LdPCoy0dqC21ktsbEyAVCcwvQiVlsz" @@ -571,21 +581,21 @@ class PUSHOVER(object): return cherrypy.config['config'].get('Pushover', options) def notify(self, message, event): - if not headphones.PUSHOVER_ENABLED: + if not headphones.CONFIG.PUSHOVER_ENABLED: return http_handler = HTTPSConnection("api.pushover.net") data = {'token': self.application_token, - 'user': headphones.PUSHOVER_KEYS, + 'user': headphones.CONFIG.PUSHOVER_KEYS, 'title': event, 'message': message.encode("utf-8"), - 'priority': headphones.PUSHOVER_PRIORITY } + 'priority': headphones.CONFIG.PUSHOVER_PRIORITY} http_handler.request("POST", "/1/messages.json", - headers = {'Content-type': "application/x-www-form-urlencoded"}, - body = urlencode(data)) + headers={'Content-type': "application/x-www-form-urlencoded"}, + body=urlencode(data)) response = http_handler.getresponse() request_status = response.status logger.debug(u"Pushover response status: %r" % request_status) @@ -613,33 +623,33 @@ class PUSHOVER(object): self.notify('Main Screen Activate', 'Test Message') + class TwitterNotifier(object): REQUEST_TOKEN_URL = 'https://api.twitter.com/oauth/request_token' - ACCESS_TOKEN_URL = 'https://api.twitter.com/oauth/access_token' + ACCESS_TOKEN_URL = 'https://api.twitter.com/oauth/access_token' AUTHORIZATION_URL = 'https://api.twitter.com/oauth/authorize' - SIGNIN_URL = 'https://api.twitter.com/oauth/authenticate' + SIGNIN_URL = 'https://api.twitter.com/oauth/authenticate' def __init__(self): self.consumer_key = "oYKnp2ddX5gbARjqX8ZAAg" self.consumer_secret = "A4Xkw9i5SjHbTk7XT8zzOPqivhj9MmRDR9Qn95YA9sk" def notify_snatch(self, title): - if headphones.TWITTER_ONSNATCH: - self._notifyTwitter(common.notifyStrings[common.NOTIFY_SNATCH]+': '+title+' at '+helpers.now()) + if headphones.CONFIG.TWITTER_ONSNATCH: + self._notifyTwitter(common.notifyStrings[common.NOTIFY_SNATCH] + ': ' + title + ' at ' + helpers.now()) def notify_download(self, title): - if headphones.TWITTER_ENABLED: - self._notifyTwitter(common.notifyStrings[common.NOTIFY_DOWNLOAD]+': '+title+' at '+helpers.now()) + if headphones.CONFIG.TWITTER_ENABLED: + self._notifyTwitter(common.notifyStrings[common.NOTIFY_DOWNLOAD] + ': ' + title + ' at ' + helpers.now()) def test_notify(self): - return self._notifyTwitter("This is a test notification from Headphones at "+helpers.now(), force=True) + return self._notifyTwitter("This is a test notification from Headphones at " + helpers.now(), force=True) def _get_authorization(self): - signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1() #@UnusedVariable - oauth_consumer = oauth.Consumer(key=self.consumer_key, secret=self.consumer_secret) - oauth_client = oauth.Client(oauth_consumer) + oauth_consumer = oauth.Consumer(key=self.consumer_key, secret=self.consumer_secret) + oauth_client = oauth.Client(oauth_consumer) logger.info('Requesting temp token from Twitter') @@ -650,72 +660,71 @@ class TwitterNotifier(object): else: request_token = dict(parse_qsl(content)) - headphones.TWITTER_USERNAME = request_token['oauth_token'] - headphones.TWITTER_PASSWORD = request_token['oauth_token_secret'] + headphones.CONFIG.TWITTER_USERNAME = request_token['oauth_token'] + headphones.CONFIG.TWITTER_PASSWORD = request_token['oauth_token_secret'] - return self.AUTHORIZATION_URL+"?oauth_token="+ request_token['oauth_token'] + return self.AUTHORIZATION_URL + "?oauth_token=" + request_token['oauth_token'] def _get_credentials(self, key): request_token = {} - request_token['oauth_token'] = headphones.TWITTER_USERNAME - request_token['oauth_token_secret'] = headphones.TWITTER_PASSWORD + request_token['oauth_token'] = headphones.CONFIG.TWITTER_USERNAME + request_token['oauth_token_secret'] = headphones.CONFIG.TWITTER_PASSWORD request_token['oauth_callback_confirmed'] = 'true' token = oauth.Token(request_token['oauth_token'], request_token['oauth_token_secret']) token.set_verifier(key) - logger.info('Generating and signing request for an access token using key '+key) + logger.info('Generating and signing request for an access token using key ' + key) - signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1() #@UnusedVariable - oauth_consumer = oauth.Consumer(key=self.consumer_key, secret=self.consumer_secret) - logger.info('oauth_consumer: '+str(oauth_consumer)) - oauth_client = oauth.Client(oauth_consumer, token) - logger.info('oauth_client: '+str(oauth_client)) + oauth_consumer = oauth.Consumer(key=self.consumer_key, secret=self.consumer_secret) + logger.info('oauth_consumer: ' + str(oauth_consumer)) + oauth_client = oauth.Client(oauth_consumer, token) + logger.info('oauth_client: ' + str(oauth_client)) resp, content = oauth_client.request(self.ACCESS_TOKEN_URL, method='POST', body='oauth_verifier=%s' % key) - logger.info('resp, content: '+str(resp)+','+str(content)) + logger.info('resp, content: ' + str(resp) + ',' + str(content)) - access_token = dict(parse_qsl(content)) - logger.info('access_token: '+str(access_token)) + access_token = dict(parse_qsl(content)) + logger.info('access_token: ' + str(access_token)) - logger.info('resp[status] = '+str(resp['status'])) + logger.info('resp[status] = ' + str(resp['status'])) if resp['status'] != '200': - logger.info('The request for a token with did not succeed: '+str(resp['status']), logger.ERROR) + logger.info('The request for a token with did not succeed: ' + str(resp['status']), logger.ERROR) return False else: logger.info('Your Twitter Access Token key: %s' % access_token['oauth_token']) logger.info('Access Token secret: %s' % access_token['oauth_token_secret']) - headphones.TWITTER_USERNAME = access_token['oauth_token'] - headphones.TWITTER_PASSWORD = access_token['oauth_token_secret'] + headphones.CONFIG.TWITTER_USERNAME = access_token['oauth_token'] + headphones.CONFIG.TWITTER_PASSWORD = access_token['oauth_token_secret'] return True - def _send_tweet(self, message=None): - username=self.consumer_key - password=self.consumer_secret - access_token_key=headphones.TWITTER_USERNAME - access_token_secret=headphones.TWITTER_PASSWORD + username = self.consumer_key + password = self.consumer_secret + access_token_key = headphones.CONFIG.TWITTER_USERNAME + access_token_secret = headphones.CONFIG.TWITTER_PASSWORD - logger.info(u"Sending tweet: "+message) + logger.info(u"Sending tweet: " + message) api = twitter.Api(username, password, access_token_key, access_token_secret) try: api.PostUpdate(message) - except Exception, e: + except Exception as e: logger.info(u"Error Sending Tweet: %s" % e) return False return True def _notifyTwitter(self, message='', force=False): - prefix = headphones.TWITTER_PREFIX + prefix = headphones.CONFIG.TWITTER_PREFIX - if not headphones.TWITTER_ENABLED and not force: + if not headphones.CONFIG.TWITTER_ENABLED and not force: return False - return self._send_tweet(prefix+": "+message) + return self._send_tweet(prefix + ": " + message) + class OSX_NOTIFY(object): @@ -727,6 +736,7 @@ class OSX_NOTIFY(object): def swizzle(self, cls, SEL, func): old_IMP = cls.instanceMethodForSelector_(SEL) + def wrapper(self, *args, **kwargs): return func(self, old_IMP, *args, **kwargs) new_IMP = self.objc.selector(wrapper, selector=old_IMP.selector, @@ -765,13 +775,14 @@ class OSX_NOTIFY(object): del pool return True - except Exception, e: + except Exception as e: logger.warn('Error sending OS X Notification: %s' % e) return False def swizzled_bundleIdentifier(self, original, swizzled): return 'ade.headphones.osxnotify' + class BOXCAR(object): def __init__(self): @@ -783,7 +794,7 @@ class BOXCAR(object): message += '

    MusicBrainz' % rgid data = urllib.urlencode({ - 'user_credentials': headphones.BOXCAR_TOKEN, + 'user_credentials': headphones.CONFIG.BOXCAR_TOKEN, 'notification[title]': title.encode('utf-8'), 'notification[long_message]': message.encode('utf-8'), 'notification[sound]': "done" @@ -798,12 +809,13 @@ class BOXCAR(object): logger.warn('Error sending Boxcar2 Notification: %s' % e) return False + class SubSonicNotifier(object): def __init__(self): - self.host = headphones.SUBSONIC_HOST - self.username = headphones.SUBSONIC_USERNAME - self.password = headphones.SUBSONIC_PASSWORD + self.host = headphones.CONFIG.SUBSONIC_HOST + self.username = headphones.CONFIG.SUBSONIC_USERNAME + self.password = headphones.CONFIG.SUBSONIC_PASSWORD def notify(self, albumpaths): # Correct URL @@ -815,4 +827,4 @@ class SubSonicNotifier(object): # Invoke request request.request_response(self.host + "musicFolderSettings.view?scanNow", - auth=(self.username, self.password)) \ No newline at end of file + auth=(self.username, self.password)) diff --git a/headphones/nzbget.py b/headphones/nzbget.py index 0d9eca19..50240f35 100644 --- a/headphones/nzbget.py +++ b/headphones/nzbget.py @@ -19,37 +19,33 @@ # along with Sick Beard. If not, see . - import httplib -import datetime import headphones from base64 import standard_b64encode import xmlrpclib -#from headphones.providers.generic import GenericProvider - from headphones import logger + def sendNZB(nzb): addToTop = False nzbgetXMLrpc = "%(username)s:%(password)s@%(host)s/xmlrpc" - if headphones.NZBGET_HOST == None: + if headphones.CONFIG.NZBGET_HOST is None: logger.error(u"No NZBget host found in configuration. Please configure it.") return False - if headphones.NZBGET_HOST.startswith('https://'): + if headphones.CONFIG.NZBGET_HOST.startswith('https://'): nzbgetXMLrpc = 'https://' + nzbgetXMLrpc - headphones.NZBGET_HOST.replace('https://','',1) + headphones.CONFIG.NZBGET_HOST.replace('https://', '', 1) else: nzbgetXMLrpc = 'http://' + nzbgetXMLrpc - headphones.NZBGET_HOST.replace('http://','',1) + headphones.CONFIG.NZBGET_HOST.replace('http://', '', 1) - - url = nzbgetXMLrpc % {"host": headphones.NZBGET_HOST, "username": headphones.NZBGET_USERNAME, "password": headphones.NZBGET_PASSWORD} + url = nzbgetXMLrpc % {"host": headphones.CONFIG.NZBGET_HOST, "username": headphones.CONFIG.NZBGET_USERNAME, "password": headphones.CONFIG.NZBGET_PASSWORD} nzbGetRPC = xmlrpclib.ServerProxy(url) try: @@ -86,35 +82,37 @@ def sendNZB(nzb): nzbget_version = int(nzbget_version_str[:nzbget_version_str.find(".")]) if nzbget_version == 0: if nzbcontent64 is not None: - nzbget_result = nzbGetRPC.append(nzb.name + ".nzb", headphones.NZBGET_CATEGORY, addToTop, nzbcontent64) + nzbget_result = nzbGetRPC.append(nzb.name + ".nzb", headphones.CONFIG.NZBGET_CATEGORY, addToTop, nzbcontent64) else: - if nzb.resultType == "nzb": - genProvider = GenericProvider("") - data = genProvider.getURL(nzb.url) - if (data == None): - return False - nzbcontent64 = standard_b64encode(data) - nzbget_result = nzbGetRPC.append(nzb.name + ".nzb", headphones.NZBGET_CATEGORY, addToTop, nzbcontent64) + # from headphones.common.providers.generic import GenericProvider + # if nzb.resultType == "nzb": + # genProvider = GenericProvider("") + # data = genProvider.getURL(nzb.url) + # if (data is None): + # return False + # nzbcontent64 = standard_b64encode(data) + # nzbget_result = nzbGetRPC.append(nzb.name + ".nzb", headphones.CONFIG.NZBGET_CATEGORY, addToTop, nzbcontent64) + return False elif nzbget_version == 12: if nzbcontent64 is not None: - nzbget_result = nzbGetRPC.append(nzb.name + ".nzb", headphones.NZBGET_CATEGORY, headphones.NZBGET_PRIORITY, False, + nzbget_result = nzbGetRPC.append(nzb.name + ".nzb", headphones.CONFIG.NZBGET_CATEGORY, headphones.CONFIG.NZBGET_PRIORITY, False, nzbcontent64, False, dupekey, dupescore, "score") else: - nzbget_result = nzbGetRPC.appendurl(nzb.name + ".nzb", headphones.NZBGET_CATEGORY, headphones.NZBGET_PRIORITY, False, + nzbget_result = nzbGetRPC.appendurl(nzb.name + ".nzb", headphones.CONFIG.NZBGET_CATEGORY, headphones.CONFIG.NZBGET_PRIORITY, False, nzb.url, False, dupekey, dupescore, "score") # v13+ has a new combined append method that accepts both (url and content) # also the return value has changed from boolean to integer # (Positive number representing NZBID of the queue item. 0 and negative numbers represent error codes.) elif nzbget_version >= 13: nzbget_result = True if nzbGetRPC.append(nzb.name + ".nzb", nzbcontent64 if nzbcontent64 is not None else nzb.url, - headphones.NZBGET_CATEGORY, headphones.NZBGET_PRIORITY, False, False, dupekey, dupescore, + headphones.CONFIG.NZBGET_CATEGORY, headphones.CONFIG.NZBGET_PRIORITY, False, False, dupekey, dupescore, "score") > 0 else False else: if nzbcontent64 is not None: - nzbget_result = nzbGetRPC.append(nzb.name + ".nzb", headphones.NZBGET_CATEGORY, headphones.NZBGET_PRIORITY, False, + nzbget_result = nzbGetRPC.append(nzb.name + ".nzb", headphones.CONFIG.NZBGET_CATEGORY, headphones.CONFIG.NZBGET_PRIORITY, False, nzbcontent64) else: - nzbget_result = nzbGetRPC.appendurl(nzb.name + ".nzb", headphones.NZBGET_CATEGORY, headphones.NZBGET_PRIORITY, False, + nzbget_result = nzbGetRPC.appendurl(nzb.name + ".nzb", headphones.CONFIG.NZBGET_CATEGORY, headphones.CONFIG.NZBGET_PRIORITY, False, nzb.url) if nzbget_result: diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py old mode 100644 new mode 100755 index a1f0bcc7..4de9de5d --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -23,7 +23,6 @@ import headphones from beets import autotag from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError -from beets import plugins from beetsplug import lyrics as beetslyrics from headphones import notifiers, utorrent, transmission @@ -32,6 +31,7 @@ from headphones import logger, helpers, request, mb, music_encoder postprocessor_lock = threading.Lock() + def checkFolder(): with postprocessor_lock: @@ -44,11 +44,11 @@ def checkFolder(): if album['FolderName']: if album['Kind'] == 'nzb': - download_dir = headphones.DOWNLOAD_DIR + download_dir = headphones.CONFIG.DOWNLOAD_DIR else: - download_dir = headphones.DOWNLOAD_TORRENT_DIR + download_dir = headphones.CONFIG.DOWNLOAD_TORRENT_DIR - album_path = os.path.join(download_dir, album['FolderName']).encode(headphones.SYS_ENCODING,'replace') + album_path = os.path.join(download_dir, album['FolderName']).encode(headphones.SYS_ENCODING, 'replace') logger.info("Checking if %s exists" % album_path) if os.path.exists(album_path): logger.info('Found "' + album['FolderName'] + '" in ' + album['Kind'] + ' download folder. Verifying....') @@ -57,6 +57,7 @@ def checkFolder(): else: logger.info("No folder name found for " + album['Title']) + def verify(albumid, albumpath, Kind=None, forced=False): myDB = db.DBConnection() @@ -69,7 +70,7 @@ def verify(albumid, albumpath, Kind=None, forced=False): # Fetch album information from MusicBrainz try: release_list = mb.getReleaseGroup(albumid) - except Exception, e: + except Exception as e: logger.error('Unable to get release information for manual album with rgid: %s. Error: %s', albumid, e) return @@ -90,7 +91,7 @@ def verify(albumid, albumpath, Kind=None, forced=False): # frozen during post processing, new artists will not be processed. This # prevents new artists from appearing suddenly. In case forced is True, # this check is skipped, since it is assumed the user wants this. - if headphones.FREEZE_DB and not forced: + if headphones.CONFIG.FREEZE_DB and not forced: artist = myDB.select("SELECT ArtistName, ArtistID FROM artists WHERE ArtistId=? OR ArtistName=?", [release_dict['artist_id'], release_dict['artist_name']]) if not artist: @@ -107,32 +108,32 @@ def verify(albumid, albumpath, Kind=None, forced=False): else: sortname = release_dict['artist_name'] - controlValueDict = {"ArtistID": release_dict['artist_id']} - newValueDict = {"ArtistName": release_dict['artist_name'], - "ArtistSortName": sortname, - "DateAdded": helpers.today(), - "Status": "Paused"} + controlValueDict = {"ArtistID": release_dict['artist_id']} + newValueDict = {"ArtistName": release_dict['artist_name'], + "ArtistSortName": sortname, + "DateAdded": helpers.today(), + "Status": "Paused"} logger.info("ArtistID: " + release_dict['artist_id'] + " , ArtistName: " + release_dict['artist_name']) - if headphones.INCLUDE_EXTRAS: + if headphones.CONFIG.INCLUDE_EXTRAS: newValueDict['IncludeExtras'] = 1 - newValueDict['Extras'] = headphones.EXTRAS + newValueDict['Extras'] = headphones.CONFIG.EXTRAS myDB.upsert("artists", newValueDict, controlValueDict) logger.info(u"Now adding album: " + release_dict['title']) - controlValueDict = {"AlbumID": albumid} + controlValueDict = {"AlbumID": albumid} - newValueDict = {"ArtistID": release_dict['artist_id'], - "ReleaseID": albumid, - "ArtistName": release_dict['artist_name'], - "AlbumTitle": release_dict['title'], - "AlbumASIN": release_dict['asin'], - "ReleaseDate": release_dict['date'], - "DateAdded": helpers.today(), - "Type": release_dict['rg_type'], - "Status": "Snatched" + newValueDict = {"ArtistID": release_dict['artist_id'], + "ReleaseID": albumid, + "ArtistName": release_dict['artist_name'], + "AlbumTitle": release_dict['title'], + "AlbumASIN": release_dict['asin'], + "ReleaseDate": release_dict['date'], + "DateAdded": helpers.today(), + "Type": release_dict['rg_type'], + "Status": "Snatched" } myDB.upsert("albums", newValueDict, controlValueDict) @@ -141,22 +142,22 @@ def verify(albumid, albumpath, Kind=None, forced=False): myDB.action('DELETE from tracks WHERE AlbumID=?', [albumid]) for track in release_dict['tracks']: - controlValueDict = {"TrackID": track['id'], - "AlbumID": albumid} + controlValueDict = {"TrackID": track['id'], + "AlbumID": albumid} - newValueDict = {"ArtistID": release_dict['artist_id'], - "ArtistName": release_dict['artist_name'], - "AlbumTitle": release_dict['title'], - "AlbumASIN": release_dict['asin'], - "TrackTitle": track['title'], - "TrackDuration": track['duration'], - "TrackNumber": track['number'] + newValueDict = {"ArtistID": release_dict['artist_id'], + "ArtistName": release_dict['artist_name'], + "AlbumTitle": release_dict['title'], + "AlbumASIN": release_dict['asin'], + "TrackTitle": track['title'], + "TrackDuration": track['duration'], + "TrackNumber": track['number'] } myDB.upsert("tracks", newValueDict, controlValueDict) - controlValueDict = {"ArtistID": release_dict['artist_id']} - newValueDict = {"Status": "Paused"} + controlValueDict = {"ArtistID": release_dict['artist_id']} + newValueDict = {"Status": "Paused"} myDB.upsert("artists", newValueDict, controlValueDict) logger.info(u"Addition complete for: " + release_dict['title'] + " - " + release_dict['artist_name']) @@ -167,7 +168,7 @@ def verify(albumid, albumpath, Kind=None, forced=False): downloaded_track_list = [] downloaded_cuecount = 0 - for r,d,f in os.walk(albumpath): + for r, d, f in os.walk(albumpath): for files in f: if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): downloaded_track_list.append(os.path.join(r, files)) @@ -178,66 +179,18 @@ def verify(albumid, albumpath, Kind=None, forced=False): logger.info("Looks like " + os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + " isn't complete yet. Will try again on the next run") return - - # use xld to split cue - - if headphones.ENCODER == 'xld' and headphones.MUSIC_ENCODER and downloaded_cuecount and downloaded_cuecount >= len(downloaded_track_list): - - import getXldProfile - - (xldProfile, xldFormat, xldBitrate) = getXldProfile.getXldProfile(headphones.XLDPROFILE) - if not xldFormat: - logger.info(u'Details for xld profile "%s" not found, cannot split cue' % (xldProfile)) + # Split cue + if headphones.CONFIG.CUE_SPLIT and downloaded_cuecount and downloaded_cuecount >= len(downloaded_track_list): + if headphones.CONFIG.KEEP_TORRENT_FILES and Kind == "torrent": + albumpath = helpers.preserve_torrent_direcory(albumpath) + if albumpath and helpers.cue_split(albumpath): + downloaded_track_list = helpers.get_downloaded_track_list(albumpath) else: - if headphones.ENCODERFOLDER: - xldencoder = os.path.join(headphones.ENCODERFOLDER, 'xld') - else: - xldencoder = os.path.join('/Applications','xld') - - for r,d,f in os.walk(albumpath): - xldfolder = r - xldfile = '' - xldcue = '' - for file in f: - if any(file.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS) and not xldfile: - xldfile = os.path.join(r, file) - elif file.lower().endswith('.cue') and not xldcue: - xldcue = os.path.join(r, file) - - if xldfile and xldcue and xldfolder: - xldcmd = xldencoder - xldcmd = xldcmd + ' "' + xldfile + '"' - xldcmd = xldcmd + ' -c' - xldcmd = xldcmd + ' "' + xldcue + '"' - xldcmd = xldcmd + ' --profile' - xldcmd = xldcmd + ' "' + xldProfile + '"' - xldcmd = xldcmd + ' -o' - xldcmd = xldcmd + ' "' + xldfolder + '"' - logger.info(u"Cue found, splitting file " + xldfile.decode(headphones.SYS_ENCODING, 'replace')) - logger.debug(xldcmd) - os.system(xldcmd) - - # count files, should now be more than original if xld successfully split - - new_downloaded_track_list_count = 0 - for r,d,f in os.walk(albumpath): - for file in f: - if any(file.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): - new_downloaded_track_list_count += 1 - - if new_downloaded_track_list_count > len(downloaded_track_list): - - # rename original unsplit files - for downloaded_track in downloaded_track_list: - os.rename(downloaded_track, downloaded_track + '.original') - - #reload - - downloaded_track_list = [] - for r,d,f in os.walk(albumpath): - for file in f: - if any(file.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): - downloaded_track_list.append(os.path.join(r, file)) + myDB.action('UPDATE snatched SET status = "Unprocessed" WHERE status NOT LIKE "Seed%" and AlbumID=?', [albumid]) + processed = re.search(r' \(Unprocessed\)(?:\[\d+\])?', albumpath) + if not processed: + renameUnprocessedFolder(albumpath) + return # test #1: metadata - usually works logger.debug('Verifying metadata...') @@ -245,7 +198,7 @@ def verify(albumid, albumpath, Kind=None, forced=False): for downloaded_track in downloaded_track_list: try: f = MediaFile(downloaded_track) - except Exception, e: + except Exception as e: logger.info(u"Exception from MediaFile for: " + downloaded_track.decode(headphones.SYS_ENCODING, 'replace') + u" : " + unicode(e)) continue @@ -295,7 +248,7 @@ def verify(albumid, albumpath, Kind=None, forced=False): for track in tracks: try: - db_track_duration += track['TrackDuration']/1000 + db_track_duration += track['TrackDuration'] / 1000 except: downloaded_track_duration = False break @@ -324,18 +277,19 @@ def verify(albumid, albumpath, Kind=None, forced=False): else: logger.info(u"Already marked as unprocessed: " + albumpath.decode(headphones.SYS_ENCODING, 'replace')) + def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, Kind=None): logger.info('Starting post-processing for: %s - %s' % (release['ArtistName'], release['AlbumTitle'])) # Check to see if we're preserving the torrent dir - if headphones.KEEP_TORRENT_FILES and Kind=="torrent": + if headphones.CONFIG.KEEP_TORRENT_FILES and Kind == "torrent" and 'headphones-modified' not in albumpath: new_folder = os.path.join(albumpath, 'headphones-modified'.encode(headphones.SYS_ENCODING, 'replace')) logger.info("Copying files to 'headphones-modified' subfolder to preserve downloaded files for seeding") try: shutil.copytree(albumpath, new_folder) # Update the album path with the new location albumpath = new_folder - except Exception, e: + except Exception as e: logger.warn("Cannot copy/move files to temp folder: " + new_folder.decode(headphones.SYS_ENCODING, 'replace') + ". Not continuing. Error: " + str(e)) return @@ -343,20 +297,20 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, # Could probably just throw in the "headphones-modified" folder, # but this is good to make sure we're not counting files that may have failed to move downloaded_track_list = [] - downloaded_cuecount = 0 - for r,d,f in os.walk(albumpath): + for r, d, f in os.walk(albumpath): for files in f: if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): downloaded_track_list.append(os.path.join(r, files)) - elif files.lower().endswith('.cue'): - downloaded_cuecount += 1 # Check if files are valid media files and are writeable, before the steps # below are executed. This simplifies errors and prevents unfinished steps. for downloaded_track in downloaded_track_list: try: - media_file = MediaFile(downloaded_track) + f = MediaFile(downloaded_track) + if f is None: + # this test is just to keep pyflakes from complaining about an unused variable + return except (FileTypeError, UnreadableFileError): logger.error("Track file is not a valid media file: %s. Not " \ "continuing.", downloaded_track.decode( @@ -369,30 +323,30 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, # If one of the options below is set, it will access/touch/modify the # files, which requires write permissions. This step just check this, so # it will not try and fail lateron, with strange exceptions. - if headphones.EMBED_ALBUM_ART or headphones.CLEANUP_FILES or \ - headphones.ADD_ALBUM_ART or headphones.CORRECT_METADATA or \ - headphones.EMBED_LYRICS or headphones.RENAME_FILES or \ - headphones.MOVE_FILES: + if headphones.CONFIG.EMBED_ALBUM_ART or headphones.CONFIG.CLEANUP_FILES or \ + headphones.CONFIG.ADD_ALBUM_ART or headphones.CONFIG.CORRECT_METADATA or \ + headphones.CONFIG.EMBED_LYRICS or headphones.CONFIG.RENAME_FILES or \ + headphones.CONFIG.MOVE_FILES: try: with open(downloaded_track, "a+b"): pass - except IOError as e: + except IOError: logger.error("Track file is not writeable. This is required " \ "for some post processing steps: %s. Not continuing.", downloaded_track.decode(headphones.SYS_ENCODING, "replace")) return #start encoding - if headphones.MUSIC_ENCODER: - downloaded_track_list=music_encoder.encode(albumpath) + if headphones.CONFIG.MUSIC_ENCODER: + downloaded_track_list = music_encoder.encode(albumpath) if not downloaded_track_list: return artwork = None album_art_path = albumart.getAlbumArt(albumid) - if headphones.EMBED_ALBUM_ART or headphones.ADD_ALBUM_ART: + if headphones.CONFIG.EMBED_ALBUM_ART or headphones.CONFIG.ADD_ALBUM_ART: if album_art_path: artwork = request.request_content(album_art_path) @@ -406,31 +360,31 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, artwork = False logger.info("No suitable album art found from Last.FM. Not adding album art") - if headphones.EMBED_ALBUM_ART and artwork: + if headphones.CONFIG.EMBED_ALBUM_ART and artwork: embedAlbumArt(artwork, downloaded_track_list) - if headphones.CLEANUP_FILES: + if headphones.CONFIG.CLEANUP_FILES: cleanupFiles(albumpath) - if headphones.KEEP_NFO: + if headphones.CONFIG.KEEP_NFO: renameNFO(albumpath) - if headphones.ADD_ALBUM_ART and artwork: + if headphones.CONFIG.ADD_ALBUM_ART and artwork: addAlbumArt(artwork, albumpath, release) - if headphones.CORRECT_METADATA: + if headphones.CONFIG.CORRECT_METADATA: correctMetadata(albumid, release, downloaded_track_list) - if headphones.EMBED_LYRICS: + if headphones.CONFIG.EMBED_LYRICS: embedLyrics(downloaded_track_list) - if headphones.RENAME_FILES: + if headphones.CONFIG.RENAME_FILES: renameFiles(albumpath, downloaded_track_list, release) - if headphones.MOVE_FILES and not headphones.DESTINATION_DIR: + if headphones.CONFIG.MOVE_FILES and not headphones.CONFIG.DESTINATION_DIR: logger.error('No DESTINATION_DIR has been set. Set "Destination Directory" to the parent directory you want to move the files to') albumpaths = [albumpath] - elif headphones.MOVE_FILES and headphones.DESTINATION_DIR: + elif headphones.CONFIG.MOVE_FILES and headphones.CONFIG.DESTINATION_DIR: albumpaths = moveFiles(albumpath, release, tracks) else: albumpaths = [albumpath] @@ -442,13 +396,13 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, myDB.action('UPDATE snatched SET status = "Processed" WHERE Status NOT LIKE "Seed%" and AlbumID=?', [albumid]) # Check if torrent has finished seeding - if headphones.TORRENT_DOWNLOADER == 1 or headphones.TORRENT_DOWNLOADER == 2: + if headphones.CONFIG.TORRENT_DOWNLOADER == 1 or headphones.CONFIG.TORRENT_DOWNLOADER == 2: seed_snatched = myDB.action('SELECT * from snatched WHERE Status="Seed_Snatched" and AlbumID=?', [albumid]).fetchone() if seed_snatched: hash = seed_snatched['FolderName'] torrent_removed = False logger.info(u'%s - %s. Checking if torrent has finished seeding and can be removed' % (release['ArtistName'], release['AlbumTitle'])) - if headphones.TORRENT_DOWNLOADER == 1: + if headphones.CONFIG.TORRENT_DOWNLOADER == 1: torrent_removed = transmission.removeTorrent(hash, True) else: torrent_removed = utorrent.removeTorrent(hash, True) @@ -468,89 +422,90 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, pushmessage = release['ArtistName'] + ' - ' + release['AlbumTitle'] statusmessage = "Download and Postprocessing completed" - if headphones.GROWL_ENABLED: + if headphones.CONFIG.GROWL_ENABLED: logger.info(u"Growl request") growl = notifiers.GROWL() growl.notify(pushmessage, statusmessage) - if headphones.PROWL_ENABLED: + if headphones.CONFIG.PROWL_ENABLED: logger.info(u"Prowl request") prowl = notifiers.PROWL() prowl.notify(pushmessage, statusmessage) - if headphones.XBMC_ENABLED: + if headphones.CONFIG.XBMC_ENABLED: xbmc = notifiers.XBMC() - if headphones.XBMC_UPDATE: + if headphones.CONFIG.XBMC_UPDATE: xbmc.update() - if headphones.XBMC_NOTIFY: + if headphones.CONFIG.XBMC_NOTIFY: xbmc.notify(release['ArtistName'], release['AlbumTitle'], album_art_path) - if headphones.LMS_ENABLED: + if headphones.CONFIG.LMS_ENABLED: lms = notifiers.LMS() lms.update() - if headphones.PLEX_ENABLED: + if headphones.CONFIG.PLEX_ENABLED: plex = notifiers.Plex() - if headphones.PLEX_UPDATE: + if headphones.CONFIG.PLEX_UPDATE: plex.update() - if headphones.PLEX_NOTIFY: + if headphones.CONFIG.PLEX_NOTIFY: plex.notify(release['ArtistName'], release['AlbumTitle'], album_art_path) - if headphones.NMA_ENABLED: + if headphones.CONFIG.NMA_ENABLED: nma = notifiers.NMA() nma.notify(release['ArtistName'], release['AlbumTitle']) - if headphones.PUSHALOT_ENABLED: + if headphones.CONFIG.PUSHALOT_ENABLED: logger.info(u"Pushalot request") pushalot = notifiers.PUSHALOT() pushalot.notify(pushmessage, statusmessage) - if headphones.SYNOINDEX_ENABLED: + if headphones.CONFIG.SYNOINDEX_ENABLED: syno = notifiers.Synoindex() for albumpath in albumpaths: syno.notify(albumpath) - if headphones.PUSHOVER_ENABLED: + if headphones.CONFIG.PUSHOVER_ENABLED: logger.info(u"Pushover request") pushover = notifiers.PUSHOVER() pushover.notify(pushmessage, "Headphones") - if headphones.PUSHBULLET_ENABLED: + if headphones.CONFIG.PUSHBULLET_ENABLED: logger.info(u"PushBullet request") pushbullet = notifiers.PUSHBULLET() pushbullet.notify(pushmessage, "Download and Postprocessing completed") - if headphones.TWITTER_ENABLED: + if headphones.CONFIG.TWITTER_ENABLED: logger.info(u"Sending Twitter notification") twitter = notifiers.TwitterNotifier() twitter.notify_download(pushmessage) - if headphones.OSX_NOTIFY_ENABLED: + if headphones.CONFIG.OSX_NOTIFY_ENABLED: logger.info(u"Sending OS X notification") osx_notify = notifiers.OSX_NOTIFY() osx_notify.notify(release['ArtistName'], release['AlbumTitle'], statusmessage) - if headphones.BOXCAR_ENABLED: + if headphones.CONFIG.BOXCAR_ENABLED: logger.info(u"Sending Boxcar2 notification") boxcar = notifiers.BOXCAR() boxcar.notify('Headphones processed: ' + pushmessage, statusmessage, release['AlbumID']) - if headphones.SUBSONIC_ENABLED: + if headphones.CONFIG.SUBSONIC_ENABLED: logger.info(u"Sending Subsonic update") subsonic = notifiers.SubSonicNotifier() subsonic.notify(albumpaths) - if headphones.MPC_ENABLED: + if headphones.CONFIG.MPC_ENABLED: mpc = notifiers.MPC() mpc.notify() + def embedAlbumArt(artwork, downloaded_track_list): logger.info('Embedding album art') @@ -566,10 +521,11 @@ def embedAlbumArt(artwork, downloaded_track_list): try: f.art = artwork f.save() - except Exception, e: + except Exception as e: logger.error(u'Error embedding album art to: %s. Error: %s' % (downloaded_track.decode(headphones.SYS_ENCODING, 'replace'), str(e))) continue + def addAlbumArt(artwork, albumpath, release): logger.info('Adding album art to folder') @@ -578,19 +534,19 @@ def addAlbumArt(artwork, albumpath, release): except TypeError: year = '' - values = { '$Artist': release['ArtistName'], - '$Album': release['AlbumTitle'], - '$Year': year, - '$artist': release['ArtistName'].lower(), - '$album': release['AlbumTitle'].lower(), - '$year': year + values = {'$Artist': release['ArtistName'], + '$Album': release['AlbumTitle'], + '$Year': year, + '$artist': release['ArtistName'].lower(), + '$album': release['AlbumTitle'].lower(), + '$year': year } - album_art_name = helpers.replace_all(headphones.ALBUM_ART_FORMAT.strip(), values) + ".jpg" + album_art_name = helpers.replace_all(headphones.CONFIG.ALBUM_ART_FORMAT.strip(), values) + ".jpg" album_art_name = helpers.replace_illegal_chars(album_art_name).encode(headphones.SYS_ENCODING, 'replace') - if headphones.FILE_UNDERSCORES: + if headphones.CONFIG.FILE_UNDERSCORES: album_art_name = album_art_name.replace(' ', '_') if album_art_name.startswith('.'): @@ -603,10 +559,11 @@ def addAlbumArt(artwork, albumpath, release): logger.error('Error saving album art: %s', e) return + def cleanupFiles(albumpath): logger.info('Cleaning up files') - for r,d,f in os.walk(albumpath): + for r, d, f in os.walk(albumpath): for files in f: if not any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): logger.debug('Removing: %s' % files) @@ -615,10 +572,11 @@ def cleanupFiles(albumpath): except Exception as e: logger.error(u'Could not remove file: %s. Error: %s' % (files.decode(headphones.SYS_ENCODING, 'replace'), e)) + def renameNFO(albumpath): logger.info('Renaming NFO') - for r,d,f in os.walk(albumpath): + for r, d, f in os.walk(albumpath): for file in f: if file.lower().endswith('.nfo'): logger.debug('Renaming: "%s" to "%s"' % (file.decode(headphones.SYS_ENCODING, 'replace'), file.decode(headphones.SYS_ENCODING, 'replace') + '-orig')) @@ -628,6 +586,7 @@ def renameNFO(albumpath): except Exception as e: logger.error(u'Could not rename file: %s. Error: %s' % (os.path.join(r, file).decode(headphones.SYS_ENCODING, 'replace'), e)) + def moveFiles(albumpath, release, tracks): logger.info("Moving files: %s" % albumpath) try: @@ -637,7 +596,7 @@ def moveFiles(albumpath, release, tracks): artist = release['ArtistName'].replace('/', '_') album = release['AlbumTitle'].replace('/', '_') - if headphones.FILE_UNDERSCORES: + if headphones.CONFIG.FILE_UNDERSCORES: artist = artist.replace(' ', '_') album = album.replace(' ', '_') @@ -653,32 +612,32 @@ def moveFiles(albumpath, release, tracks): else: firstchar = sortname[0] - for r,d,f in os.walk(albumpath): + for r, d, f in os.walk(albumpath): try: origfolder = os.path.basename(os.path.normpath(r).decode(headphones.SYS_ENCODING, 'replace')) except: origfolder = u'' - values = { '$Artist': artist, + values = {'$Artist': artist, '$SortArtist': sortname, - '$Album': album, - '$Year': year, - '$Type': releasetype, + '$Album': album, + '$Year': year, + '$Type': releasetype, '$OriginalFolder': origfolder, - '$First': firstchar.upper(), - '$artist': artist.lower(), + '$First': firstchar.upper(), + '$artist': artist.lower(), '$sortartist': sortname.lower(), - '$album': album.lower(), - '$year': year, - '$type': releasetype.lower(), - '$first': firstchar.lower(), + '$album': album.lower(), + '$year': year, + '$type': releasetype.lower(), + '$first': firstchar.lower(), '$originalfolder': origfolder.lower() } - folder = helpers.replace_all(headphones.FOLDER_FORMAT.strip(), values, normalize=True) + folder = helpers.replace_all(headphones.CONFIG.FOLDER_FORMAT.strip(), values, normalize=True) folder = helpers.replace_illegal_chars(folder, type="folder") - folder = folder.replace('./', '_/').replace('/.','/_') + folder = folder.replace('./', '_/').replace('/.', '/_') if folder.endswith('.'): folder = folder[:-1] + '_' @@ -692,7 +651,7 @@ def moveFiles(albumpath, release, tracks): lossy_media = False lossless_media = False - for r,d,f in os.walk(albumpath): + for r, d, f in os.walk(albumpath): for files in f: files_to_move.append(os.path.join(r, files)) if any(files.lower().endswith('.' + x.lower()) for x in headphones.LOSSY_MEDIA_FORMATS): @@ -704,11 +663,11 @@ def moveFiles(albumpath, release, tracks): make_lossy_folder = False make_lossless_folder = False - lossy_destination_path = os.path.normpath(os.path.join(headphones.DESTINATION_DIR, folder)).encode(headphones.SYS_ENCODING, 'replace') - lossless_destination_path = os.path.normpath(os.path.join(headphones.LOSSLESS_DESTINATION_DIR, folder)).encode(headphones.SYS_ENCODING, 'replace') + lossy_destination_path = os.path.normpath(os.path.join(headphones.CONFIG.DESTINATION_DIR, folder)).encode(headphones.SYS_ENCODING, 'replace') + lossless_destination_path = os.path.normpath(os.path.join(headphones.CONFIG.LOSSLESS_DESTINATION_DIR, folder)).encode(headphones.SYS_ENCODING, 'replace') # If they set a destination dir for lossless media, only create the lossy folder if there is lossy media - if headphones.LOSSLESS_DESTINATION_DIR: + if headphones.CONFIG.LOSSLESS_DESTINATION_DIR: if lossy_media: make_lossy_folder = True if lossless_media: @@ -717,7 +676,7 @@ def moveFiles(albumpath, release, tracks): else: make_lossy_folder = True - last_folder = headphones.FOLDER_FORMAT.strip().split('/')[-1] + last_folder = headphones.CONFIG.FOLDER_FORMAT.strip().split('/')[-1] if make_lossless_folder: # Only rename the folder if they use the album name, otherwise merge into existing folder @@ -725,20 +684,20 @@ def moveFiles(albumpath, release, tracks): create_duplicate_folder = False - if headphones.REPLACE_EXISTING_FOLDERS: + if headphones.CONFIG.REPLACE_EXISTING_FOLDERS: try: shutil.rmtree(lossless_destination_path) - except Exception, e: + except Exception as e: logger.error("Error deleting existing folder: %s. Creating duplicate folder. Error: %s" % (lossless_destination_path.decode(headphones.SYS_ENCODING, 'replace'), e)) create_duplicate_folder = True - if not headphones.REPLACE_EXISTING_FOLDERS or create_duplicate_folder: + if not headphones.CONFIG.REPLACE_EXISTING_FOLDERS or create_duplicate_folder: temp_folder = folder i = 1 while True: newfolder = temp_folder + '[%i]' % i - lossless_destination_path = os.path.normpath(os.path.join(headphones.LOSSLESS_DESTINATION_DIR, newfolder)).encode(headphones.SYS_ENCODING, 'replace') + lossless_destination_path = os.path.normpath(os.path.join(headphones.CONFIG.LOSSLESS_DESTINATION_DIR, newfolder)).encode(headphones.SYS_ENCODING, 'replace') if os.path.exists(lossless_destination_path): i += 1 else: @@ -748,7 +707,7 @@ def moveFiles(albumpath, release, tracks): if not os.path.exists(lossless_destination_path): try: os.makedirs(lossless_destination_path) - except Exception, e: + except Exception as e: logger.error('Could not create lossless folder for %s. (Error: %s)' % (release['AlbumTitle'], e)) if not make_lossy_folder: return [albumpath] @@ -758,20 +717,20 @@ def moveFiles(albumpath, release, tracks): create_duplicate_folder = False - if headphones.REPLACE_EXISTING_FOLDERS: + if headphones.CONFIG.REPLACE_EXISTING_FOLDERS: try: shutil.rmtree(lossy_destination_path) - except Exception, e: + except Exception as e: logger.error("Error deleting existing folder: %s. Creating duplicate folder. Error: %s" % (lossy_destination_path.decode(headphones.SYS_ENCODING, 'replace'), e)) create_duplicate_folder = True - if not headphones.REPLACE_EXISTING_FOLDERS or create_duplicate_folder: + if not headphones.CONFIG.REPLACE_EXISTING_FOLDERS or create_duplicate_folder: temp_folder = folder i = 1 while True: newfolder = temp_folder + '[%i]' % i - lossy_destination_path = os.path.normpath(os.path.join(headphones.DESTINATION_DIR, newfolder)).encode(headphones.SYS_ENCODING, 'replace') + lossy_destination_path = os.path.normpath(os.path.join(headphones.CONFIG.DESTINATION_DIR, newfolder)).encode(headphones.SYS_ENCODING, 'replace') if os.path.exists(lossy_destination_path): i += 1 else: @@ -781,7 +740,7 @@ def moveFiles(albumpath, release, tracks): if not os.path.exists(lossy_destination_path): try: os.makedirs(lossy_destination_path) - except Exception, e: + except Exception as e: logger.error('Could not create folder for %s. Not moving: %s' % (release['AlbumTitle'], e)) return [albumpath] @@ -809,7 +768,7 @@ def moveFiles(albumpath, release, tracks): if moved_to_lossy_folder or moved_to_lossless_folder: try: os.remove(file_to_move) - except Exception, e: + except Exception as e: logger.error("Error deleting file '" + file_to_move.decode(headphones.SYS_ENCODING, 'replace') + "' from source directory") else: logger.error("Error copying '" + file_to_move.decode(headphones.SYS_ENCODING, 'replace') + "'. Not deleting from download directory") @@ -829,10 +788,10 @@ def moveFiles(albumpath, release, tracks): temp_fs = [] if make_lossless_folder: - temp_fs.append(headphones.LOSSLESS_DESTINATION_DIR) + temp_fs.append(headphones.CONFIG.LOSSLESS_DESTINATION_DIR) if make_lossy_folder: - temp_fs.append(headphones.DESTINATION_DIR) + temp_fs.append(headphones.CONFIG.DESTINATION_DIR) for temp_f in temp_fs: @@ -841,14 +800,14 @@ def moveFiles(albumpath, release, tracks): temp_f = os.path.join(temp_f, f) try: - os.chmod(os.path.normpath(temp_f).encode(headphones.SYS_ENCODING, 'replace'), int(headphones.FOLDER_PERMISSIONS, 8)) - except Exception, e: + os.chmod(os.path.normpath(temp_f).encode(headphones.SYS_ENCODING, 'replace'), int(headphones.CONFIG.FOLDER_PERMISSIONS, 8)) + except Exception as e: logger.error("Error trying to change permissions on folder: %s. %s", temp_f, e) # If we failed to move all the files out of the directory, this will fail too try: shutil.rmtree(albumpath) - except Exception, e: + except Exception as e: logger.error('Could not remove directory: %s. %s', albumpath, e) destination_paths = [] @@ -860,6 +819,7 @@ def moveFiles(albumpath, release, tracks): return destination_paths + def correctMetadata(albumid, release, downloaded_track_list): logger.info('Preparing to write metadata to tracks....') @@ -877,7 +837,7 @@ def correctMetadata(albumid, release, downloaded_track_list): lossy_items.append(beets.library.Item.from_path(downloaded_track)) else: logger.warn("Skipping: %s because it is not a mutagen friendly file format", downloaded_track.decode(headphones.SYS_ENCODING, 'replace')) - except Exception, e: + except Exception as e: logger.error("Beets couldn't create an Item from: %s - not a media file? %s", downloaded_track.decode(headphones.SYS_ENCODING, 'replace'), str(e)) for items in [lossy_items, lossless_items]: @@ -887,7 +847,7 @@ def correctMetadata(albumid, release, downloaded_track_list): try: cur_artist, cur_album, candidates, rec = autotag.tag_album(items, search_artist=helpers.latinToAscii(release['ArtistName']), search_album=helpers.latinToAscii(release['AlbumTitle'])) - except Exception, e: + except Exception as e: logger.error('Error getting recommendation: %s. Not writing metadata', e) return if str(rec) == 'recommendation.none': @@ -910,9 +870,10 @@ def correctMetadata(albumid, release, downloaded_track_list): try: item.write() logger.info("Successfully applied metadata to: %s", item.path.decode(headphones.SYS_ENCODING, 'replace')) - except Exception, e: + except Exception as e: logger.warn("Error writing metadata to '%s': %s", item.path.decode(headphones.SYS_ENCODING, 'replace'), str(e)) + def embedLyrics(downloaded_track_list): logger.info('Adding lyrics') @@ -932,7 +893,7 @@ def embedLyrics(downloaded_track_list): lossy_items.append(beets.library.Item.from_path(downloaded_track)) else: logger.warn("Skipping: %s because it is not a mutagen friendly file format", downloaded_track.decode(headphones.SYS_ENCODING, 'replace')) - except Exception, e: + except Exception as e: logger.error("Beets couldn't create an Item from: %s - not a media file? %s", downloaded_track.decode(headphones.SYS_ENCODING, 'replace'), str(e)) for items in [lossy_items, lossless_items]: @@ -955,11 +916,12 @@ def embedLyrics(downloaded_track_list): item.lyrics = lyrics try: item.write() - except Exception, e: + except Exception as e: logger.error('Cannot save lyrics to: %s. Skipping', item.title) else: logger.debug('No lyrics found for track: %s', item.title) + def renameFiles(albumpath, downloaded_track_list, release): logger.info('Renaming files') try: @@ -1006,30 +968,29 @@ def renameFiles(albumpath, downloaded_track_list, release): else: sortname = artistname - values = { '$Disc': discnumber, - '$Track': tracknumber, - '$Title': title, - '$Artist': artistname, - '$SortArtist': sortname, - '$Album': release['AlbumTitle'], - '$Year': year, - '$disc': discnumber, - '$track': tracknumber, - '$title': title.lower(), - '$artist': artistname.lower(), - '$sortartist': sortname.lower(), - '$album': release['AlbumTitle'].lower(), - '$year': year + values = {'$Disc': discnumber, + '$Track': tracknumber, + '$Title': title, + '$Artist': artistname, + '$SortArtist': sortname, + '$Album': release['AlbumTitle'], + '$Year': year, + '$disc': discnumber, + '$track': tracknumber, + '$title': title.lower(), + '$artist': artistname.lower(), + '$sortartist': sortname.lower(), + '$album': release['AlbumTitle'].lower(), + '$year': year } ext = os.path.splitext(downloaded_track)[1] - new_file_name = helpers.replace_all(headphones.FILE_FORMAT.strip(), values).replace('/','_') + ext - + new_file_name = helpers.replace_all(headphones.CONFIG.FILE_FORMAT.strip(), values).replace('/', '_') + ext new_file_name = helpers.replace_illegal_chars(new_file_name).encode(headphones.SYS_ENCODING, 'replace') - if headphones.FILE_UNDERSCORES: + if headphones.CONFIG.FILE_UNDERSCORES: new_file_name = new_file_name.replace(' ', '_') if new_file_name.startswith('.'): @@ -1041,26 +1002,28 @@ def renameFiles(albumpath, downloaded_track_list, release): logger.debug("Renaming for: " + downloaded_track.decode(headphones.SYS_ENCODING, 'replace') + " is not neccessary") continue - logger.debug('Renaming %s ---> %s', downloaded_track.decode(headphones.SYS_ENCODING,'replace'), new_file_name.decode(headphones.SYS_ENCODING,'replace')) + logger.debug('Renaming %s ---> %s', downloaded_track.decode(headphones.SYS_ENCODING, 'replace'), new_file_name.decode(headphones.SYS_ENCODING, 'replace')) try: os.rename(downloaded_track, new_file) - except Exception, e: + except Exception as e: logger.error('Error renaming file: %s. Error: %s', downloaded_track.decode(headphones.SYS_ENCODING, 'replace'), e) continue + def updateFilePermissions(albumpaths): for folder in albumpaths: logger.info("Updating file permissions in %s", folder) - for r,d,f in os.walk(folder): + for r, d, f in os.walk(folder): for files in f: full_path = os.path.join(r, files) try: - os.chmod(full_path, int(headphones.FILE_PERMISSIONS, 8)) + os.chmod(full_path, int(headphones.CONFIG.FILE_PERMISSIONS, 8)) except: logger.error("Could not change permissions for file: %s", full_path) continue + def renameUnprocessedFolder(albumpath): i = 0 @@ -1077,6 +1040,7 @@ def renameUnprocessedFolder(albumpath): os.rename(albumpath, new_folder_name) return + def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None): if album_dir: @@ -1086,15 +1050,15 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None): download_dirs = [] if dir: download_dirs.append(dir.encode(headphones.SYS_ENCODING, 'replace')) - if headphones.DOWNLOAD_DIR and not dir: - download_dirs.append(headphones.DOWNLOAD_DIR.encode(headphones.SYS_ENCODING, 'replace')) - if headphones.DOWNLOAD_TORRENT_DIR and not dir: - download_dirs.append(headphones.DOWNLOAD_TORRENT_DIR.encode(headphones.SYS_ENCODING, 'replace')) + if headphones.CONFIG.DOWNLOAD_DIR and not dir: + download_dirs.append(headphones.CONFIG.DOWNLOAD_DIR.encode(headphones.SYS_ENCODING, 'replace')) + if headphones.CONFIG.DOWNLOAD_TORRENT_DIR and not dir: + download_dirs.append(headphones.CONFIG.DOWNLOAD_TORRENT_DIR.encode(headphones.SYS_ENCODING, 'replace')) # If DOWNLOAD_DIR and DOWNLOAD_TORRENT_DIR are the same, remove the duplicate to prevent us from trying to process the same folder twice. download_dirs = list(set(download_dirs)) + logger.debug('Post processing folders: %s', download_dirs) - logger.info('Checking to see if there are any folders to process in download_dir(s): %s', download_dirs) # Get a list of folders in the download_dir folders = [] @@ -1113,10 +1077,13 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None): else: folders.append(path_to_folder) - if len(folders): - logger.info('Found %i folders to process', len(folders)) - else: - logger.info('Found no folders to process in: %s', download_dirs) + # Log number of folders + if folders: + logger.info('Found %i folders to process.', len(folders)) + logger.debug('Expanded post processing folders: %s', folders) + else: + logger.info('Found no folders to process. Aborting.') + return # Parse the folder names to get artist album info myDB = db.DBConnection() @@ -1134,7 +1101,7 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None): snatched = myDB.action('SELECT AlbumID, Title, Kind, Status from snatched WHERE FolderName LIKE ?', [folder_basename]).fetchone() if snatched: - if headphones.KEEP_TORRENT_FILES and snatched['Kind'] == 'torrent' and snatched['Status'] == 'Processed': + if headphones.CONFIG.KEEP_TORRENT_FILES and snatched['Kind'] == 'torrent' and snatched['Status'] == 'Processed': logger.info('%s is a torrent folder being preserved for seeding and has already been processed. Skipping.', folder_basename) continue else: @@ -1142,68 +1109,13 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None): verify(snatched['AlbumID'], folder, snatched['Kind']) continue - # Attempt 2a: parse the folder name into a valid format + # Attempt 2: strip release group id from filename + logger.debug('Attempting to extract release group from folder name') + try: - logger.debug('Attempting to extract name, album and year from folder name') - name, album, year = helpers.extract_data(folder_basename) - except Exception as e: - name = album = year = None - - if name and album: - release = myDB.action('SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?', [name, album]).fetchone() - if release: - logger.info('Found a match in the database: %s - %s. Verifying to make sure it is the correct album', release['ArtistName'], release['AlbumTitle']) - verify(release['AlbumID'], folder) - continue - else: - logger.info('Querying MusicBrainz for the release group id for: %s - %s', name, album) - try: - rgid = mb.findAlbumID(helpers.latinToAscii(name), helpers.latinToAscii(album)) - except: - logger.error('Can not get release information for this album') - rgid = None - - if rgid: - verify(rgid, folder) - continue - else: - logger.info('No match found on MusicBrainz for: %s - %s', name, album) - - # Attempt 2b: deduce meta data into a valid format - try: - logger.debug('Attempting to extract name, album and year from metadata') - name, album, year = helpers.extract_metadata(folder) - except Exception as e: - name = album = year = None - - if name and album: - release = myDB.action('SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?', [name, album]).fetchone() - if release: - logger.info('Found a match in the database: %s - %s. Verifying to make sure it is the correct album', release['ArtistName'], release['AlbumTitle']) - verify(release['AlbumID'], folder) - continue - else: - logger.info('Querying MusicBrainz for the release group id for: %s - %s', name, album) - try: - rgid = mb.findAlbumID(helpers.latinToAscii(name), helpers.latinToAscii(album)) - except: - logger.error('Can not get release information for this album') - rgid = None - - if rgid: - verify(rgid, folder) - continue - else: - logger.info('No match found on MusicBrainz for: %s - %s', name, album) - - # Attempt 3: strip release group id from filename - try: - logger.debug('Attempting to extract release group from folder name') possible_rgid = folder_basename[-36:] - # re pattern match: [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} rgid = uuid.UUID(possible_rgid) except: - logger.info("Couldn't parse '%s' into any valid format. If adding albums from another source, they must be in an 'Artist - Album [Year]' format, or end with the musicbrainz release group id", folder_basename) rgid = possible_rgid = None if rgid: @@ -1214,11 +1126,75 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None): verify(release['AlbumID'], folder, forced=True) continue else: - logger.info('Found a (possibly) valid Musicbrainz identifier in album folder name - continuing post-processing') + logger.info('Found a (possibly) valid Musicbrainz realse group id in album folder name.') verify(rgid, folder, forced=True) continue - # Attempt 4: Hail mary. Just assume the folder name is the album name if it doesn't have a separator in it + # Attempt 3a: parse the folder name into a valid format + logger.debug('Attempting to extract name, album and year from folder name') + + try: + name, album, year = helpers.extract_data(folder_basename) + except Exception: + name = album = year = None + + if name and album: + release = myDB.action('SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?', [name, album]).fetchone() + if release: + logger.info('Found a match in the database: %s - %s. Verifying to make sure it is the correct album', release['ArtistName'], release['AlbumTitle']) + verify(release['AlbumID'], folder) + continue + else: + logger.info('Querying MusicBrainz for the release group id for: %s - %s', name, album) + try: + rgid = mb.findAlbumID(helpers.latinToAscii(name), helpers.latinToAscii(album)) + except: + logger.error('Can not get release information for this album') + rgid = None + + if rgid: + verify(rgid, folder) + continue + else: + logger.info('No match found on MusicBrainz for: %s - %s', name, album) + + # Attempt 3b: deduce meta data into a valid format + logger.debug('Attempting to extract name, album and year from metadata') + + try: + name, album, year = helpers.extract_metadata(folder) + except Exception: + name = album = None + + # Check if there's a cue to split + if headphones.CONFIG.CUE_SPLIT and not name and not album and helpers.cue_split(folder): + try: + name, album, year = helpers.extract_metadata(folder) + except Exception: + name = album = None + + if name and album: + release = myDB.action('SELECT AlbumID, ArtistName, AlbumTitle from albums WHERE ArtistName LIKE ? and AlbumTitle LIKE ?', [name, album]).fetchone() + if release: + logger.info('Found a match in the database: %s - %s. Verifying to make sure it is the correct album', release['ArtistName'], release['AlbumTitle']) + verify(release['AlbumID'], folder) + continue + else: + logger.info('Querying MusicBrainz for the release group id for: %s - %s', name, album) + try: + rgid = mb.findAlbumID(helpers.latinToAscii(name), helpers.latinToAscii(album)) + except: + logger.error('Can not get release information for this album') + rgid = None + + if rgid: + verify(rgid, folder) + continue + else: + logger.info('No match found on MusicBrainz for: %s - %s', name, album) + + # Attempt 4: Hail mary. Just assume the folder name is the album name + # if it doesn't have a separator in it logger.debug('Attempt to extract album name by assuming it is the folder name') if '-' not in folder_basename: @@ -1240,3 +1216,9 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None): continue else: logger.info('No match found on MusicBrainz for: %s - %s', name, album) + + # Fail here + logger.info("Couldn't parse '%s' into any valid format. If adding " \ + "albums from another source, they must be in an 'Artist - Album " \ + "[Year]' format, or end with the musicbrainz release group id.", + folder_basename) \ No newline at end of file diff --git a/headphones/request.py b/headphones/request.py index 1b672b1e..d52e1b39 100644 --- a/headphones/request.py +++ b/headphones/request.py @@ -27,6 +27,7 @@ import collections # Dictionary with last request times, for rate limiting. last_requests = collections.defaultdict(int) + def request_response(url, method="get", auto_raise=True, whitelist_status_code=None, rate_limit=None, **kwargs): """ @@ -47,7 +48,7 @@ def request_response(url, method="get", auto_raise=True, # Disable verification of SSL certificates if requested. Note: this could # pose a security issue! - kwargs["verify"] = headphones.VERIFY_SSL_CERT + kwargs["verify"] = bool(headphones.CONFIG.VERIFY_SSL_CERT) # Map method to the request.XXX method. This is a simple hack, but it allows # requests to apply more magic per method. See lib/requests/api.py. @@ -125,6 +126,7 @@ def request_response(url, method="get", auto_raise=True, except requests.RequestException as e: logger.error("Request raised exception: %s", e) + def request_soup(url, **kwargs): """ Wrapper for `request_response', which will return a BeatifulSoup object if @@ -137,6 +139,7 @@ def request_soup(url, **kwargs): if response is not None: return BeautifulSoup(response.content, parser) + def request_minidom(url, **kwargs): """ Wrapper for `request_response', which will return a Minidom object if no @@ -148,6 +151,7 @@ def request_minidom(url, **kwargs): if response is not None: return minidom.parseString(response.content) + def request_json(url, **kwargs): """ Wrapper for `request_response', which will decode the response as JSON @@ -175,6 +179,7 @@ def request_json(url, **kwargs): if headphones.VERBOSE: server_message(response) + def request_content(url, **kwargs): """ Wrapper for `request_response', which will return the raw content. @@ -185,6 +190,7 @@ def request_content(url, **kwargs): if response is not None: return response.content + def request_feed(url, **kwargs): """ Wrapper for `request_response', which will return a feed object. @@ -195,6 +201,7 @@ def request_feed(url, **kwargs): if response is not None: return feedparser.parse(response.content) + def server_message(response): """ Extract server message from response and log in to logger with DEBUG level. @@ -235,4 +242,4 @@ def server_message(response): if len(message) > 150: message = message[:150] + "..." - logger.debug("Server responded with message: %s", message) \ No newline at end of file + logger.debug("Server responded with message: %s", message) diff --git a/headphones/sab.py b/headphones/sab.py index 30c90db9..45565f77 100644 --- a/headphones/sab.py +++ b/headphones/sab.py @@ -19,7 +19,6 @@ import MultipartPostHandler import headphones -import datetime import cookielib import urllib2 import httplib @@ -28,20 +27,21 @@ import ast from headphones.common import USER_AGENT from headphones import logger -from headphones import notifiers, helpers +from headphones import helpers + def sendNZB(nzb): params = {} - if headphones.SAB_USERNAME: - params['ma_username'] = headphones.SAB_USERNAME - if headphones.SAB_PASSWORD: - params['ma_password'] = headphones.SAB_PASSWORD - if headphones.SAB_APIKEY: - params['apikey'] = headphones.SAB_APIKEY - if headphones.SAB_CATEGORY: - params['cat'] = headphones.SAB_CATEGORY + if headphones.CONFIG.SAB_USERNAME: + params['ma_username'] = headphones.CONFIG.SAB_USERNAME + if headphones.CONFIG.SAB_PASSWORD: + params['ma_password'] = headphones.CONFIG.SAB_PASSWORD + if headphones.CONFIG.SAB_APIKEY: + params['apikey'] = headphones.CONFIG.SAB_APIKEY + if headphones.CONFIG.SAB_CATEGORY: + params['cat'] = headphones.CONFIG.SAB_CATEGORY # if it's a normal result we just pass SAB the URL if nzb.resultType == "nzb": @@ -49,7 +49,7 @@ def sendNZB(nzb): if nzb.provider.getID() == 'newzbin': id = nzb.provider.getIDFromURL(nzb.url) if not id: - logger.info("Unable to send NZB to sab, can't find ID in URL "+str(nzb.url)) + logger.info("Unable to send NZB to sab, can't find ID in URL " + str(nzb.url)) return False params['mode'] = 'addid' params['name'] = id @@ -62,15 +62,15 @@ def sendNZB(nzb): # Sanitize the file a bit, since we can only use ascii chars with MultiPartPostHandler nzbdata = helpers.latinToAscii(nzb.extraInfo[0]) params['mode'] = 'addfile' - multiPartParams = {"nzbfile": (helpers.latinToAscii(nzb.name)+".nzb", nzbdata)} + multiPartParams = {"nzbfile": (helpers.latinToAscii(nzb.name) + ".nzb", nzbdata)} - if not headphones.SAB_HOST.startswith('http'): - headphones.SAB_HOST = 'http://' + headphones.SAB_HOST + if not headphones.CONFIG.SAB_HOST.startswith('http'): + headphones.CONFIG.SAB_HOST = 'http://' + headphones.CONFIG.SAB_HOST - if headphones.SAB_HOST.endswith('/'): - headphones.SAB_HOST = headphones.SAB_HOST[0:len(headphones.SAB_HOST)-1] + if headphones.CONFIG.SAB_HOST.endswith('/'): + headphones.CONFIG.SAB_HOST = headphones.CONFIG.SAB_HOST[0:len(headphones.CONFIG.SAB_HOST) - 1] - url = headphones.SAB_HOST + "/" + "api?" + urllib.urlencode(params) + url = headphones.CONFIG.SAB_HOST + "/" + "api?" + urllib.urlencode(params) try: @@ -87,25 +87,25 @@ def sendNZB(nzb): f = opener.open(req) - except (EOFError, IOError), e: + except (EOFError, IOError) as e: logger.error(u"Unable to connect to SAB with URL: %s" % url) return False - except httplib.InvalidURL, e: - logger.error(u"Invalid SAB host, check your config. Current host: %s" % headphones.SAB_HOST) + except httplib.InvalidURL as e: + logger.error(u"Invalid SAB host, check your config. Current host: %s" % headphones.CONFIG.SAB_HOST) return False - - except Exception, e: + + except Exception as e: logger.error(u"Error: " + str(e)) return False - - if f == None: + + if f is None: logger.info(u"No data returned from SABnzbd, NZB not sent") return False try: result = f.readlines() - except Exception, e: + except Exception as e: logger.info(u"Error trying to get result from SAB, NZB not sent: ") return False @@ -126,37 +126,38 @@ def sendNZB(nzb): else: logger.info(u"Unknown failure sending NZB to sab. Return text is: " + sabText) return False - + + def checkConfig(): - params = { 'mode' : 'get_config', - 'section' : 'misc' + params = {'mode': 'get_config', + 'section': 'misc' } - if headphones.SAB_USERNAME: - params['ma_username'] = headphones.SAB_USERNAME - if headphones.SAB_PASSWORD: - params['ma_password'] = headphones.SAB_PASSWORD - if headphones.SAB_APIKEY: - params['apikey'] = headphones.SAB_APIKEY + if headphones.CONFIG.SAB_USERNAME: + params['ma_username'] = headphones.CONFIG.SAB_USERNAME + if headphones.CONFIG.SAB_PASSWORD: + params['ma_password'] = headphones.CONFIG.SAB_PASSWORD + if headphones.CONFIG.SAB_APIKEY: + params['apikey'] = headphones.CONFIG.SAB_APIKEY - if not headphones.SAB_HOST.startswith('http'): - headphones.SAB_HOST = 'http://' + headphones.SAB_HOST + if not headphones.CONFIG.SAB_HOST.startswith('http'): + headphones.CONFIG.SAB_HOST = 'http://' + headphones.CONFIG.SAB_HOST + + if headphones.CONFIG.SAB_HOST.endswith('/'): + headphones.CONFIG.SAB_HOST = headphones.CONFIG.SAB_HOST[0:len(headphones.CONFIG.SAB_HOST) - 1] + + url = headphones.CONFIG.SAB_HOST + "/" + "api?" + urllib.urlencode(params) - if headphones.SAB_HOST.endswith('/'): - headphones.SAB_HOST = headphones.SAB_HOST[0:len(headphones.SAB_HOST)-1] - - url = headphones.SAB_HOST + "/" + "api?" + urllib.urlencode(params) - try: f = urllib.urlopen(url).read() - except Exception, e: + except Exception: logger.warn("Unable to read SABnzbd config file - cannot determine renaming options (might affect auto & forced post processing)") return (0, 0) - + config_options = ast.literal_eval(f) - + replace_spaces = config_options['misc']['replace_spaces'] replace_dots = config_options['misc']['replace_dots'] - + return (replace_spaces, replace_dots) diff --git a/headphones/searcher.py b/headphones/searcher.py index 3ea3e0d8..5de7d785 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -15,23 +15,23 @@ # NZBGet support added by CurlyMo as a part of XBian - XBMC on the Raspberry Pi -import urllib, urlparse +import urllib +import urlparse from pygazelle import api as gazelleapi from pygazelle import encoding as gazelleencoding from pygazelle import format as gazelleformat -from pygazelle import media as gazellemedia -from xml.dom import minidom from base64 import b16encode, b32decode from hashlib import sha1 -import os, re, time +import os +import re import string import shutil -import requests +import random +import headphones import subprocess import unicodedata -import headphones from headphones.common import USER_AGENT from headphones import logger, db, helpers, classes, sab, nzbget, request from headphones import utorrent, transmission, notifiers @@ -39,19 +39,142 @@ from headphones import utorrent, transmission, notifiers from bencode import bencode, bdecode import headphones.searcher_rutracker as rutrackersearch -rutracker = rutrackersearch.Rutracker() + +# Magnet to torrent services, for Black hole. Stolen from CouchPotato. +TORRENT_TO_MAGNET_SERVICES = [ + 'https://zoink.it/torrent/%s.torrent', + 'http://torrage.com/torrent/%s.torrent', + 'https://torcache.net/torrent/%s.torrent', +] # Persistent What.cd API object gazelle = None -def url_fix(s, charset='utf-8'): +# RUtracker search object +rutracker = rutrackersearch.Rutracker() + + +def fix_url(s, charset="utf-8"): + """ + Fix the URL so it is proper formatted and encoded. + """ + if isinstance(s, unicode): s = s.encode(charset, 'ignore') + scheme, netloc, path, qs, anchor = urlparse.urlsplit(s) path = urllib.quote(path, '/%') qs = urllib.quote_plus(qs, ':&=') + return urlparse.urlunsplit((scheme, netloc, path, qs, anchor)) + +def torrent_to_file(target_file, data): + """ + Write torrent data to file, and change permissions accordingly. Will return + None in case of a write error. If changing permissions fails, it will + continue anyway. + """ + + # Write data to file + try: + with open(target_file, "wb") as fp: + fp.write(data) + except IOError as e: + logger.error("Could not write torrent file '%s': %s. Skipping.", + target_file, e.message) + return + + # Try to change permissions + try: + os.chmod(target_file, int(headphones.CONFIG.FILE_PERMISSIONS, 8)) + except OSError as e: + logger.warn("Could not change permissions for file '%s': %s. " \ + "Continuing.", target_file, e.message) + + # Done + return True + + +def read_torrent_name(torrent_file, default_name=None): + """ + Read the torrent file and return the torrent name. If the torrent name + cannot be determined, it will return the `default_name`. + """ + + # Open file + try: + with open(torrent_file, "rb") as fp: + torrent_info = bdecode(fp.read()) + except IOError as e: + logger.error("Unable to open torrent file: %s", torrent_file) + return + + # Read dictionary + if torrent_info: + try: + return torrent_info["info"]["name"] + except KeyError: + if default_name: + logger.warning("Couldn't get name from torrent file: %s. " \ + "Defaulting to '%s'", e, default_name) + else: + logger.warning("Couldn't get name from torrent file: %s. No " \ + "default given", e) + + # Return default + return default_name + + +def calculate_torrent_hash(link, data=None): + """ + Calculate the torrent hash from a magnet link or data. + """ + + if link.startswith("magnet:"): + torrent_hash = re.findall("urn:btih:([\w]{32,40})", link)[0] + if len(torrent_hash) == 32: + torrent_hash = b16encode(b32decode(torrent_hash)).lower() + elif data: + info = bdecode(data)["info"] + torrent_hash = sha1(bencode(info)).hexdigest() + else: + raise ValueError("Cannot calculate torrent hash without magnet link " \ + "or data") + + return torrent_hash + + +def get_seed_ratio(provider): + """ + Return the seed ratio for the specified provider, if applicable. Defaults to + None in case of an error. + """ + + if provider == 'rutracker.org': + seed_ratio = headphones.CONFIG.RUTRACKER_RATIO + elif provider == 'Kick Ass Torrents': + seed_ratio = headphones.CONFIG.KAT_RATIO + elif provider == 'What.cd': + seed_ratio = headphones.CONFIG.WHATCD_RATIO + elif provider == 'The Pirate Bay': + seed_ratio = headphones.CONFIG.PIRATEBAY_RATIO + elif provider == 'Waffles.fm': + seed_ratio = headphones.CONFIG.WAFFLES_RATIO + elif provider == 'Mininova': + seed_ratio = headphones.CONFIG.MININOVA_RATIO + else: + seed_ratio = None + + if seed_ratio is not None: + try: + seed_ratio = float(seed_ratio) + except ValueError: + logger.warn("Could not get seed ratio for %s" % provider) + + return seed_ratio + + def searchforalbum(albumid=None, new=False, losslessOnly=False, choose_specific_download=False): myDB = db.DBConnection() @@ -80,24 +203,24 @@ def searchforalbum(albumid=None, new=False, losslessOnly=False, choose_specific_ return results else: - album = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone() logger.info('Searching for "%s - %s" since it was marked as wanted' % (album['ArtistName'], album['AlbumTitle'])) do_sorted_search(album, new, losslessOnly) logger.info('Search for Wanted albums complete') + def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): - NZB_PROVIDERS = (headphones.HEADPHONES_INDEXER or headphones.NEWZNAB or headphones.NZBSORG or headphones.OMGWTFNZBS) - NZB_DOWNLOADERS = (headphones.SAB_HOST or headphones.BLACKHOLE_DIR or headphones.NZBGET_HOST) - TORRENT_PROVIDERS = (headphones.KAT or headphones.PIRATEBAY or headphones.MININOVA or headphones.WAFFLES or headphones.RUTRACKER or headphones.WHATCD) + NZB_PROVIDERS = (headphones.CONFIG.HEADPHONES_INDEXER or headphones.CONFIG.NEWZNAB or headphones.CONFIG.NZBSORG or headphones.CONFIG.OMGWTFNZBS) + NZB_DOWNLOADERS = (headphones.CONFIG.SAB_HOST or headphones.CONFIG.BLACKHOLE_DIR or headphones.CONFIG.NZBGET_HOST) + TORRENT_PROVIDERS = (headphones.CONFIG.KAT or headphones.CONFIG.PIRATEBAY or headphones.CONFIG.MININOVA or headphones.CONFIG.WAFFLES or headphones.CONFIG.RUTRACKER or headphones.CONFIG.WHATCD) results = [] myDB = db.DBConnection() albumlength = myDB.select('SELECT sum(TrackDuration) from tracks WHERE AlbumID=?', [album['AlbumID']])[0][0] - if headphones.PREFER_TORRENTS == 0: + if headphones.CONFIG.PREFER_TORRENTS == 0: if NZB_PROVIDERS and NZB_DOWNLOADERS: results = searchNZB(album, new, losslessOnly, albumlength) @@ -105,7 +228,7 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): if not results and TORRENT_PROVIDERS: results = searchTorrent(album, new, losslessOnly, albumlength) - elif headphones.PREFER_TORRENTS == 1: + elif headphones.CONFIG.PREFER_TORRENTS == 1: if TORRENT_PROVIDERS: results = searchTorrent(album, new, losslessOnly, albumlength) @@ -132,7 +255,6 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): results = nzb_results + torrent_results - if choose_specific_download: return results @@ -147,11 +269,13 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): if data and bestqual: send_to_downloader(data, bestqual, album) + def removeDisallowedFilenameChars(filename): validFilenameChars = "-_.() %s%s" % (string.ascii_letters, string.digits) cleanedFilename = unicodedata.normalize('NFKD', filename).encode('ASCII', 'ignore').lower() return ''.join(c for c in cleanedFilename if c in validFilenameChars) + def more_filtering(results, album, albumlength, new): low_size_limit = None @@ -160,23 +284,23 @@ def more_filtering(results, album, albumlength, new): myDB = db.DBConnection() # Lossless - ignore results if target size outside bitrate range - if headphones.PREFERRED_QUALITY == 3 and albumlength and (headphones.LOSSLESS_BITRATE_FROM or headphones.LOSSLESS_BITRATE_TO): - if headphones.LOSSLESS_BITRATE_FROM: - low_size_limit = albumlength/1000 * int(headphones.LOSSLESS_BITRATE_FROM) * 128 - if headphones.LOSSLESS_BITRATE_TO: - high_size_limit = albumlength/1000 * int(headphones.LOSSLESS_BITRATE_TO) * 128 + if headphones.CONFIG.PREFERRED_QUALITY == 3 and albumlength and (headphones.CONFIG.LOSSLESS_BITRATE_FROM or headphones.CONFIG.LOSSLESS_BITRATE_TO): + if headphones.CONFIG.LOSSLESS_BITRATE_FROM: + low_size_limit = albumlength / 1000 * int(headphones.CONFIG.LOSSLESS_BITRATE_FROM) * 128 + if headphones.CONFIG.LOSSLESS_BITRATE_TO: + high_size_limit = albumlength / 1000 * int(headphones.CONFIG.LOSSLESS_BITRATE_TO) * 128 # Preferred Bitrate - ignore results if target size outside % buffer - elif headphones.PREFERRED_QUALITY == 2 and headphones.PREFERRED_BITRATE: - logger.debug('Target bitrate: %s kbps' % headphones.PREFERRED_BITRATE) + elif headphones.CONFIG.PREFERRED_QUALITY == 2 and headphones.CONFIG.PREFERRED_BITRATE: + logger.debug('Target bitrate: %s kbps' % headphones.CONFIG.PREFERRED_BITRATE) if albumlength: - targetsize = albumlength/1000 * int(headphones.PREFERRED_BITRATE) * 128 + targetsize = albumlength / 1000 * int(headphones.CONFIG.PREFERRED_BITRATE) * 128 logger.info('Target size: %s' % helpers.bytes_to_mb(targetsize)) - if headphones.PREFERRED_BITRATE_LOW_BUFFER: - low_size_limit = targetsize - (targetsize * int(headphones.PREFERRED_BITRATE_LOW_BUFFER)/100) - if headphones.PREFERRED_BITRATE_HIGH_BUFFER: - high_size_limit = targetsize + (targetsize * int(headphones.PREFERRED_BITRATE_HIGH_BUFFER)/100) - if headphones.PREFERRED_BITRATE_ALLOW_LOSSLESS: + if headphones.CONFIG.PREFERRED_BITRATE_LOW_BUFFER: + low_size_limit = targetsize - (targetsize * int(headphones.CONFIG.PREFERRED_BITRATE_LOW_BUFFER) / 100) + if headphones.CONFIG.PREFERRED_BITRATE_HIGH_BUFFER: + high_size_limit = targetsize + (targetsize * int(headphones.CONFIG.PREFERRED_BITRATE_HIGH_BUFFER) / 100) + if headphones.CONFIG.PREFERRED_BITRATE_ALLOW_LOSSLESS: allow_lossless = True newlist = [] @@ -185,7 +309,7 @@ def more_filtering(results, album, albumlength, new): normalizedAlbumArtist = removeDisallowedFilenameChars(album['ArtistName']) normalizedAlbumTitle = removeDisallowedFilenameChars(album['AlbumTitle']) - normalizedResultTitle = removeDisallowedFilenameChars(result[0]); + normalizedResultTitle = removeDisallowedFilenameChars(result[0]) artistTitleCount = normalizedResultTitle.count(normalizedAlbumArtist) if normalizedAlbumArtist in normalizedAlbumTitle and artistTitleCount < 2: @@ -215,6 +339,7 @@ def more_filtering(results, album, albumlength, new): return results + def sort_search_results(resultlist, album, new, albumlength): if new and not len(resultlist): @@ -224,8 +349,8 @@ def sort_search_results(resultlist, album, new, albumlength): # Add a priority if it has any of the preferred words temp_list = [] preferred_words = None - if headphones.PREFERRED_WORDS: - preferred_words = helpers.split_string(headphones.PREFERRED_WORDS) + if headphones.CONFIG.PREFERRED_WORDS: + preferred_words = helpers.split_string(headphones.CONFIG.PREFERRED_WORDS) for result in resultlist: priority = 0 if preferred_words: @@ -233,17 +358,17 @@ def sort_search_results(resultlist, album, new, albumlength): priority = 1 # add a search provider priority (weighted based on position) i = next((i for i, word in enumerate(preferred_words) if word in result[3].lower()), None) - if i != None: - priority += round((len(preferred_words) - i) / float(len(preferred_words)),2) + if i is not None: + priority += round((len(preferred_words) - i) / float(len(preferred_words)), 2) - temp_list.append((result[0],result[1],result[2],result[3],result[4],priority)) + temp_list.append((result[0], result[1], result[2], result[3], result[4], priority)) resultlist = temp_list - if headphones.PREFERRED_QUALITY == 2 and headphones.PREFERRED_BITRATE: + if headphones.CONFIG.PREFERRED_QUALITY == 2 and headphones.CONFIG.PREFERRED_BITRATE: try: - targetsize = albumlength/1000 * int(headphones.PREFERRED_BITRATE) * 128 + targetsize = albumlength / 1000 * int(headphones.CONFIG.PREFERRED_BITRATE) * 128 if not targetsize: logger.info('No track information for %s - %s. Defaulting to highest quality' % (album['ArtistName'], album['AlbumTitle'])) @@ -265,12 +390,12 @@ def sort_search_results(resultlist, album, new, albumlength): finallist = sorted(newlist, key=lambda title: (-title[5], title[6])) - if not len(finallist) and len(flac_list) and headphones.PREFERRED_BITRATE_ALLOW_LOSSLESS: + if not len(finallist) and len(flac_list) and headphones.CONFIG.PREFERRED_BITRATE_ALLOW_LOSSLESS: logger.info("Since there were no appropriate lossy matches (and at least one lossless match, going to use lossless instead") finallist = sorted(flac_list, key=lambda title: (title[5], int(title[1])), reverse=True) - except Exception as e: + except Exception: logger.exception('Unhandled exception') - logger.info('No track information for %s - %s. Defaulting to highest quality', (album['ArtistName'], album['AlbumTitle'])) + logger.info('No track information for %s - %s. Defaulting to highest quality', album['ArtistName'], album['AlbumTitle']) finallist = sorted(resultlist, key=lambda title: (title[5], int(title[1])), reverse=True) @@ -284,6 +409,7 @@ def sort_search_results(resultlist, album, new, albumlength): return finallist + def get_year_from_release_date(release_date): try: @@ -293,13 +419,12 @@ def get_year_from_release_date(release_date): return year -def searchNZB(album, new=False, losslessOnly=False, albumlength=None): - albumid = album['AlbumID'] +def searchNZB(album, new=False, losslessOnly=False, albumlength=None): reldate = album['ReleaseDate'] year = get_year_from_release_date(reldate) - dic = {'...':'', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':'', '*':'', '.':'', ':':''} + dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': '', '*': '', '.': '', ':': ''} cleanalbum = helpers.latinToAscii(helpers.replace_all(album['AlbumTitle'], dic)).strip() cleanartist = helpers.latinToAscii(helpers.replace_all(album['ArtistName'], dic)).strip() @@ -323,7 +448,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None): artistterm = re.sub('[\.\-\/]', ' ', cleanartist).encode('utf-8') # If Preferred Bitrate and High Limit and Allow Lossless then get both lossy and lossless - if headphones.PREFERRED_QUALITY == 2 and headphones.PREFERRED_BITRATE and headphones.PREFERRED_BITRATE_HIGH_BUFFER and headphones.PREFERRED_BITRATE_ALLOW_LOSSLESS: + if headphones.CONFIG.PREFERRED_QUALITY == 2 and headphones.CONFIG.PREFERRED_BITRATE and headphones.CONFIG.PREFERRED_BITRATE_HIGH_BUFFER and headphones.CONFIG.PREFERRED_BITRATE_ALLOW_LOSSLESS: allow_lossless = True else: allow_lossless = False @@ -332,12 +457,12 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None): resultlist = [] - if headphones.HEADPHONES_INDEXER: + if headphones.CONFIG.HEADPHONES_INDEXER: provider = "headphones" - if headphones.PREFERRED_QUALITY == 3 or losslessOnly: + if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly: categories = "3040" - elif headphones.PREFERRED_QUALITY == 1 or allow_lossless: + elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless: categories = "3040,3010" else: categories = "3010" @@ -349,19 +474,19 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None): # Request results logger.info('Parsing results from Headphones Indexer') - headers = { 'User-Agent': USER_AGENT } + headers = {'User-Agent': USER_AGENT} params = { "t": "search", "cat": categories, "apikey": '964d601959918a578a670984bdee9357', - "maxage": headphones.USENET_RETENTION, + "maxage": headphones.CONFIG.USENET_RETENTION, "q": term } data = request.request_feed( url="http://indexer.codeshy.com/api", params=params, headers=headers, - auth=(headphones.HPUSER, headphones.HPPASS) + auth=(headphones.CONFIG.HPUSER, headphones.CONFIG.HPPASS) ) # Process feed @@ -381,21 +506,20 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None): except Exception as e: logger.error(u"An unknown error occurred trying to parse the feed: %s" % e) - if headphones.NEWZNAB: + if headphones.CONFIG.NEWZNAB: provider = "newznab" newznab_hosts = [] - if headphones.NEWZNAB_HOST and headphones.NEWZNAB_ENABLED: + if headphones.CONFIG.NEWZNAB_HOST and headphones.CONFIG.NEWZNAB_ENABLED: + newznab_hosts.append((headphones.CONFIG.NEWZNAB_HOST, headphones.CONFIG.NEWZNAB_APIKEY, headphones.CONFIG.NEWZNAB_ENABLED)) - newznab_hosts.append((headphones.NEWZNAB_HOST, headphones.NEWZNAB_APIKEY, headphones.NEWZNAB_ENABLED)) - - for newznab_host in headphones.EXTRA_NEWZNABS: + for newznab_host in headphones.CONFIG.get_extra_newznabs(): if newznab_host[2] == '1' or newznab_host[2] == 1: newznab_hosts.append(newznab_host) - if headphones.PREFERRED_QUALITY == 3 or losslessOnly: + if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly: categories = "3040" - elif headphones.PREFERRED_QUALITY == 1 or allow_lossless: + elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless: categories = "3040,3010" else: categories = "3010" @@ -422,12 +546,12 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None): # Request results logger.info('Parsing results from %s', newznab_host[0]) - headers = { 'User-Agent': USER_AGENT } + headers = {'User-Agent': USER_AGENT} params = { "t": "search", "apikey": newznab_host[1], "cat": categories, - "maxage": headphones.USENET_RETENTION, + "maxage": headphones.CONFIG.USENET_RETENTION, "q": term } @@ -455,11 +579,11 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None): except Exception as e: logger.exception("An unknown error occurred trying to parse the feed: %s" % e) - if headphones.NZBSORG: + if headphones.CONFIG.NZBSORG: provider = "nzbsorg" - if headphones.PREFERRED_QUALITY == 3 or losslessOnly: + if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly: categories = "3040" - elif headphones.PREFERRED_QUALITY == 1 or allow_lossless: + elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless: categories = "3040,3010" else: categories = "3010" @@ -471,12 +595,12 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None): # Request results logger.info('Parsing results from nzbs.org') - headers = { 'User-Agent': USER_AGENT } + headers = {'User-Agent': USER_AGENT} params = { "t": "search", - "apikey": headphones.NZBSORG_HASH, + "apikey": headphones.CONFIG.NZBSORG_HASH, "cat": categories, - "maxage": headphones.USENET_RETENTION, + "maxage": headphones.CONFIG.USENET_RETENTION, "q": term } @@ -501,12 +625,12 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None): except Exception as e: logger.exception("Unhandled exception while parsing feed") - if headphones.OMGWTFNZBS: + if headphones.CONFIG.OMGWTFNZBS: provider = "omgwtfnzbs" - if headphones.PREFERRED_QUALITY == 3 or losslessOnly: + if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly: categories = "22" - elif headphones.PREFERRED_QUALITY == 1 or allow_lossless: + elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless: categories = "22,7" else: categories = "7" @@ -518,12 +642,12 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None): # Request results logger.info('Parsing results from omgwtfnzbs') - headers = { 'User-Agent': USER_AGENT } + headers = {'User-Agent': USER_AGENT} params = { - "user": headphones.OMGWTFNZBS_UID, - "api": headphones.OMGWTFNZBS_APIKEY, + "user": headphones.CONFIG.OMGWTFNZBS_UID, + "api": headphones.CONFIG.OMGWTFNZBS_APIKEY, "catid": categories, - "retention": headphones.USENET_RETENTION, + "retention": headphones.CONFIG.USENET_RETENTION, "search": term } @@ -562,6 +686,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None): return results + def send_to_downloader(data, bestqual, album): logger.info(u'Found best result from %s: %s - %s', bestqual[3], bestqual[2], bestqual[0], helpers.bytes_to_mb(bestqual[1])) @@ -573,7 +698,7 @@ def send_to_downloader(data, bestqual, album): if kind == 'nzb': folder_name = helpers.sab_sanitize_foldername(bestqual[0]) - if headphones.NZB_DOWNLOADER == 1: + if headphones.CONFIG.NZB_DOWNLOADER == 1: nzb = classes.NZBDataSearchResult() nzb.extraInfo.append(data) @@ -581,7 +706,7 @@ def send_to_downloader(data, bestqual, album): if not nzbget.sendNZB(nzb): return - elif headphones.NZB_DOWNLOADER == 0: + elif headphones.CONFIG.NZB_DOWNLOADER == 0: nzb = classes.NZBDataSearchResult() nzb.extraInfo.append(data) @@ -599,7 +724,7 @@ def send_to_downloader(data, bestqual, album): else: nzb_name = folder_name + '.nzb' - download_path = os.path.join(headphones.BLACKHOLE_DIR, nzb_name) + download_path = os.path.join(headphones.CONFIG.BLACKHOLE_DIR, nzb_name) try: prev = os.umask(headphones.UMASK) @@ -616,14 +741,14 @@ def send_to_downloader(data, bestqual, album): folder_name = '%s - %s [%s]' % (helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'), helpers.latinToAscii(album['AlbumTitle']).encode('UTF-8').replace('/', '_'), get_year_from_release_date(album['ReleaseDate'])) # Blackhole - if headphones.TORRENT_DOWNLOADER == 0: + if headphones.CONFIG.TORRENT_DOWNLOADER == 0: # Get torrent name from .torrent, this is usually used by the torrent client as the folder name torrent_name = helpers.replace_illegal_chars(folder_name) + '.torrent' - download_path = os.path.join(headphones.TORRENTBLACKHOLE_DIR, torrent_name) + download_path = os.path.join(headphones.CONFIG.TORRENTBLACKHOLE_DIR, torrent_name) - if bestqual[2].startswith("magnet:"): - if headphones.OPEN_MAGNET_LINKS: + if bestqual[2].lower().startswith("magnet:"): + if headphones.CONFIG.MAGNET_LINKS == 1: try: if headphones.SYS_PLATFORM == 'win32': os.startfile(bestqual[2]) @@ -637,39 +762,54 @@ def send_to_downloader(data, bestqual, album): except Exception as e: logger.error("Error opening magnet link: %s" % str(e)) return - else: - logger.error("Cannot save magnet files to blackhole. Please switch your torrent downloader to Transmission or uTorrent or allow Headphones to try to open magnet links") - return + elif headphones.CONFIG.MAGNET_LINKS == 2: + # Procedure adapted from CouchPotato + torrent_hash = calculate_torrent_hash(bestqual[2]) - else: - try: + # Randomize list of services + services = TORRENT_TO_MAGNET_SERVICES[:] + random.shuffle(services) - if bestqual[3] == 'rutracker.org': - download_path = rutracker.get_torrent(bestqual[2], headphones.TORRENTBLACKHOLE_DIR) - if not download_path: - return + for service in services: + data = request.request_content(service % torrent_hash) + + if data and "torcache" in data: + if not torrent_to_file(download_path, data): + return + # Extract folder name from torrent + folder_name = read_torrent_name(download_path, + bestqual[0]) + + # Break for loop + break else: - #Write the torrent file to a path derived from the TORRENTBLACKHOLE_DIR and file name. - with open(download_path, 'wb') as fp: - fp.write(data) + # No service succeeded + logger.warning("Unable to convert magnet with hash " \ + "'%s' into a torrent file.", torrent_hash) + return + else: + logger.error("Cannot save magnet link in blackhole. " \ + "Please switch your torrent downloader to " \ + "Transmission or uTorrent, or allow Headphones " \ + "to open or convert magnet links") + return + else: + if bestqual[3] == "rutracker.org": + download_path, _ = rutracker.get_torrent(bestqual[2], + headphones.CONFIG.TORRENTBLACKHOLE_DIR) - try: - os.chmod(download_path, int(headphones.FILE_PERMISSIONS, 8)) - except: - logger.error("Could not change permissions for file: %s", download_path) + if not download_path: + return + else: + if not torrent_to_file(download_path, data): + return - # Open the fresh torrent file again so we can extract the - # proper torrent name Used later in post-processing. - with open(download_path, 'rb') as fp: - torrent_info = bdecode(fp.read()) - - folder_name = torrent_info['info'].get('name', '') + # Extract folder name from torrent + folder_name = read_torrent_name(download_path, bestqual[0]) + if folder_name: logger.info('Torrent folder name: %s' % folder_name) - except Exception as e: - logger.error('Couldn\'t get name from Torrent file: %s. Defaulting to torrent title' % e) - folder_name = bestqual[0] - elif headphones.TORRENT_DOWNLOADER == 1: + elif headphones.CONFIG.TORRENT_DOWNLOADER == 1: logger.info("Sending torrent to Transmission") # rutracker needs cookies to be set, pass the .torrent file instead of url @@ -699,11 +839,11 @@ def send_to_downloader(data, bestqual, album): logger.exception("Unhandled exception") # Set Seed Ratio - seed_ratio = getSeedRatio(bestqual[3]) - if seed_ratio != None: + seed_ratio = get_seed_ratio(bestqual[3]) + if seed_ratio is not None: transmission.setSeedRatio(torrentid, seed_ratio) - else:# if headphones.TORRENT_DOWNLOADER == 2: + else:# if headphones.CONFIG.TORRENT_DOWNLOADER == 2: logger.info("Sending torrent to uTorrent") # rutracker needs cookies to be set, pass the .torrent file instead of url @@ -714,7 +854,7 @@ def send_to_downloader(data, bestqual, album): utorrent.labelTorrent(torrentid) else: file_or_url = bestqual[2] - torrentid = CalculateTorrentHash(file_or_url, data) + torrentid = calculate_torrent_hash(file_or_url, data) folder_name = utorrent.addTorrent(file_or_url, torrentid) if folder_name: @@ -731,8 +871,8 @@ def send_to_downloader(data, bestqual, album): logger.exception("Unhandled exception") # Set Seed Ratio - seed_ratio = getSeedRatio(bestqual[3]) - if seed_ratio != None: + seed_ratio = get_seed_ratio(bestqual[3]) + if seed_ratio is not None: utorrent.setSeedRatio(torrentid, seed_ratio) myDB = db.DBConnection() @@ -740,7 +880,7 @@ def send_to_downloader(data, bestqual, album): myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?, ?)', [album['AlbumID'], bestqual[0], bestqual[1], bestqual[2], "Snatched", folder_name, kind]) # Store the torrent id so we can check later if it's finished seeding and can be removed - if seed_ratio != None and seed_ratio != 0 and torrentid: + if seed_ratio is not None and seed_ratio != 0 and torrentid: myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?, ?)', [album['AlbumID'], bestqual[0], bestqual[1], bestqual[2], "Seed_Snatched", torrentid, kind]) # notify @@ -753,44 +893,45 @@ def send_to_downloader(data, bestqual, album): provider = provider.split("//")[1] name = folder_name if folder_name else None - if headphones.GROWL_ENABLED and headphones.GROWL_ONSNATCH: + if headphones.CONFIG.GROWL_ENABLED and headphones.CONFIG.GROWL_ONSNATCH: logger.info(u"Sending Growl notification") growl = notifiers.GROWL() - growl.notify(name,"Download started") - if headphones.PROWL_ENABLED and headphones.PROWL_ONSNATCH: + growl.notify(name, "Download started") + if headphones.CONFIG.PROWL_ENABLED and headphones.CONFIG.PROWL_ONSNATCH: logger.info(u"Sending Prowl notification") prowl = notifiers.PROWL() - prowl.notify(name,"Download started") - if headphones.PUSHOVER_ENABLED and headphones.PUSHOVER_ONSNATCH: + prowl.notify(name, "Download started") + if headphones.CONFIG.PUSHOVER_ENABLED and headphones.CONFIG.PUSHOVER_ONSNATCH: logger.info(u"Sending Pushover notification") prowl = notifiers.PUSHOVER() - prowl.notify(name,"Download started") - if headphones.PUSHBULLET_ENABLED and headphones.PUSHBULLET_ONSNATCH: + prowl.notify(name, "Download started") + if headphones.CONFIG.PUSHBULLET_ENABLED and headphones.CONFIG.PUSHBULLET_ONSNATCH: logger.info(u"Sending PushBullet notification") pushbullet = notifiers.PUSHBULLET() pushbullet.notify(name + " has been snatched!", "Download started") - if headphones.TWITTER_ENABLED and headphones.TWITTER_ONSNATCH: + if headphones.CONFIG.TWITTER_ENABLED and headphones.CONFIG.TWITTER_ONSNATCH: logger.info(u"Sending Twitter notification") twitter = notifiers.TwitterNotifier() twitter.notify_snatch(name) - if headphones.NMA_ENABLED and headphones.NMA_ONSNATCH: + if headphones.CONFIG.NMA_ENABLED and headphones.CONFIG.NMA_ONSNATCH: logger.info(u"Sending NMA notification") nma = notifiers.NMA() nma.notify(snatched=name) - if headphones.PUSHALOT_ENABLED and headphones.PUSHALOT_ONSNATCH: + if headphones.CONFIG.PUSHALOT_ENABLED and headphones.CONFIG.PUSHALOT_ONSNATCH: logger.info(u"Sending Pushalot notification") pushalot = notifiers.PUSHALOT() - pushalot.notify(name,"Download started") - if headphones.OSX_NOTIFY_ENABLED and headphones.OSX_NOTIFY_ONSNATCH: + pushalot.notify(name, "Download started") + if headphones.CONFIG.OSX_NOTIFY_ENABLED and headphones.CONFIG.OSX_NOTIFY_ONSNATCH: logger.info(u"Sending OS X notification") osx_notify = notifiers.OSX_NOTIFY() osx_notify.notify(artist, albumname, 'Snatched: ' + provider + '. ' + name) - if headphones.BOXCAR_ENABLED and headphones.BOXCAR_ONSNATCH: + if headphones.CONFIG.BOXCAR_ENABLED and headphones.CONFIG.BOXCAR_ONSNATCH: logger.info(u"Sending Boxcar2 notification") b2msg = 'From ' + provider + '

    ' + name boxcar = notifiers.BOXCAR() boxcar.notify('Headphones snatched: ' + title, b2msg, rgid) + def verifyresult(title, artistterm, term, lossless): title = re.sub('[\.\-\/\_]', ' ', title) @@ -815,18 +956,18 @@ def verifyresult(title, artistterm, term, lossless): return False # Filter out FLAC if we're not specifically looking for it - if headphones.PREFERRED_QUALITY == (0 or '0') and 'flac' in title.lower() and not lossless: + if headphones.CONFIG.PREFERRED_QUALITY == (0 or '0') and 'flac' in title.lower() and not lossless: logger.info("Removed %s from results because it's a lossless album and we're not looking for a lossless album right now.", title) return False - if headphones.IGNORED_WORDS: - for each_word in helpers.split_string(headphones.IGNORED_WORDS): + if headphones.CONFIG.IGNORED_WORDS: + for each_word in helpers.split_string(headphones.CONFIG.IGNORED_WORDS): if each_word.lower() in title.lower(): logger.info("Removed '%s' from results because it contains ignored word: '%s'", title, each_word) return False - if headphones.REQUIRED_WORDS: - for each_word in helpers.split_string(headphones.REQUIRED_WORDS): + if headphones.CONFIG.REQUIRED_WORDS: + for each_word in helpers.split_string(headphones.CONFIG.REQUIRED_WORDS): if ' OR ' in each_word: or_words = helpers.split_string(each_word, 'OR') if any(word.lower() in title.lower() for word in or_words): @@ -848,7 +989,7 @@ def verifyresult(title, artistterm, term, lossless): if not re.search('(?:\W|^)+' + token + '(?:\W|$)+', title, re.IGNORECASE | re.UNICODE): cleantoken = ''.join(c for c in token if c not in string.punctuation) if not not re.search('(?:\W|^)+' + cleantoken + '(?:\W|$)+', title, re.IGNORECASE | re.UNICODE): - dic = {'!':'i', '$':'s'} + dic = {'!': 'i', '$': 's'} dumbtoken = helpers.replace_all(token, dic) if not not re.search('(?:\W|^)+' + dumbtoken + '(?:\W|$)+', title, re.IGNORECASE | re.UNICODE): logger.info("Removed from results: %s (missing tokens: %s and %s)", title, token, cleantoken) @@ -856,13 +997,14 @@ def verifyresult(title, artistterm, term, lossless): return True + def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): global gazelle # persistent what.cd api object to reduce number of login attempts # rutracker login - if headphones.RUTRACKER and album: - rulogin = rutracker.login(headphones.RUTRACKER_USER, headphones.RUTRACKER_PASSWORD) + if headphones.CONFIG.RUTRACKER and album: + rulogin = rutracker.login(headphones.CONFIG.RUTRACKER_USER, headphones.CONFIG.RUTRACKER_PASSWORD) if not rulogin: logger.info(u'Could not login to rutracker, search results will exclude this provider') @@ -872,7 +1014,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): year = get_year_from_release_date(reldate) # MERGE THIS WITH THE TERM CLEANUP FROM searchNZB - dic = {'...':'', ' & ':' ', ' = ': ' ', '?':'', '$':'s', ' + ':' ', '"':'', ',':' ', '*':''} + dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': ' ', '*': ''} semi_cleanalbum = helpers.replace_all(album['AlbumTitle'], dic) cleanalbum = helpers.latinToAscii(semi_cleanalbum) @@ -906,10 +1048,10 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): # Replace bad characters in the term and unicode it term = re.sub('[\.\-\/]', ' ', term).encode('utf-8') artistterm = re.sub('[\.\-\/]', ' ', cleanartist).encode('utf-8', 'replace') - albumterm = re.sub('[\.\-\/]', ' ', cleanalbum).encode('utf-8', 'replace') + albumterm = re.sub('[\.\-\/]', ' ', cleanalbum).encode('utf-8', 'replace') # If Preferred Bitrate and High Limit and Allow Lossless then get both lossy and lossless - if headphones.PREFERRED_QUALITY == 2 and headphones.PREFERRED_BITRATE and headphones.PREFERRED_BITRATE_HIGH_BUFFER and headphones.PREFERRED_BITRATE_ALLOW_LOSSLESS: + if headphones.CONFIG.PREFERRED_QUALITY == 2 and headphones.CONFIG.PREFERRED_BITRATE and headphones.CONFIG.PREFERRED_BITRATE_HIGH_BUFFER and headphones.CONFIG.PREFERRED_BITRATE_ALLOW_LOSSLESS: allow_lossless = True else: allow_lossless = False @@ -917,8 +1059,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): logger.debug("Using search term: %s" % term) resultlist = [] - pre_sorted_results = False - minimumseeders = int(headphones.NUMBEROFSEEDERS) - 1 + minimumseeders = int(headphones.CONFIG.NUMBEROFSEEDERS) - 1 def set_proxy(proxy_url): if not proxy_url.startswith('http'): @@ -928,31 +1069,27 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): return proxy_url - - if headphones.KAT: + if headphones.CONFIG.KAT: provider = "Kick Ass Torrents" ka_term = term.replace("!", "") # Use proxy if specified - if headphones.KAT_PROXY_URL: - providerurl = url_fix(set_proxy(headphones.KAT_PROXY_URL)) + if headphones.CONFIG.KAT_PROXY_URL: + providerurl = fix_url(set_proxy(headphones.CONFIG.KAT_PROXY_URL)) else: - providerurl = url_fix("https://kickass.to") + providerurl = fix_url("https://kickass.to") # Build URL providerurl = providerurl + "/usearch/" + ka_term # Pick category for torrents - if headphones.PREFERRED_QUALITY == 3 or losslessOnly: - categories = "7" # Music + if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly: format = "2" # FLAC maxsize = 10000000000 - elif headphones.PREFERRED_QUALITY == 1 or allow_lossless: - categories = "7" # Music + elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless: format = "10" # MP3 and FLAC maxsize = 10000000000 else: - categories = "7" # Music format = "8" # MP3 only maxsize = 300000000 @@ -985,7 +1122,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): if not torrent or (int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1): rightformat = False - if rightformat == True and size < maxsize and minimumseeders < int(seeders): + if rightformat and size < maxsize and minimumseeders < int(seeders): resultlist.append((title, size, url, provider, 'torrent')) logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) else: @@ -993,16 +1130,16 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): except Exception as e: logger.exception("Unhandled exception in the KAT parser") - if headphones.WAFFLES: + if headphones.CONFIG.WAFFLES: provider = "Waffles.fm" - providerurl = url_fix("https://www.waffles.fm/browse.php") + providerurl = fix_url("https://www.waffles.fm/browse.php") bitrate = None - if headphones.PREFERRED_QUALITY == 3 or losslessOnly: + if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly: format = "FLAC" bitrate = "(Lossless)" maxsize = 10000000000 - elif headphones.PREFERRED_QUALITY == 1 or allow_lossless: + elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless: format = "FLAC OR MP3" maxsize = 10000000000 else: @@ -1027,8 +1164,8 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): logger.info('Parsing results from Waffles') params = { - "uid": headphones.WAFFLES_UID, - "passkey": headphones.WAFFLES_PASSKEY, + "uid": headphones.CONFIG.WAFFLES_UID, + "passkey": headphones.CONFIG.WAFFLES_PASSKEY, "rss": "1", "c0": "1", "s": "seeders", # sort by @@ -1059,7 +1196,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): logger.error(u"An error occurred while trying to parse the response from Waffles.fm: %s", e) # rutracker.org - if headphones.RUTRACKER and rulogin: + if headphones.CONFIG.RUTRACKER and rulogin: provider = "rutracker.org" @@ -1068,10 +1205,10 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): logger.info(u'Release date not specified, ignoring for rutracker.org') else: - if headphones.PREFERRED_QUALITY == 3 or losslessOnly: + if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly: format = 'lossless' maxsize = 10000000000 - elif headphones.PREFERRED_QUALITY == 1 or allow_lossless: + elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless: format = 'lossless+mp3' maxsize = 10000000000 else: @@ -1100,19 +1237,19 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): else: logger.info(u"No valid results found from %s" % (provider)) - if headphones.WHATCD: + if headphones.CONFIG.WHATCD: provider = "What.cd" providerurl = "http://what.cd/" bitrate = None bitrate_string = bitrate - if headphones.PREFERRED_QUALITY == 3 or losslessOnly: # Lossless Only mode + if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly: # Lossless Only mode search_formats = [gazelleformat.FLAC] maxsize = 10000000000 - elif headphones.PREFERRED_QUALITY == 2: # Preferred quality mode + elif headphones.CONFIG.PREFERRED_QUALITY == 2: # Preferred quality mode search_formats = [None] # should return all - bitrate = headphones.PREFERRED_BITRATE + bitrate = headphones.CONFIG.PREFERRED_BITRATE if bitrate: for encoding_string in gazelleencoding.ALL_ENCODINGS: if re.search(bitrate, encoding_string, flags=re.I): @@ -1120,7 +1257,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): if bitrate_string not in gazelleencoding.ALL_ENCODINGS: logger.info(u"Your preferred bitrate is not one of the available What.cd filters, so not using it as a search parameter.") maxsize = 10000000000 - elif headphones.PREFERRED_QUALITY == 1 or allow_lossless: # Highest quality including lossless + elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless: # Highest quality including lossless search_formats = [gazelleformat.FLAC, gazelleformat.MP3] maxsize = 10000000000 else: # Highest quality excluding lossless @@ -1130,7 +1267,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): if not gazelle or not gazelle.logged_in(): try: logger.info(u"Attempting to log in to What.cd...") - gazelle = gazelleapi.GazelleAPI(headphones.WHATCD_USERNAME, headphones.WHATCD_PASSWORD) + gazelle = gazelleapi.GazelleAPI(headphones.CONFIG.WHATCD_USERNAME, headphones.CONFIG.WHATCD_PASSWORD) gazelle._login() except Exception as e: gazelle = None @@ -1146,8 +1283,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): # filter on format, size, and num seeders logger.info(u"Filtering torrents by format, maximum size, and minimum seeders...") - match_torrents = [ torrent for torrent in all_torrents if torrent.size <= maxsize ] - match_torrents = [ torrent for torrent in match_torrents if torrent.seeders >= minimumseeders ] + match_torrents = [t for t in all_torrents if t.size <= maxsize and t.seeders >= minimumseeders] logger.info(u"Remaining torrents: %s" % ", ".join(repr(torrent) for torrent in match_torrents)) @@ -1161,7 +1297,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): match_torrents.sort(key=lambda x: int(x.snatched), reverse=True) if gazelleformat.MP3 in search_formats: # sort by size after rounding to nearest 10MB...hacky, but will favor highest quality - match_torrents.sort(key=lambda x: int(10 * round(x.size/1024./1024./10.)), reverse=True) + match_torrents.sort(key=lambda x: int(10 * round(x.size / 1024. / 1024. / 10.)), reverse=True) if search_formats and None not in search_formats: match_torrents.sort(key=lambda x: int(search_formats.index(x.format))) # prefer lossless # if bitrate: @@ -1169,7 +1305,6 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): # match_torrents.sort(key=lambda x: str(bitrate) in x.getTorrentFolderName(), reverse=True) logger.info(u"New order: %s" % ", ".join(repr(torrent) for torrent in match_torrents)) - pre_sorted_results = True for torrent in match_torrents: if not torrent.file_path: torrent.group.update_group_data() # will load the file_path for the individual torrents @@ -1180,24 +1315,24 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): 'torrent')) # Pirate Bay - if headphones.PIRATEBAY: + if headphones.CONFIG.PIRATEBAY: provider = "The Pirate Bay" tpb_term = term.replace("!", "") # Use proxy if specified - if headphones.PIRATEBAY_PROXY_URL: - providerurl = url_fix(set_proxy(headphones.PIRATEBAY_PROXY_URL)) + if headphones.CONFIG.PIRATEBAY_PROXY_URL: + providerurl = fix_url(set_proxy(headphones.CONFIG.PIRATEBAY_PROXY_URL)) else: - providerurl = url_fix("https://thepiratebay.se") + providerurl = fix_url("https://thepiratebay.se") # Build URL providerurl = providerurl + "/search/" + tpb_term + "/0/7/" # 7 is sort by seeders # Pick category for torrents - if headphones.PREFERRED_QUALITY == 3 or losslessOnly: + if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly: category = '104' # FLAC maxsize = 10000000000 - elif headphones.PREFERRED_QUALITY == 1 or allow_lossless: + elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless: category = '100' # General audio category maxsize = 10000000000 else: @@ -1207,7 +1342,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): # Request content logger.info("Searching The Pirate Bay using term: %s", tpb_term) - headers = {'User-Agent' : 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36'} + headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36'} data = request.request_soup(url=providerurl + category, headers=headers) # Process content @@ -1221,14 +1356,14 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): try: url = None rightformat = True - title = ''.join(item.find("a", {"class" : "detLink"})) - seeds = int(''.join(item.find("td", {"align" : "right"}))) + title = ''.join(item.find("a", {"class": "detLink"})) + seeds = int(''.join(item.find("td", {"align": "right"}))) - if headphones.TORRENT_DOWNLOADER == 0: + if headphones.CONFIG.TORRENT_DOWNLOADER == 0: try: - url = item.find("a", {"title":"Download this torrent"})['href'] + url = item.find("a", {"title": "Download this torrent"})['href'] except TypeError: - if headphones.OPEN_MAGNET_LINKS: + if headphones.CONFIG.MAGNET_LINKS != 0: url = item.findAll("a")[3]['href'] else: logger.info('"%s" only has a magnet link, skipping' % title) @@ -1250,20 +1385,20 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): except Exception as e: logger.error(u"An unknown error occurred in the Pirate Bay parser: %s" % e) - if headphones.MININOVA: + if headphones.CONFIG.MININOVA: provider = "Mininova" - providerurl = url_fix("http://www.mininova.org/rss/" + term + "/5") + providerurl = fix_url("http://www.mininova.org/rss/" + term + "/5") - if headphones.PREFERRED_QUALITY == 3 or losslessOnly: - categories = "7" #music + if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly: + # categories = "7" #music format = "2" #flac maxsize = 10000000000 - elif headphones.PREFERRED_QUALITY == 1 or allow_lossless: - categories = "7" #music + elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless: + # categories = "7" #music format = "10" #mp3+flac maxsize = 10000000000 else: - categories = "7" #music + # categories = "7" #music format = "8" #mp3 maxsize = 300000000 @@ -1293,10 +1428,9 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): size = int(item.links[1]['length']) if format == "2": torrent = request.request_content(url) - if not torrent or (int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1): rightformat = False - if rightformat == True and size < maxsize and minimumseeders < seeds: + if rightformat and size < maxsize and minimumseeders < seeds: resultlist.append((title, size, url, provider, 'torrent')) logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) else: @@ -1317,19 +1451,20 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): return results # THIS IS KIND OF A MESS AND PROBABLY NEEDS TO BE CLEANED UP + + def preprocess(resultlist): for result in resultlist: - if result[4] == 'torrent': #Get out of here if we're using Transmission - if headphones.TORRENT_DOWNLOADER == 1: ## if not a magnet link still need the .torrent to generate hash... uTorrent support labeling + if headphones.CONFIG.TORRENT_DOWNLOADER == 1: ## if not a magnet link still need the .torrent to generate hash... uTorrent support labeling return True, result # get outta here if rutracker if result[3] == 'rutracker.org': return True, result # Get out of here if it's a magnet link - if result[2].startswith("magnet"): + if result[2].lower().startswith("magnet:"): return True, result # Download the torrent file @@ -1344,48 +1479,9 @@ def preprocess(resultlist): return request.request_content(url=result[2], headers=headers), result else: - headers = {'User-Agent': USER_AGENT} if result[3] == 'headphones': - return request.request_content(url=result[2], headers=headers, auth=(headphones.HPUSER, headphones.HPPASS)), result + return request.request_content(url=result[2], headers=headers, auth=(headphones.CONFIG.HPUSER, headphones.CONFIG.HPPASS)), result else: return request.request_content(url=result[2], headers=headers), result - -def CalculateTorrentHash(link, data): - - if link.startswith('magnet'): - tor_hash = re.findall('urn:btih:([\w]{32,40})', link)[0] - if len(tor_hash) == 32: - tor_hash = b16encode(b32decode(tor_hash)).lower() - else: - info = bdecode(data)["info"] - tor_hash = sha1(bencode(info)).hexdigest() - - logger.debug('Torrent Hash: ' + str(tor_hash)) - - return tor_hash - -def getSeedRatio(provider): - seed_ratio = '' - if provider == 'rutracker.org': - seed_ratio = headphones.RUTRACKER_RATIO - elif provider == 'Kick Ass Torrents': - seed_ratio = headphones.KAT_RATIO - elif provider == 'What.cd': - seed_ratio = headphones.WHATCD_RATIO - elif provider == 'The Pirate Bay': - seed_ratio = headphones.PIRATEBAY_RATIO - elif provider == 'Waffles.fm': - seed_ratio = headphones.WAFFLES_RATIO - elif provider == 'Mininova': - seed_ratio = headphones.MININOVA_RATIO - if seed_ratio != '': - try: - seed_ratio_float = float(seed_ratio) - except: - seed_ratio_float = None - logger.warn('Could not get Seed Ratio for %s' % provider) - return seed_ratio_float - else: - return None \ No newline at end of file diff --git a/headphones/searcher_rutracker.py b/headphones/searcher_rutracker.py index 8be71915..0817cc50 100644 --- a/headphones/searcher_rutracker.py +++ b/headphones/searcher_rutracker.py @@ -4,8 +4,6 @@ # Headphones rutracker.org search # Functions called from searcher.py -from headphones import logger, db, utorrent - from bencode import bencode as bencode, bdecode from urlparse import urlparse from bs4 import BeautifulSoup @@ -20,6 +18,9 @@ import urllib import re import os +from headphones import db, logger + + class Rutracker(): logged_in = False @@ -47,9 +48,9 @@ class Rutracker(): #if self.login_counter > 1: # return False - params = urllib.urlencode({"login_username" : login, - "login_password" : password, - "login" : "Вход"}) + params = urllib.urlencode({"login_username": login, + "login_password": password, + "login": "Вход"}) try: self.opener.open("http://login.rutracker.org/forum/login.php", params) @@ -114,26 +115,26 @@ class Rutracker(): #logger.debug (soup.prettify()) # Title - for link in soup.find_all('a', attrs={'class' : 'med tLink hl-tags bold'}): + for link in soup.find_all('a', attrs={'class': 'med tLink hl-tags bold'}): title = link.get_text() titles.append(title) # Download URL - for link in soup.find_all('a', attrs={'class' : 'small tr-dl dl-stub'}): + for link in soup.find_all('a', attrs={'class': 'small tr-dl dl-stub'}): url = link.get('href') urls.append(url) # Seeders - for link in soup.find_all('b', attrs={'class' : 'seedmed'}): + for link in soup.find_all('b', attrs={'class': 'seedmed'}): seeder = link.get_text() seeders.append(seeder) # Size - for link in soup.find_all('td', attrs={'class' : 'row4 small nowrap tor-size'}): + for link in soup.find_all('td', attrs={'class': 'row4 small nowrap tor-size'}): size = link.u.string sizes.append(size) - except : + except: pass # Combine lists @@ -196,8 +197,8 @@ class Rutracker(): if torrent: decoded = bdecode(torrent) metainfo = decoded['info'] - page.close () - except Exception, e: + page.close() + except Exception as e: logger.error('Error getting torrent: %s' % e) return False @@ -215,9 +216,9 @@ class Rutracker(): cuecount += 1 title = returntitle.lower() - logger.debug ('torrent title: %s' % title) - logger.debug ('headphones trackcount: %s' % hptrackcount) - logger.debug ('rutracker trackcount: %s' % trackcount) + logger.debug('torrent title: %s' % title) + logger.debug('headphones trackcount: %s' % hptrackcount) + logger.debug('rutracker trackcount: %s' % trackcount) # If torrent track count less than headphones track count, and there's a cue, then attempt to get track count from log(s) # This is for the case where we have a single .flac/.wav which can be split by cue @@ -245,7 +246,7 @@ class Rutracker(): if totallogcount > 0: trackcount = totallogcount - logger.debug ('rutracker logtrackcount: %s' % totallogcount) + logger.debug('rutracker logtrackcount: %s' % totallogcount) # If torrent track count = hp track count then return torrent, # if greater, check for deluxe/special/foreign editions @@ -294,7 +295,7 @@ class Rutracker(): os.umask(prev) # Add file to utorrent - if headphones.TORRENT_DOWNLOADER == 2: + if headphones.CONFIG.TORRENT_DOWNLOADER == 2: self.utorrent_add_file(download_path) except Exception as e: @@ -306,7 +307,7 @@ class Rutracker(): #TODO get this working in utorrent.py def utorrent_add_file(self, filename): - host = headphones.UTORRENT_HOST + host = headphones.CONFIG.UTORRENT_HOST if not host.startswith('http'): host = 'http://' + host if host.endswith('/'): @@ -315,8 +316,8 @@ class Rutracker(): host = host[:-4] base_url = host - username = headphones.UTORRENT_USERNAME - password = headphones.UTORRENT_PASSWORD + username = headphones.CONFIG.UTORRENT_USERNAME + password = headphones.CONFIG.UTORRENT_PASSWORD session = requests.Session() url = base_url + '/gui/' @@ -346,4 +347,3 @@ class Rutracker(): except Exception: logger.exception('Error adding file to utorrent') return - diff --git a/headphones/torrentfinished.py b/headphones/torrentfinished.py index 2c9c555b..38b5b30b 100644 --- a/headphones/torrentfinished.py +++ b/headphones/torrentfinished.py @@ -20,6 +20,8 @@ from headphones import db, utorrent, transmission, logger postprocessor_lock = threading.Lock() # Remove Torrent + data if Post Processed and finished Seeding + + def checkTorrentFinished(): logger.info("Checking if any torrents have finished seeding and can be removed") @@ -33,7 +35,7 @@ def checkTorrentFinished(): hash = album['FolderName'] albumid = album['AlbumID'] torrent_removed = False - if headphones.TORRENT_DOWNLOADER == 1: + if headphones.CONFIG.TORRENT_DOWNLOADER == 1: torrent_removed = transmission.removeTorrent(hash, True) else: torrent_removed = utorrent.removeTorrent(hash, True) diff --git a/headphones/transmission.py b/headphones/transmission.py index 3a687a73..f6cbf582 100644 --- a/headphones/transmission.py +++ b/headphones/transmission.py @@ -13,9 +13,8 @@ # You should have received a copy of the GNU General Public License # along with Headphones. If not, see . -from headphones import logger, notifiers, request +from headphones import logger, request -import re import time import json import base64 @@ -27,30 +26,28 @@ import headphones # TODO: Store the session id so we don't need to make 2 calls # Store torrent id so we can check up on it + def addTorrent(link): method = 'torrent-add' if link.endswith('.torrent'): with open(link, 'rb') as f: metainfo = str(base64.b64encode(f.read())) - arguments = {'metainfo': metainfo, 'download-dir':headphones.DOWNLOAD_TORRENT_DIR} + arguments = {'metainfo': metainfo, 'download-dir': headphones.CONFIG.DOWNLOAD_TORRENT_DIR} else: - arguments = {'filename': link, 'download-dir': headphones.DOWNLOAD_TORRENT_DIR} + arguments = {'filename': link, 'download-dir': headphones.CONFIG.DOWNLOAD_TORRENT_DIR} - response = torrentAction(method,arguments) + response = torrentAction(method, arguments) if not response: return False if response['result'] == 'success': if 'torrent-added' in response['arguments']: - name = response['arguments']['torrent-added']['name'] retid = response['arguments']['torrent-added']['hashString'] elif 'torrent-duplicate' in response['arguments']: - name = response['arguments']['torrent-duplicate']['name'] retid = response['arguments']['torrent-duplicate']['hashString'] else: - name = link retid = False logger.info(u"Torrent sent to Transmission successfully") @@ -60,9 +57,10 @@ def addTorrent(link): logger.info('Transmission returned status %s' % response['result']) return False + def getTorrentFolder(torrentid): method = 'torrent-get' - arguments = { 'ids': torrentid, 'fields': ['name','percentDone']} + arguments = {'ids': torrentid, 'fields': ['name', 'percentDone']} response = torrentAction(method, arguments) percentdone = response['arguments']['torrents'][0]['percentDone'] @@ -70,8 +68,8 @@ def getTorrentFolder(torrentid): tries = 1 - while percentdone == 0 and tries <10: - tries+=1 + while percentdone == 0 and tries < 10: + tries += 1 time.sleep(5) response = torrentAction(method, arguments) percentdone = response['arguments']['torrents'][0]['percentDone'] @@ -80,6 +78,7 @@ def getTorrentFolder(torrentid): return torrent_folder_name + def setSeedRatio(torrentid, ratio): method = 'torrent-set' if ratio != 0: @@ -91,10 +90,11 @@ def setSeedRatio(torrentid, ratio): if not response: return False -def removeTorrent(torrentid, remove_data = False): + +def removeTorrent(torrentid, remove_data=False): method = 'torrent-get' - arguments = { 'ids': torrentid, 'fields': ['isFinished', 'name']} + arguments = {'ids': torrentid, 'fields': ['isFinished', 'name']} response = torrentAction(method, arguments) if not response: @@ -120,11 +120,12 @@ def removeTorrent(torrentid, remove_data = False): return False + def torrentAction(method, arguments): - host = headphones.TRANSMISSION_HOST - username = headphones.TRANSMISSION_USERNAME - password = headphones.TRANSMISSION_PASSWORD + host = headphones.CONFIG.TRANSMISSION_HOST + username = headphones.CONFIG.TRANSMISSION_USERNAME + password = headphones.CONFIG.TRANSMISSION_PASSWORD sessionid = None if not host.startswith('http'): @@ -141,12 +142,14 @@ def torrentAction(method, arguments): # Check if it ends in a port number i = host.rfind(':') if i >= 0: - possible_port = host[i+1:] + possible_port = host[i + 1:] + host = host + "/rpc" try: port = int(possible_port) - host = host + "/transmission/rpc" + if port: + host = host + "/transmission/rpc" except ValueError: - host = host + "/rpc" + logger.debug('No port, assuming not transmission') else: logger.error('Transmission port missing') return @@ -176,8 +179,8 @@ def torrentAction(method, arguments): return # Prepare next request - headers = { 'x-transmission-session-id': sessionid } - data = { 'method': method, 'arguments': arguments } + headers = {'x-transmission-session-id': sessionid} + data = {'method': method, 'arguments': arguments} response = request.request_json(host, method="post", data=json.dumps(data), headers=headers, auth=auth) diff --git a/headphones/updater.py b/headphones/updater.py index 30b9f057..b18db5bc 100644 --- a/headphones/updater.py +++ b/headphones/updater.py @@ -13,10 +13,9 @@ # You should have received a copy of the GNU General Public License # along with Headphones. If not, see . -import headphones - from headphones import logger, db, importer + def dbUpdate(forcefull=False): myDB = db.DBConnection() diff --git a/headphones/utorrent.py b/headphones/utorrent.py index 1a71e552..67d9bad7 100644 --- a/headphones/utorrent.py +++ b/headphones/utorrent.py @@ -13,22 +13,29 @@ # You should have received a copy of the GNU General Public License # along with Headphones. If not, see . -import urllib, urllib2, urlparse, cookielib -import json, re, os, time +import urllib +import urllib2 +import urlparse +import cookielib +import json +import re +import os +import time import headphones from headphones import logger from collections import namedtuple + class utorrentclient(object): TOKEN_REGEX = "" UTSetting = namedtuple("UTSetting", ["name", "int", "str", "access"]) - def __init__(self, base_url = None, username = None, password = None,): + def __init__(self, base_url=None, username=None, password=None,): - host = headphones.UTORRENT_HOST + host = headphones.CONFIG.UTORRENT_HOST if not host.startswith('http'): host = 'http://' + host @@ -39,8 +46,8 @@ class utorrentclient(object): host = host[:-4] self.base_url = host - self.username = headphones.UTORRENT_USERNAME - self.password = headphones.UTORRENT_PASSWORD + self.username = headphones.CONFIG.UTORRENT_USERNAME + self.password = headphones.CONFIG.UTORRENT_PASSWORD self.opener = self._make_opener('uTorrent', self.base_url, self.username, self.password) self.token = self._get_token() #TODO refresh token, when necessary @@ -48,7 +55,7 @@ class utorrentclient(object): def _make_opener(self, realm, base_url, username, password): """uTorrent API need HTTP Basic Auth and cookie support for token verify.""" auth = urllib2.HTTPBasicAuthHandler() - auth.add_password(realm=realm,uri=base_url,user=username,passwd=password) + auth.add_password(realm=realm, uri=base_url, user=username, passwd=password) opener = urllib2.build_opener(auth) urllib2.install_opener(opener) @@ -132,7 +139,7 @@ class utorrentclient(object): return settings[key] return settings - def remove(self, hash, remove_data = False): + def remove(self, hash, remove_data=False): if remove_data: params = [('action', 'removedata'), ('hash', hash)] else: @@ -156,13 +163,15 @@ class utorrentclient(object): logger.debug('URL: ' + str(url)) logger.debug('uTorrent webUI raised the following error: ' + str(err)) + def labelTorrent(hash): - label = headphones.UTORRENT_LABEL + label = headphones.CONFIG.UTORRENT_LABEL uTorrentClient = utorrentclient() if label: - uTorrentClient.setprops(hash,'label',label) + uTorrentClient.setprops(hash, 'label', label) -def removeTorrent(hash, remove_data = False): + +def removeTorrent(hash, remove_data=False): uTorrentClient = utorrentclient() status, torrentList = uTorrentClient.list() torrents = torrentList['torrents'] @@ -177,14 +186,16 @@ def removeTorrent(hash, remove_data = False): return False return False + def setSeedRatio(hash, ratio): uTorrentClient = utorrentclient() uTorrentClient.setprops(hash, 'seed_override', '1') if ratio != 0: - uTorrentClient.setprops(hash,'seed_ratio', ratio * 10) + uTorrentClient.setprops(hash, 'seed_ratio', ratio * 10) else: # TODO passing -1 should be unlimited - uTorrentClient.setprops(hash,'seed_ratio', -10) + uTorrentClient.setprops(hash, 'seed_ratio', -10) + def dirTorrent(hash, cacheid=None, return_name=None): @@ -212,6 +223,7 @@ def dirTorrent(hash, cacheid=None, return_name=None): return None, None + def addTorrent(link, hash): uTorrentClient = utorrentclient() @@ -230,7 +242,7 @@ def addTorrent(link, hash): # If there's no folder yet then it's probably a magnet, try until folder is populated if torrent_folder == active_dir or not torrent_folder: tries = 1 - while (torrent_folder == active_dir or torrent_folder == None) and tries <= 10: + while (torrent_folder == active_dir or torrent_folder is None) and tries <= 10: tries += 1 time.sleep(6) torrent_folder, cacheid = dirTorrent(hash, cacheid) @@ -241,8 +253,11 @@ def addTorrent(link, hash): return torrent_folder else: labelTorrent(hash) + if headphones.SYS_PLATFORM != "win32": + torrent_folder = torrent_folder.replace('\\', '/') return os.path.basename(os.path.normpath(torrent_folder)) + def getSettingsDirectories(): uTorrentClient = utorrentclient() settings = uTorrentClient.get_settings() @@ -253,4 +268,3 @@ def getSettingsDirectories(): if 'dir_completed_download' in settings: completed = settings['dir_completed_download'][2] return active, completed - diff --git a/headphones/version.py b/headphones/version.py index 0dda129e..1c69552f 100644 --- a/headphones/version.py +++ b/headphones/version.py @@ -1 +1 @@ -HEADPHONES_VERSION = "master" \ No newline at end of file +HEADPHONES_VERSION = "master" diff --git a/headphones/versioncheck.py b/headphones/versioncheck.py index 236c590d..ea2a8e3f 100644 --- a/headphones/versioncheck.py +++ b/headphones/versioncheck.py @@ -22,10 +22,11 @@ import subprocess from headphones import logger, version, request + def runGit(args): - if headphones.GIT_PATH: - git_locations = ['"'+headphones.GIT_PATH+'"'] + if headphones.CONFIG.GIT_PATH: + git_locations = ['"' + headphones.CONFIG.GIT_PATH + '"'] else: git_locations = ['git'] @@ -35,7 +36,7 @@ def runGit(args): output = err = None for cur_git in git_locations: - cmd = cur_git+' '+args + cmd = cur_git + ' ' + args try: logger.debug('Trying to execute: "' + cmd + '" with shell in ' + headphones.PROG_DIR) @@ -59,6 +60,7 @@ def runGit(args): return (output, err) + def getVersion(): if version.HEADPHONES_VERSION.startswith('win32build'): @@ -82,16 +84,16 @@ def getVersion(): logger.error('Output doesn\'t look like a hash, not using it') cur_commit_hash = None - if headphones.DO_NOT_OVERRIDE_GIT_BRANCH and headphones.GIT_BRANCH: - branch_name = headphones.GIT_BRANCH + if headphones.CONFIG.DO_NOT_OVERRIDE_GIT_BRANCH and headphones.CONFIG.GIT_BRANCH: + branch_name = headphones.CONFIG.GIT_BRANCH else: branch_name, err = runGit('rev-parse --abbrev-ref HEAD') branch_name = branch_name - if not branch_name and headphones.GIT_BRANCH: - logger.error('Could not retrieve branch name from git. Falling back to %s' % headphones.GIT_BRANCH) - branch_name = headphones.GIT_BRANCH + if not branch_name and headphones.CONFIG.GIT_BRANCH: + logger.error('Could not retrieve branch name from git. Falling back to %s' % headphones.CONFIG.GIT_BRANCH) + branch_name = headphones.CONFIG.GIT_BRANCH if not branch_name: logger.error('Could not retrieve branch name from git. Defaulting to master') branch_name = 'master' @@ -111,16 +113,17 @@ def getVersion(): current_version = f.read().strip(' \n\r') if current_version: - return current_version, headphones.GIT_BRANCH + return current_version, headphones.CONFIG.GIT_BRANCH else: return None, 'master' + def checkGithub(): headphones.COMMITS_BEHIND = 0 # Get the latest version available from github logger.info('Retrieving latest version information from GitHub') - url = 'https://api.github.com/repos/%s/headphones/commits/%s' % (headphones.GIT_USER, headphones.GIT_BRANCH) + url = 'https://api.github.com/repos/%s/headphones/commits/%s' % (headphones.CONFIG.GIT_USER, headphones.CONFIG.GIT_BRANCH) version = request.request_json(url, timeout=20, validator=lambda x: type(x) == dict) if version is None: @@ -140,7 +143,7 @@ def checkGithub(): return headphones.LATEST_VERSION logger.info('Comparing currently installed version with latest GitHub version') - url = 'https://api.github.com/repos/%s/headphones/compare/%s...%s' % (headphones.GIT_USER, headphones.LATEST_VERSION, headphones.CURRENT_VERSION) + url = 'https://api.github.com/repos/%s/headphones/compare/%s...%s' % (headphones.CONFIG.GIT_USER, headphones.LATEST_VERSION, headphones.CURRENT_VERSION) commits = request.request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) if commits is None: @@ -161,12 +164,13 @@ def checkGithub(): return headphones.LATEST_VERSION + def update(): if headphones.INSTALL_TYPE == 'win': logger.info('Windows .exe updating not supported yet.') elif headphones.INSTALL_TYPE == 'git': - output, err = runGit('pull origin ' + headphones.GIT_BRANCH) + output, err = runGit('pull origin ' + headphones.CONFIG.GIT_BRANCH) if not output: logger.error('Couldn\'t download latest version') @@ -177,22 +181,22 @@ def update(): logger.info('No update available, not updating') logger.info('Output: ' + str(output)) elif line.endswith('Aborting.'): - logger.error('Unable to update from git: '+line) + logger.error('Unable to update from git: ' + line) logger.info('Output: ' + str(output)) else: - tar_download_url = 'https://github.com/%s/headphones/tarball/%s' % (headphones.GIT_USER, headphones.GIT_BRANCH) + tar_download_url = 'https://github.com/%s/headphones/tarball/%s' % (headphones.CONFIG.GIT_USER, headphones.CONFIG.GIT_BRANCH) update_dir = os.path.join(headphones.PROG_DIR, 'update') version_path = os.path.join(headphones.PROG_DIR, 'version.txt') - logger.info('Downloading update from: '+ tar_download_url) + logger.info('Downloading update from: ' + tar_download_url) data = request.request_content(tar_download_url) if not data: logger.error("Unable to retrieve new version from '%s', can't update", tar_download_url) return - download_name = headphones.GIT_BRANCH + '-github' + download_name = headphones.CONFIG.GIT_BRANCH + '-github' tar_download_path = os.path.join(headphones.PROG_DIR, download_name) # Save tar to disk @@ -212,13 +216,13 @@ def update(): # Find update dir name update_dir_contents = [x for x in os.listdir(update_dir) if os.path.isdir(os.path.join(update_dir, x))] if len(update_dir_contents) != 1: - logger.error("Invalid update data, update failed: "+str(update_dir_contents)) + logger.error("Invalid update data, update failed: " + str(update_dir_contents)) return content_dir = os.path.join(update_dir, update_dir_contents[0]) # walk temp folder and move files to main folder for dirname, dirnames, filenames in os.walk(content_dir): - dirname = dirname[len(content_dir)+1:] + dirname = dirname[len(content_dir) + 1:] for curfile in filenames: old_path = os.path.join(content_dir, dirname, curfile) new_path = os.path.join(headphones.PROG_DIR, dirname, curfile) @@ -232,6 +236,8 @@ def update(): with open(version_path, 'w') as f: f.write(str(headphones.LATEST_VERSION)) except IOError as e: - logger.error("Unable to write current version to version.txt, " \ - "update not complete: ", e) + logger.error( + "Unable to write current version to version.txt, update not complete: %s", + e + ) return diff --git a/headphones/webserve.py b/headphones/webserve.py index a91305c3..fa5c0c56 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -16,9 +16,8 @@ # NZBGet support added by CurlyMo as a part of XBian - XBMC on the Raspberry Pi from headphones import logger, searcher, db, importer, mb, lastfm, librarysync, helpers, notifiers -from headphones.helpers import checked, radio,today, cleanName +from headphones.helpers import checked, radio, today, cleanName -from mako.template import Template from mako.lookup import TemplateLookup from mako import exceptions @@ -28,21 +27,24 @@ import os import sys import json import time -import string import cherrypy import threading import headphones try: + # pylint:disable=E0611 + # ignore this error because we are catching the ImportError from collections import OrderedDict + # pylint:enable=E0611 except ImportError: # Python 2.6.x fallback, from libs from ordereddict import OrderedDict + def serve_template(templatename, **kwargs): interface_dir = os.path.join(str(headphones.PROG_DIR), 'data/interfaces/') - template_dir = os.path.join(str(interface_dir), headphones.INTERFACE) + template_dir = os.path.join(str(interface_dir), headphones.CONFIG.INTERFACE) _hplookup = TemplateLookup(directories=[template_dir]) @@ -52,11 +54,12 @@ def serve_template(templatename, **kwargs): except: return exceptions.html_error_template().render() + class WebInterface(object): def index(self): raise cherrypy.HTTPRedirect("home") - index.exposed=True + index.exposed = True def home(self): myDB = db.DBConnection() @@ -85,7 +88,7 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("home") # Serve the extras up as a dict to make things easier for new templates (append new extras to the end) - extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "spokenword", "audiobook", "other", "djmix", "mixtape_street", "broadcast", "interview", "demo"] + extras_list = headphones.POSSIBLE_EXTRAS if artist['Extras']: artist_extras = map(int, artist['Extras'].split(',')) else: @@ -99,12 +102,11 @@ class WebInterface(object): extras_dict[extra] = "checked" else: extras_dict[extra] = "" - i+=1 + i += 1 return serve_template(templatename="artist.html", title=artist['ArtistName'], artist=artist, albums=albums, extras=extras_dict) artistPage.exposed = True - def albumPage(self, AlbumID): myDB = db.DBConnection() album = myDB.action('SELECT * from albums WHERE AlbumID=?', [AlbumID]).fetchone() @@ -124,7 +126,7 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("home") if not album['ArtistName']: - title = ' - ' + title = ' - ' else: title = album['ArtistName'] + ' - ' if not album['AlbumTitle']: @@ -134,7 +136,6 @@ class WebInterface(object): return serve_template(templatename="album.html", title=title, album=album, tracks=tracks, description=description) albumPage.exposed = True - def search(self, name, type): if len(name) == 0: raise cherrypy.HTTPRedirect("home") @@ -142,7 +143,7 @@ class WebInterface(object): searchresults = mb.findArtist(name, limit=100) else: searchresults = mb.findRelease(name, limit=100) - return serve_template(templatename="searchresults.html", title='Search Results for: "' + name + '"', searchresults=searchresults, type=type) + return serve_template(templatename="searchresults.html", title='Search Results for: "' + name + '"', searchresults=searchresults, name=name, type=type) search.exposed = True def addArtist(self, artistid): @@ -159,9 +160,8 @@ class WebInterface(object): extras = "1,2,3,4,5,6,7,8,9,10,11,12,13,14" else: temp_extras_list = [] - # TODO: Put these extras as a global variable i = 1 - for extra in ["single", "ep", "compilation", "soundtrack", "live", "remix", "spokenword", "audiobook", "other", "djmix", "mixtape_street", "broadcast", "interview", "demo"]: + for extra in headphones.POSSIBLE_EXTRAS: if extra in kwargs: temp_extras_list.append(i) i += 1 @@ -170,7 +170,7 @@ class WebInterface(object): myDB = db.DBConnection() controlValueDict = {'ArtistID': ArtistID} newValueDict = {'IncludeExtras': 1, - 'Extras': extras} + 'Extras': extras} myDB.upsert("artists", newValueDict, controlValueDict) threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, True, False]).start() raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) @@ -188,6 +188,9 @@ class WebInterface(object): myDB.action('DELETE from allalbums WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']]) myDB.action('DELETE from alltracks WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']]) myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [album['AlbumID']]) + from headphones import cache + c = cache.Cache() + c.remove_from_cache(AlbumID=album['AlbumID']) importer.finalize_update(ArtistID, ArtistName) raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) removeExtras.exposed = True @@ -210,31 +213,37 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) resumeArtist.exposed = True - def deleteArtist(self, ArtistID): + def removeArtist(self, ArtistID): logger.info(u"Deleting all traces of artist: " + ArtistID) myDB = db.DBConnection() namecheck = myDB.select('SELECT ArtistName from artists where ArtistID=?', [ArtistID]) for name in namecheck: - artistname=name['ArtistName'] + artistname = name['ArtistName'] myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID]) - rgids = myDB.select('SELECT DISTINCT ReleaseGroupID FROM albums JOIN releases ON AlbumID = ReleaseGroupID WHERE ArtistID=?', [ArtistID]) + from headphones import cache + c = cache.Cache() + + rgids = myDB.select('SELECT AlbumID FROM albums WHERE ArtistID=? UNION SELECT AlbumID FROM allalbums WHERE ArtistID=?', [ArtistID, ArtistID]) for rgid in rgids: - myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [rgid['ReleaseGroupID']]) - myDB.action('DELETE from have WHERE Matched=?', [rgid['ReleaseGroupID']]) + albumid = rgid['AlbumID'] + myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [albumid]) + myDB.action('DELETE from have WHERE Matched=?', [albumid]) + c.remove_from_cache(AlbumID=albumid) + myDB.action('DELETE from descriptions WHERE ReleaseGroupID=?', [albumid]) myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID]) - rgids = myDB.select('SELECT DISTINCT ReleaseGroupID FROM allalbums JOIN releases ON AlbumID = ReleaseGroupID WHERE ArtistID=?', [ArtistID]) - for rgid in rgids: - myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [rgid['ReleaseGroupID']]) - myDB.action('DELETE from have WHERE Matched=?', [rgid['ReleaseGroupID']]) - myDB.action('DELETE from allalbums WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from alltracks WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from have WHERE ArtistName=?', [artistname]) + c.remove_from_cache(ArtistID=ArtistID) + myDB.action('DELETE from descriptions WHERE ArtistID=?', [ArtistID]) myDB.action('INSERT OR REPLACE into blacklist VALUES (?)', [ArtistID]) + + def deleteArtist(self, ArtistID): + self.removeArtist(ArtistID) raise cherrypy.HTTPRedirect("home") deleteArtist.exposed = True @@ -243,29 +252,13 @@ class WebInterface(object): myDB = db.DBConnection() emptyArtistIDs = [row['ArtistID'] for row in myDB.select("SELECT ArtistID FROM artists WHERE LatestAlbum IS NULL")] for ArtistID in emptyArtistIDs: - logger.info(u"Deleting all traces of artist: " + ArtistID) - myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID]) - - rgids = myDB.select('SELECT DISTINCT ReleaseGroupID FROM albums JOIN releases ON AlbumID = ReleaseGroupID WHERE ArtistID=?', [ArtistID]) - for rgid in rgids: - myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [rgid['ReleaseGroupID']]) - - myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID]) - myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID]) - - rgids = myDB.select('SELECT DISTINCT ReleaseGroupID FROM allalbums JOIN releases ON AlbumID = ReleaseGroupID WHERE ArtistID=?', [ArtistID]) - for rgid in rgids: - myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [rgid['ReleaseGroupID']]) - - myDB.action('DELETE from allalbums WHERE ArtistID=?', [ArtistID]) - myDB.action('DELETE from alltracks WHERE ArtistID=?', [ArtistID]) - myDB.action('INSERT OR REPLACE into blacklist VALUES (?)', [ArtistID]) + self.removeArtist(ArtistID) deleteEmptyArtists.exposed = True def refreshArtist(self, ArtistID): threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, False, True]).start() raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) - refreshArtist.exposed=True + refreshArtist.exposed = True def markAlbums(self, ArtistID=None, action=None, **args): myDB = db.DBConnection() @@ -332,11 +325,11 @@ class WebInterface(object): for result in results: result_dict = { - 'title':result[0], - 'size':result[1], - 'url':result[2], - 'provider':result[3], - 'kind':result[4] + 'title': result[0], + 'size': result[1], + 'url': result[2], + 'provider': result[3], + 'kind': result[4] } results_as_dicts.append(result_dict) @@ -350,13 +343,14 @@ class WebInterface(object): # Handle situations where the torrent url contains arguments that are parsed if kwargs: - import urllib, urllib2 + import urllib + import urllib2 url = urllib2.quote(url, safe=":?/=&") + '&' + urllib.urlencode(kwargs) try: - result = [(title,int(size),url,provider,kind)] + result = [(title, int(size), url, provider, kind)] except ValueError: - result = [(title,float(size),url,provider,kind)] + result = [(title, float(size), url, provider, kind)] logger.info(u"Making sure we can download the chosen result") (data, bestqual) = searcher.preprocess(result) @@ -391,6 +385,12 @@ class WebInterface(object): myDB.action('DELETE from allalbums WHERE AlbumID=?', [AlbumID]) myDB.action('DELETE from alltracks WHERE AlbumID=?', [AlbumID]) myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [AlbumID]) + myDB.action('DELETE from descriptions WHERE ReleaseGroupID=?', [AlbumID]) + + from headphones import cache + c = cache.Cache() + c.remove_from_cache(AlbumID=AlbumID) + if ArtistID: raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) else: @@ -455,28 +455,26 @@ class WebInterface(object): myDB = db.DBConnection() have_album_dictionary = [] headphones_album_dictionary = [] - unmatched_albums = [] have_albums = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName from have WHERE Matched = "Failed" GROUP BY AlbumTitle ORDER BY ArtistName') for albums in have_albums: #Have to skip over manually matched tracks if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']: - original_clean = helpers.cleanName(albums['ArtistName']+" "+albums['AlbumTitle']+" "+albums['TrackTitle']) + original_clean = helpers.cleanName(albums['ArtistName'] + " " + albums['AlbumTitle'] + " " + albums['TrackTitle']) # else: # original_clean = None if original_clean == albums['CleanName']: - have_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] } + have_dict = {'ArtistName': albums['ArtistName'], 'AlbumTitle': albums['AlbumTitle']} have_album_dictionary.append(have_dict) headphones_albums = myDB.select('SELECT ArtistName, AlbumTitle from albums ORDER BY ArtistName') for albums in headphones_albums: if albums['ArtistName'] and albums['AlbumTitle']: - headphones_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] } + headphones_dict = {'ArtistName': albums['ArtistName'], 'AlbumTitle': albums['AlbumTitle']} headphones_album_dictionary.append(headphones_dict) #unmatchedalbums = [f for f in have_album_dictionary if f not in [x for x in headphones_album_dictionary]] check = set([(cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) for d in headphones_album_dictionary]) unmatchedalbums = [d for d in have_album_dictionary if (cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) not in check] - return serve_template(templatename="manageunmatched.html", title="Manage Unmatched Items", unmatchedalbums=unmatchedalbums) manageUnmatched.exposed = True @@ -504,9 +502,9 @@ class WebInterface(object): new_clean_filename = old_clean_filename.replace(existing_artist_clean, new_artist_clean, 1) myDB.action('UPDATE have SET CleanName=? WHERE ArtistName=? AND CleanName=?', [new_clean_filename, existing_artist, old_clean_filename]) controlValueDict = {"CleanName": new_clean_filename} - newValueDict = {"Location" : entry['Location'], - "BitRate" : entry['BitRate'], - "Format" : entry['Format'] + newValueDict = {"Location": entry['Location'], + "BitRate": entry['BitRate'], + "Format": entry['Format'] } #Attempt to match tracks with new CleanName match_alltracks = myDB.action('SELECT CleanName from alltracks WHERE CleanName=?', [new_clean_filename]).fetchone() @@ -516,7 +514,7 @@ class WebInterface(object): if match_tracks: myDB.upsert("tracks", newValueDict, controlValueDict) myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', [new_clean_filename]) - update_count+=1 + update_count += 1 #This was throwing errors and I don't know why, but it seems to be working fine. #else: #logger.info("There was an error modifying Artist %s. This should not have happened" % existing_artist) @@ -531,8 +529,8 @@ class WebInterface(object): new_artist_clean = helpers.cleanName(new_artist).lower() existing_album_clean = helpers.cleanName(existing_album).lower() new_album_clean = helpers.cleanName(new_album).lower() - existing_clean_string = existing_artist_clean+" "+existing_album_clean - new_clean_string = new_artist_clean+" "+new_album_clean + existing_clean_string = existing_artist_clean + " " + existing_album_clean + new_clean_string = new_artist_clean + " " + new_album_clean if existing_clean_string != new_clean_string: have_tracks = myDB.action('SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=? AND AlbumTitle=?', (existing_artist, existing_album)) update_count = 0 @@ -542,9 +540,9 @@ class WebInterface(object): new_clean_filename = old_clean_filename.replace(existing_clean_string, new_clean_string, 1) myDB.action('UPDATE have SET CleanName=? WHERE ArtistName=? AND AlbumTitle=? AND CleanName=?', [new_clean_filename, existing_artist, existing_album, old_clean_filename]) controlValueDict = {"CleanName": new_clean_filename} - newValueDict = {"Location" : entry['Location'], - "BitRate" : entry['BitRate'], - "Format" : entry['Format'] + newValueDict = {"Location": entry['Location'], + "BitRate": entry['BitRate'], + "Format": entry['Format'] } #Attempt to match tracks with new CleanName match_alltracks = myDB.action('SELECT CleanName from alltracks WHERE CleanName=?', [new_clean_filename]).fetchone() @@ -555,7 +553,7 @@ class WebInterface(object): myDB.upsert("tracks", newValueDict, controlValueDict) myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', [new_clean_filename]) album_id = match_tracks['AlbumID'] - update_count+=1 + update_count += 1 #This was throwing errors and I don't know why, but it seems to be working fine. #else: #logger.info("There was an error modifying Artist %s / Album %s with clean name %s" % (existing_artist, existing_album, existing_clean_string)) @@ -573,13 +571,13 @@ class WebInterface(object): manualalbums = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have') for albums in manualalbums: if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']: - original_clean = helpers.cleanName(albums['ArtistName']+" "+albums['AlbumTitle']+" "+albums['TrackTitle']) + original_clean = helpers.cleanName(albums['ArtistName'] + " " + albums['AlbumTitle'] + " " + albums['TrackTitle']) if albums['Matched'] == "Ignored" or albums['Matched'] == "Manual" or albums['CleanName'] != original_clean: if albums['Matched'] == "Ignored": album_status = "Ignored" elif albums['Matched'] == "Manual" or albums['CleanName'] != original_clean: album_status = "Matched" - manual_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'], 'AlbumStatus' : album_status } + manual_dict = {'ArtistName': albums['ArtistName'], 'AlbumTitle': albums['AlbumTitle'], 'AlbumStatus': album_status} if manual_dict not in manual_albums: manual_albums.append(manual_dict) manual_albums_sorted = sorted(manual_albums, key=itemgetter('ArtistName', 'AlbumTitle')) @@ -605,14 +603,14 @@ class WebInterface(object): update_clean = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have WHERE ArtistName=?', [artist]) update_count = 0 for tracks in update_clean: - original_clean = helpers.cleanName(tracks['ArtistName']+" "+tracks['AlbumTitle']+" "+tracks['TrackTitle']).lower() + original_clean = helpers.cleanName(tracks['ArtistName'] + " " + tracks['AlbumTitle'] + " " + tracks['TrackTitle']).lower() album = tracks['AlbumTitle'] track_title = tracks['TrackTitle'] if tracks['CleanName'] != original_clean: myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']]) myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']]) myDB.action('UPDATE have SET CleanName=?, Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?', (original_clean, artist, album, track_title)) - update_count+=1 + update_count += 1 if update_count > 0: librarysync.update_album_status() logger.info("Artist: %s successfully restored to unmatched list" % artist) @@ -623,7 +621,7 @@ class WebInterface(object): update_clean = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have WHERE ArtistName=? AND AlbumTitle=?', (artist, album)) update_count = 0 for tracks in update_clean: - original_clean = helpers.cleanName(tracks['ArtistName']+" "+tracks['AlbumTitle']+" "+tracks['TrackTitle']).lower() + original_clean = helpers.cleanName(tracks['ArtistName'] + " " + tracks['AlbumTitle'] + " " + tracks['TrackTitle']).lower() track_title = tracks['TrackTitle'] if tracks['CleanName'] != original_clean: album_id_check = myDB.action('SELECT AlbumID from tracks WHERE CleanName=?', [tracks['CleanName']]).fetchone() @@ -632,7 +630,7 @@ class WebInterface(object): myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']]) myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']]) myDB.action('UPDATE have SET CleanName=?, Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?', (original_clean, artist, album, track_title)) - update_count+=1 + update_count += 1 if update_count > 0: librarysync.update_album_status(album_id) logger.info("Album: %s successfully restored to unmatched list" % album) @@ -644,22 +642,7 @@ class WebInterface(object): artistsToAdd = [] for ArtistID in args: if action == 'delete': - myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID]) - - rgids = myDB.select('SELECT DISTINCT ReleaseGroupID FROM albums JOIN releases ON AlbumID = ReleaseGroupID WHERE ArtistID=?', [ArtistID]) - for rgid in rgids: - myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [rgid['ReleaseGroupID']]) - - myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID]) - myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID]) - - rgids = myDB.select('SELECT DISTINCT ReleaseGroupID FROM allalbums JOIN releases ON AlbumID = ReleaseGroupID WHERE ArtistID=?', [ArtistID]) - for rgid in rgids: - myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [rgid['ReleaseGroupID']]) - - myDB.action('DELETE from allalbums WHERE ArtistID=?', [ArtistID]) - myDB.action('DELETE from alltracks WHERE ArtistID=?', [ArtistID]) - myDB.action('INSERT OR REPLACE into blacklist VALUES (?)', [ArtistID]) + self.removeArtist(ArtistID) elif action == 'pause': controlValueDict = {'ArtistID': ArtistID} newValueDict = {'Status': 'Paused'} @@ -677,8 +660,8 @@ class WebInterface(object): markArtists.exposed = True def importLastFM(self, username): - headphones.LASTFM_USERNAME = username - headphones.config_write() + headphones.CONFIG.LASTFM_USERNAME = username + headphones.CONFIG.write() threading.Thread(target=lastfm.getArtists).start() raise cherrypy.HTTPRedirect("home") importLastFM.exposed = True @@ -689,22 +672,22 @@ class WebInterface(object): importLastFMTag.exposed = True def importItunes(self, path): - headphones.PATH_TO_XML = path - headphones.config_write() + headphones.CONFIG.PATH_TO_XML = path + headphones.CONFIG.write() threading.Thread(target=importer.itunesImport, args=[path]).start() time.sleep(10) raise cherrypy.HTTPRedirect("home") importItunes.exposed = True def musicScan(self, path, scan=0, redirect=None, autoadd=0, libraryscan=0): - headphones.LIBRARYSCAN = libraryscan - headphones.ADD_ARTISTS = autoadd - headphones.MUSIC_DIR = path - headphones.config_write() + headphones.CONFIG.LIBRARYSCAN = libraryscan + headphones.CONFIG.AUTO_ADD_ARTISTS = autoadd + headphones.CONFIG.MUSIC_DIR = path + headphones.CONFIG.write() if scan: try: threading.Thread(target=librarysync.libraryScan).start() - except Exception, e: + except Exception as e: logger.error('Unable to complete the scan: %s' % e) if redirect: raise cherrypy.HTTPRedirect(redirect) @@ -732,7 +715,7 @@ class WebInterface(object): def forcePostProcess(self, dir=None, album_dir=None): from headphones import postprocessor - threading.Thread(target=postprocessor.forcePostProcess, kwargs={'dir':dir,'album_dir':album_dir}).start() + threading.Thread(target=postprocessor.forcePostProcess, kwargs={'dir': dir, 'album_dir': album_dir}).start() raise cherrypy.HTTPRedirect("home") forcePostProcess.exposed = True @@ -766,7 +749,7 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("logs") toggleVerbose.exposed = True - def getLog(self,iDisplayStart=0,iDisplayLength=100,iSortCol_0=0,sSortDir_0="desc",sSearch="",**kwargs): + def getLog(self, iDisplayStart=0, iDisplayLength=100, iSortCol_0=0, sSortDir_0="desc", sSearch="", **kwargs): iDisplayStart = int(iDisplayStart) iDisplayLength = int(iDisplayLength) @@ -782,26 +765,25 @@ class WebInterface(object): sortcolumn = 2 elif iSortCol_0 == '2': sortcolumn = 1 - filtered.sort(key=lambda x:x[sortcolumn],reverse=sSortDir_0 == "desc") + filtered.sort(key=lambda x: x[sortcolumn], reverse=sSortDir_0 == "desc") - rows = filtered[iDisplayStart:(iDisplayStart+iDisplayLength)] - rows = [[row[0],row[2],row[1]] for row in rows] + rows = filtered[iDisplayStart:(iDisplayStart + iDisplayLength)] + rows = [[row[0], row[2], row[1]] for row in rows] return json.dumps({ - 'iTotalDisplayRecords':len(filtered), - 'iTotalRecords':len(headphones.LOG_LIST), - 'aaData':rows, + 'iTotalDisplayRecords': len(filtered), + 'iTotalRecords': len(headphones.LOG_LIST), + 'aaData': rows, }) getLog.exposed = True - def getArtists_json(self,iDisplayStart=0,iDisplayLength=100,sSearch="",iSortCol_0='0',sSortDir_0='asc',**kwargs): + def getArtists_json(self, iDisplayStart=0, iDisplayLength=100, sSearch="", iSortCol_0='0', sSortDir_0='asc', **kwargs): iDisplayStart = int(iDisplayStart) iDisplayLength = int(iDisplayLength) filtered = [] totalcount = 0 myDB = db.DBConnection() - sortcolumn = 'ArtistSortName' sortbyhavepercent = False if iSortCol_0 == '2': @@ -812,36 +794,35 @@ class WebInterface(object): sortbyhavepercent = True if sSearch == "": - query = 'SELECT * from artists order by %s COLLATE NOCASE %s' % (sortcolumn,sSortDir_0) + query = 'SELECT * from artists order by %s COLLATE NOCASE %s' % (sortcolumn, sSortDir_0) filtered = myDB.select(query) totalcount = len(filtered) else: - query = 'SELECT * from artists WHERE ArtistSortName LIKE "%' + sSearch + '%" OR LatestAlbum LIKE "%' + sSearch +'%"' + 'ORDER BY %s COLLATE NOCASE %s' % (sortcolumn,sSortDir_0) + query = 'SELECT * from artists WHERE ArtistSortName LIKE "%' + sSearch + '%" OR LatestAlbum LIKE "%' + sSearch + '%"' + 'ORDER BY %s COLLATE NOCASE %s' % (sortcolumn, sSortDir_0) filtered = myDB.select(query) totalcount = myDB.select('SELECT COUNT(*) from artists')[0][0] if sortbyhavepercent: - filtered.sort(key=lambda x:(float(x['HaveTracks'])/x['TotalTracks'] if x['TotalTracks'] > 0 else 0.0,x['HaveTracks'] if x['HaveTracks'] else 0.0),reverse=sSortDir_0 == "asc") + filtered.sort(key=lambda x: (float(x['HaveTracks']) / x['TotalTracks'] if x['TotalTracks'] > 0 else 0.0, x['HaveTracks'] if x['HaveTracks'] else 0.0), reverse=sSortDir_0 == "asc") #can't figure out how to change the datatables default sorting order when its using an ajax datasource so ill #just reverse it here and the first click on the "Latest Album" header will sort by descending release date if sortcolumn == 'ReleaseDate': filtered.reverse() - - artists = filtered[iDisplayStart:(iDisplayStart+iDisplayLength)] + artists = filtered[iDisplayStart:(iDisplayStart + iDisplayLength)] rows = [] for artist in artists: - row = {"ArtistID":artist['ArtistID'], - "ArtistName":artist["ArtistName"], - "ArtistSortName":artist["ArtistSortName"], - "Status":artist["Status"], - "TotalTracks":artist["TotalTracks"], - "HaveTracks":artist["HaveTracks"], - "LatestAlbum":"", - "ReleaseDate":"", - "ReleaseInFuture":"False", - "AlbumID":"", + row = {"ArtistID": artist['ArtistID'], + "ArtistName": artist["ArtistName"], + "ArtistSortName": artist["ArtistSortName"], + "Status": artist["Status"], + "TotalTracks": artist["TotalTracks"], + "HaveTracks": artist["HaveTracks"], + "LatestAlbum": "", + "ReleaseDate": "", + "ReleaseInFuture": "False", + "AlbumID": "", } if not row['HaveTracks']: @@ -859,15 +840,14 @@ class WebInterface(object): rows.append(row) - - dict = {'iTotalDisplayRecords':len(filtered), - 'iTotalRecords':totalcount, - 'aaData':rows, + dict = {'iTotalDisplayRecords': len(filtered), + 'iTotalRecords': totalcount, + 'aaData': rows, } s = json.dumps(dict) cherrypy.response.headers['Content-type'] = 'application/json' return s - getArtists_json.exposed=True + getArtists_json.exposed = True def getAlbumsByArtist_json(self, artist=None): myDB = db.DBConnection() @@ -876,22 +856,22 @@ class WebInterface(object): album_list = myDB.select("SELECT AlbumTitle from albums WHERE ArtistName=?", [artist]) for album in album_list: album_json[counter] = album['AlbumTitle'] - counter+=1 + counter += 1 json_albums = json.dumps(album_json) cherrypy.response.headers['Content-type'] = 'application/json' return json_albums - getAlbumsByArtist_json.exposed=True + getAlbumsByArtist_json.exposed = True def getArtistjson(self, ArtistID, **kwargs): myDB = db.DBConnection() artist = myDB.action('SELECT * FROM artists WHERE ArtistID=?', [ArtistID]).fetchone() artist_json = json.dumps({ 'ArtistName': artist['ArtistName'], - 'Status': artist['Status'] + 'Status': artist['Status'] }) return artist_json - getArtistjson.exposed=True + getArtistjson.exposed = True def getAlbumjson(self, AlbumID, **kwargs): myDB = db.DBConnection() @@ -899,10 +879,10 @@ class WebInterface(object): album_json = json.dumps({ 'AlbumTitle': album['AlbumTitle'], 'ArtistName': album['ArtistName'], - 'Status': album['Status'] + 'Status': album['Status'] }) return album_json - getAlbumjson.exposed=True + getAlbumjson.exposed = True def clearhistory(self, type=None, date_added=None, title=None): myDB = db.DBConnection() @@ -921,9 +901,10 @@ class WebInterface(object): def generateAPI(self): - import hashlib, random + import hashlib + import random - apikey = hashlib.sha224( str(random.getrandbits(256)) ).hexdigest()[0:32] + apikey = hashlib.sha224(str(random.getrandbits(256))).hexdigest()[0:32] logger.info("New API generated") return apikey @@ -944,7 +925,7 @@ class WebInterface(object): logger.info('Marking all unwanted albums as Skipped') try: threading.Thread(target=librarysync.libraryScan).start() - except Exception, e: + except Exception as e: logger.error('Unable to complete the scan: %s' % e) raise cherrypy.HTTPRedirect("home") forceScan.exposed = True @@ -952,215 +933,224 @@ class WebInterface(object): def config(self): interface_dir = os.path.join(headphones.PROG_DIR, 'data/interfaces/') - interface_list = [ name for name in os.listdir(interface_dir) if os.path.isdir(os.path.join(interface_dir, name)) ] + interface_list = [name for name in os.listdir(interface_dir) if os.path.isdir(os.path.join(interface_dir, name))] config = { - "http_host" : headphones.HTTP_HOST, - "http_user" : headphones.HTTP_USERNAME, - "http_port" : headphones.HTTP_PORT, - "http_pass" : headphones.HTTP_PASSWORD, - "launch_browser" : checked(headphones.LAUNCH_BROWSER), - "enable_https" : checked(headphones.ENABLE_HTTPS), - "https_cert" : headphones.HTTPS_CERT, - "https_key" : headphones.HTTPS_KEY, - "api_enabled" : checked(headphones.API_ENABLED), - "api_key" : headphones.API_KEY, - "download_scan_interval" : headphones.DOWNLOAD_SCAN_INTERVAL, - "update_db_interval" : headphones.UPDATE_DB_INTERVAL, - "mb_ignore_age" : headphones.MB_IGNORE_AGE, - "search_interval" : headphones.SEARCH_INTERVAL, - "libraryscan_interval" : headphones.LIBRARYSCAN_INTERVAL, - "sab_host" : headphones.SAB_HOST, - "sab_user" : headphones.SAB_USERNAME, - "sab_api" : headphones.SAB_APIKEY, - "sab_pass" : headphones.SAB_PASSWORD, - "sab_cat" : headphones.SAB_CATEGORY, - "nzbget_host" : headphones.NZBGET_HOST, - "nzbget_user" : headphones.NZBGET_USERNAME, - "nzbget_pass" : headphones.NZBGET_PASSWORD, - "nzbget_cat" : headphones.NZBGET_CATEGORY, - "nzbget_priority" : headphones.NZBGET_PRIORITY, - "transmission_host" : headphones.TRANSMISSION_HOST, - "transmission_user" : headphones.TRANSMISSION_USERNAME, - "transmission_pass" : headphones.TRANSMISSION_PASSWORD, - "utorrent_host" : headphones.UTORRENT_HOST, - "utorrent_user" : headphones.UTORRENT_USERNAME, - "utorrent_pass" : headphones.UTORRENT_PASSWORD, - "utorrent_label" : headphones.UTORRENT_LABEL, - "nzb_downloader_sabnzbd" : radio(headphones.NZB_DOWNLOADER, 0), - "nzb_downloader_nzbget" : radio(headphones.NZB_DOWNLOADER, 1), - "nzb_downloader_blackhole" : radio(headphones.NZB_DOWNLOADER, 2), - "torrent_downloader_blackhole" : radio(headphones.TORRENT_DOWNLOADER, 0), - "torrent_downloader_transmission" : radio(headphones.TORRENT_DOWNLOADER, 1), - "torrent_downloader_utorrent" : radio(headphones.TORRENT_DOWNLOADER, 2), - "download_dir" : headphones.DOWNLOAD_DIR, - "use_blackhole" : checked(headphones.BLACKHOLE), - "blackhole_dir" : headphones.BLACKHOLE_DIR, - "usenet_retention" : headphones.USENET_RETENTION, - "use_headphones_indexer" : checked(headphones.HEADPHONES_INDEXER), - "use_newznab" : checked(headphones.NEWZNAB), - "newznab_host" : headphones.NEWZNAB_HOST, - "newznab_api" : headphones.NEWZNAB_APIKEY, - "newznab_enabled" : checked(headphones.NEWZNAB_ENABLED), - "extra_newznabs" : headphones.EXTRA_NEWZNABS, - "use_nzbsorg" : checked(headphones.NZBSORG), - "nzbsorg_uid" : headphones.NZBSORG_UID, - "nzbsorg_hash" : headphones.NZBSORG_HASH, - "use_omgwtfnzbs" : checked(headphones.OMGWTFNZBS), - "omgwtfnzbs_uid" : headphones.OMGWTFNZBS_UID, - "omgwtfnzbs_apikey" : headphones.OMGWTFNZBS_APIKEY, - "preferred_words" : headphones.PREFERRED_WORDS, - "ignored_words" : headphones.IGNORED_WORDS, - "required_words" : headphones.REQUIRED_WORDS, - "torrentblackhole_dir" : headphones.TORRENTBLACKHOLE_DIR, - "download_torrent_dir" : headphones.DOWNLOAD_TORRENT_DIR, - "numberofseeders" : headphones.NUMBEROFSEEDERS, - "use_kat" : checked(headphones.KAT), - "kat_proxy_url" : headphones.KAT_PROXY_URL, - "kat_ratio": headphones.KAT_RATIO, - "use_piratebay" : checked(headphones.PIRATEBAY), - "piratebay_proxy_url" : headphones.PIRATEBAY_PROXY_URL, - "piratebay_ratio": headphones.PIRATEBAY_RATIO, - "use_mininova" : checked(headphones.MININOVA), - "mininova_ratio": headphones.MININOVA_RATIO, - "use_waffles" : checked(headphones.WAFFLES), - "waffles_uid" : headphones.WAFFLES_UID, - "waffles_passkey": headphones.WAFFLES_PASSKEY, - "waffles_ratio": headphones.WAFFLES_RATIO, - "use_rutracker" : checked(headphones.RUTRACKER), - "rutracker_user" : headphones.RUTRACKER_USER, - "rutracker_password": headphones.RUTRACKER_PASSWORD, - "rutracker_ratio": headphones.RUTRACKER_RATIO, - "use_whatcd" : checked(headphones.WHATCD), - "whatcd_username" : headphones.WHATCD_USERNAME, - "whatcd_password": headphones.WHATCD_PASSWORD, - "whatcd_ratio": headphones.WHATCD_RATIO, - "pref_qual_0" : radio(headphones.PREFERRED_QUALITY, 0), - "pref_qual_1" : radio(headphones.PREFERRED_QUALITY, 1), - "pref_qual_3" : radio(headphones.PREFERRED_QUALITY, 3), - "pref_qual_2" : radio(headphones.PREFERRED_QUALITY, 2), - "pref_bitrate" : headphones.PREFERRED_BITRATE, - "pref_bitrate_high" : headphones.PREFERRED_BITRATE_HIGH_BUFFER, - "pref_bitrate_low" : headphones.PREFERRED_BITRATE_LOW_BUFFER, - "pref_bitrate_allow_lossless" : checked(headphones.PREFERRED_BITRATE_ALLOW_LOSSLESS), - "detect_bitrate" : checked(headphones.DETECT_BITRATE), - "lossless_bitrate_from" : headphones.LOSSLESS_BITRATE_FROM, - "lossless_bitrate_to" : headphones.LOSSLESS_BITRATE_TO, - "freeze_db" : checked(headphones.FREEZE_DB), - "move_files" : checked(headphones.MOVE_FILES), - "rename_files" : checked(headphones.RENAME_FILES), - "correct_metadata" : checked(headphones.CORRECT_METADATA), - "cleanup_files" : checked(headphones.CLEANUP_FILES), - "keep_nfo" : checked(headphones.KEEP_NFO), - "add_album_art" : checked(headphones.ADD_ALBUM_ART), - "album_art_format" : headphones.ALBUM_ART_FORMAT, - "embed_album_art" : checked(headphones.EMBED_ALBUM_ART), - "embed_lyrics" : checked(headphones.EMBED_LYRICS), - "replace_existing_folders" : checked(headphones.REPLACE_EXISTING_FOLDERS), - "dest_dir" : headphones.DESTINATION_DIR, - "lossless_dest_dir" : headphones.LOSSLESS_DESTINATION_DIR, - "folder_format" : headphones.FOLDER_FORMAT, - "file_format" : headphones.FILE_FORMAT, - "file_underscores" : checked(headphones.FILE_UNDERSCORES), - "include_extras" : checked(headphones.INCLUDE_EXTRAS), - "autowant_upcoming" : checked(headphones.AUTOWANT_UPCOMING), - "autowant_all" : checked(headphones.AUTOWANT_ALL), - "autowant_manually_added" : checked(headphones.AUTOWANT_MANUALLY_ADDED), - "keep_torrent_files" : checked(headphones.KEEP_TORRENT_FILES), - "prefer_torrents_0" : radio(headphones.PREFER_TORRENTS, 0), - "prefer_torrents_1" : radio(headphones.PREFER_TORRENTS, 1), - "prefer_torrents_2" : radio(headphones.PREFER_TORRENTS, 2), - "open_magnet_links" : checked(headphones.OPEN_MAGNET_LINKS), - "log_dir" : headphones.LOG_DIR, - "cache_dir" : headphones.CACHE_DIR, - "interface_list" : interface_list, - "music_encoder": checked(headphones.MUSIC_ENCODER), - "encoder": headphones.ENCODER, - "xldprofile": headphones.XLDPROFILE, - "bitrate": int(headphones.BITRATE), - "encoderfolder": headphones.ENCODER_PATH, - "advancedencoder": headphones.ADVANCEDENCODER, - "encoderoutputformat": headphones.ENCODEROUTPUTFORMAT, - "samplingfrequency": headphones.SAMPLINGFREQUENCY, - "encodervbrcbr": headphones.ENCODERVBRCBR, - "encoderquality": headphones.ENCODERQUALITY, - "encoderlossless": checked(headphones.ENCODERLOSSLESS), - "encoder_multicore": checked(headphones.ENCODER_MULTICORE), - "encoder_multicore_count": int(headphones.ENCODER_MULTICORE_COUNT), - "delete_lossless_files": checked(headphones.DELETE_LOSSLESS_FILES), - "growl_enabled": checked(headphones.GROWL_ENABLED), - "growl_onsnatch": checked(headphones.GROWL_ONSNATCH), - "growl_host": headphones.GROWL_HOST, - "growl_password": headphones.GROWL_PASSWORD, - "prowl_enabled": checked(headphones.PROWL_ENABLED), - "prowl_onsnatch": checked(headphones.PROWL_ONSNATCH), - "prowl_keys": headphones.PROWL_KEYS, - "prowl_priority": headphones.PROWL_PRIORITY, - "xbmc_enabled": checked(headphones.XBMC_ENABLED), - "xbmc_host": headphones.XBMC_HOST, - "xbmc_username": headphones.XBMC_USERNAME, - "xbmc_password": headphones.XBMC_PASSWORD, - "xbmc_update": checked(headphones.XBMC_UPDATE), - "xbmc_notify": checked(headphones.XBMC_NOTIFY), - "lms_enabled": checked(headphones.LMS_ENABLED), - "lms_host": headphones.LMS_HOST, - "plex_enabled": checked(headphones.PLEX_ENABLED), - "plex_server_host": headphones.PLEX_SERVER_HOST, - "plex_client_host": headphones.PLEX_CLIENT_HOST, - "plex_username": headphones.PLEX_USERNAME, - "plex_password": headphones.PLEX_PASSWORD, - "plex_update": checked(headphones.PLEX_UPDATE), - "plex_notify": checked(headphones.PLEX_NOTIFY), - "nma_enabled": checked(headphones.NMA_ENABLED), - "nma_apikey": headphones.NMA_APIKEY, - "nma_priority": int(headphones.NMA_PRIORITY), - "nma_onsnatch": checked(headphones.NMA_ONSNATCH), - "pushalot_enabled": checked(headphones.PUSHALOT_ENABLED), - "pushalot_apikey": headphones.PUSHALOT_APIKEY, - "pushalot_onsnatch": checked(headphones.PUSHALOT_ONSNATCH), - "synoindex_enabled": checked(headphones.SYNOINDEX_ENABLED), - "pushover_enabled": checked(headphones.PUSHOVER_ENABLED), - "pushover_onsnatch": checked(headphones.PUSHOVER_ONSNATCH), - "pushover_keys": headphones.PUSHOVER_KEYS, - "pushover_apitoken": headphones.PUSHOVER_APITOKEN, - "pushover_priority": headphones.PUSHOVER_PRIORITY, - "pushbullet_enabled": checked(headphones.PUSHBULLET_ENABLED), - "pushbullet_onsnatch": checked(headphones.PUSHBULLET_ONSNATCH), - "pushbullet_apikey": headphones.PUSHBULLET_APIKEY, - "pushbullet_deviceid": headphones.PUSHBULLET_DEVICEID, - "subsonic_enabled": checked(headphones.SUBSONIC_ENABLED), - "subsonic_host": headphones.SUBSONIC_HOST, - "subsonic_username": headphones.SUBSONIC_USERNAME, - "subsonic_password": headphones.SUBSONIC_PASSWORD, - "twitter_enabled": checked(headphones.TWITTER_ENABLED), - "twitter_onsnatch": checked(headphones.TWITTER_ONSNATCH), - "osx_notify_enabled": checked(headphones.OSX_NOTIFY_ENABLED), - "osx_notify_onsnatch": checked(headphones.OSX_NOTIFY_ONSNATCH), - "osx_notify_app": headphones.OSX_NOTIFY_APP, - "boxcar_enabled": checked(headphones.BOXCAR_ENABLED), - "boxcar_onsnatch": checked(headphones.BOXCAR_ONSNATCH), - "boxcar_token": headphones.BOXCAR_TOKEN, - "mirror_list": headphones.MIRRORLIST, - "mirror": headphones.MIRROR, - "customhost": headphones.CUSTOMHOST, - "customport": headphones.CUSTOMPORT, - "customsleep": headphones.CUSTOMSLEEP, - "hpuser": headphones.HPUSER, - "hppass": headphones.HPPASS, - "songkick_enabled": checked(headphones.SONGKICK_ENABLED), - "songkick_apikey": headphones.SONGKICK_APIKEY, - "songkick_location": headphones.SONGKICK_LOCATION, - "songkick_filter_enabled": checked(headphones.SONGKICK_FILTER_ENABLED), - "cache_sizemb": headphones.CACHE_SIZEMB, - "file_permissions": headphones.FILE_PERMISSIONS, - "folder_permissions": headphones.FOLDER_PERMISSIONS, - "mpc_enabled": checked(headphones.MPC_ENABLED) - } + "http_host": headphones.CONFIG.HTTP_HOST, + "http_username": headphones.CONFIG.HTTP_USERNAME, + "http_port": headphones.CONFIG.HTTP_PORT, + "http_password": headphones.CONFIG.HTTP_PASSWORD, + "launch_browser": checked(headphones.CONFIG.LAUNCH_BROWSER), + "enable_https": checked(headphones.CONFIG.ENABLE_HTTPS), + "https_cert": headphones.CONFIG.HTTPS_CERT, + "https_key": headphones.CONFIG.HTTPS_KEY, + "api_enabled": checked(headphones.CONFIG.API_ENABLED), + "api_key": headphones.CONFIG.API_KEY, + "download_scan_interval": headphones.CONFIG.DOWNLOAD_SCAN_INTERVAL, + "update_db_interval": headphones.CONFIG.UPDATE_DB_INTERVAL, + "mb_ignore_age": headphones.CONFIG.MB_IGNORE_AGE, + "search_interval": headphones.CONFIG.SEARCH_INTERVAL, + "libraryscan_interval": headphones.CONFIG.LIBRARYSCAN_INTERVAL, + "sab_host": headphones.CONFIG.SAB_HOST, + "sab_username": headphones.CONFIG.SAB_USERNAME, + "sab_apikey": headphones.CONFIG.SAB_APIKEY, + "sab_password": headphones.CONFIG.SAB_PASSWORD, + "sab_category": headphones.CONFIG.SAB_CATEGORY, + "nzbget_host": headphones.CONFIG.NZBGET_HOST, + "nzbget_username": headphones.CONFIG.NZBGET_USERNAME, + "nzbget_password": headphones.CONFIG.NZBGET_PASSWORD, + "nzbget_category": headphones.CONFIG.NZBGET_CATEGORY, + "nzbget_priority": headphones.CONFIG.NZBGET_PRIORITY, + "transmission_host": headphones.CONFIG.TRANSMISSION_HOST, + "transmission_username": headphones.CONFIG.TRANSMISSION_USERNAME, + "transmission_password": headphones.CONFIG.TRANSMISSION_PASSWORD, + "utorrent_host": headphones.CONFIG.UTORRENT_HOST, + "utorrent_username": headphones.CONFIG.UTORRENT_USERNAME, + "utorrent_password": headphones.CONFIG.UTORRENT_PASSWORD, + "utorrent_label": headphones.CONFIG.UTORRENT_LABEL, + "nzb_downloader_sabnzbd": radio(headphones.CONFIG.NZB_DOWNLOADER, 0), + "nzb_downloader_nzbget": radio(headphones.CONFIG.NZB_DOWNLOADER, 1), + "nzb_downloader_blackhole": radio(headphones.CONFIG.NZB_DOWNLOADER, 2), + "torrent_downloader_blackhole": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 0), + "torrent_downloader_transmission": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 1), + "torrent_downloader_utorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 2), + "download_dir": headphones.CONFIG.DOWNLOAD_DIR, + "use_blackhole": checked(headphones.CONFIG.BLACKHOLE), + "blackhole_dir": headphones.CONFIG.BLACKHOLE_DIR, + "usenet_retention": headphones.CONFIG.USENET_RETENTION, + "headphones_indexer": checked(headphones.CONFIG.HEADPHONES_INDEXER), + "use_newznab": checked(headphones.CONFIG.NEWZNAB), + "newznab_host": headphones.CONFIG.NEWZNAB_HOST, + "newznab_apikey": headphones.CONFIG.NEWZNAB_APIKEY, + "newznab_enabled": checked(headphones.CONFIG.NEWZNAB_ENABLED), + "extra_newznabs": headphones.CONFIG.get_extra_newznabs(), + "use_nzbsorg": checked(headphones.CONFIG.NZBSORG), + "nzbsorg_uid": headphones.CONFIG.NZBSORG_UID, + "nzbsorg_hash": headphones.CONFIG.NZBSORG_HASH, + "use_omgwtfnzbs": checked(headphones.CONFIG.OMGWTFNZBS), + "omgwtfnzbs_uid": headphones.CONFIG.OMGWTFNZBS_UID, + "omgwtfnzbs_apikey": headphones.CONFIG.OMGWTFNZBS_APIKEY, + "preferred_words": headphones.CONFIG.PREFERRED_WORDS, + "ignored_words": headphones.CONFIG.IGNORED_WORDS, + "required_words": headphones.CONFIG.REQUIRED_WORDS, + "torrentblackhole_dir": headphones.CONFIG.TORRENTBLACKHOLE_DIR, + "download_torrent_dir": headphones.CONFIG.DOWNLOAD_TORRENT_DIR, + "numberofseeders": headphones.CONFIG.NUMBEROFSEEDERS, + "use_kat": checked(headphones.CONFIG.KAT), + "kat_proxy_url": headphones.CONFIG.KAT_PROXY_URL, + "kat_ratio": headphones.CONFIG.KAT_RATIO, + "use_piratebay": checked(headphones.CONFIG.PIRATEBAY), + "piratebay_proxy_url": headphones.CONFIG.PIRATEBAY_PROXY_URL, + "piratebay_ratio": headphones.CONFIG.PIRATEBAY_RATIO, + "use_mininova": checked(headphones.CONFIG.MININOVA), + "mininova_ratio": headphones.CONFIG.MININOVA_RATIO, + "use_waffles": checked(headphones.CONFIG.WAFFLES), + "waffles_uid": headphones.CONFIG.WAFFLES_UID, + "waffles_passkey": headphones.CONFIG.WAFFLES_PASSKEY, + "waffles_ratio": headphones.CONFIG.WAFFLES_RATIO, + "use_rutracker": checked(headphones.CONFIG.RUTRACKER), + "rutracker_user": headphones.CONFIG.RUTRACKER_USER, + "rutracker_password": headphones.CONFIG.RUTRACKER_PASSWORD, + "rutracker_ratio": headphones.CONFIG.RUTRACKER_RATIO, + "use_whatcd": checked(headphones.CONFIG.WHATCD), + "whatcd_username": headphones.CONFIG.WHATCD_USERNAME, + "whatcd_password": headphones.CONFIG.WHATCD_PASSWORD, + "whatcd_ratio": headphones.CONFIG.WHATCD_RATIO, + "pref_qual_0": radio(headphones.CONFIG.PREFERRED_QUALITY, 0), + "pref_qual_1": radio(headphones.CONFIG.PREFERRED_QUALITY, 1), + "pref_qual_2": radio(headphones.CONFIG.PREFERRED_QUALITY, 2), + "pref_qual_3": radio(headphones.CONFIG.PREFERRED_QUALITY, 3), + "preferred_bitrate": headphones.CONFIG.PREFERRED_BITRATE, + "preferred_bitrate_high": headphones.CONFIG.PREFERRED_BITRATE_HIGH_BUFFER, + "preferred_bitrate_low": headphones.CONFIG.PREFERRED_BITRATE_LOW_BUFFER, + "preferred_bitrate_allow_lossless": checked(headphones.CONFIG.PREFERRED_BITRATE_ALLOW_LOSSLESS), + "detect_bitrate": checked(headphones.CONFIG.DETECT_BITRATE), + "lossless_bitrate_from": headphones.CONFIG.LOSSLESS_BITRATE_FROM, + "lossless_bitrate_to": headphones.CONFIG.LOSSLESS_BITRATE_TO, + "freeze_db": checked(headphones.CONFIG.FREEZE_DB), + "cue_split": checked(headphones.CONFIG.CUE_SPLIT), + "move_files": checked(headphones.CONFIG.MOVE_FILES), + "rename_files": checked(headphones.CONFIG.RENAME_FILES), + "correct_metadata": checked(headphones.CONFIG.CORRECT_METADATA), + "cleanup_files": checked(headphones.CONFIG.CLEANUP_FILES), + "keep_nfo": checked(headphones.CONFIG.KEEP_NFO), + "add_album_art": checked(headphones.CONFIG.ADD_ALBUM_ART), + "album_art_format": headphones.CONFIG.ALBUM_ART_FORMAT, + "embed_album_art": checked(headphones.CONFIG.EMBED_ALBUM_ART), + "embed_lyrics": checked(headphones.CONFIG.EMBED_LYRICS), + "replace_existing_folders": checked(headphones.CONFIG.REPLACE_EXISTING_FOLDERS), + "destination_dir": headphones.CONFIG.DESTINATION_DIR, + "lossless_destination_dir": headphones.CONFIG.LOSSLESS_DESTINATION_DIR, + "folder_format": headphones.CONFIG.FOLDER_FORMAT, + "file_format": headphones.CONFIG.FILE_FORMAT, + "file_underscores": checked(headphones.CONFIG.FILE_UNDERSCORES), + "include_extras": checked(headphones.CONFIG.INCLUDE_EXTRAS), + "autowant_upcoming": checked(headphones.CONFIG.AUTOWANT_UPCOMING), + "autowant_all": checked(headphones.CONFIG.AUTOWANT_ALL), + "autowant_manually_added": checked(headphones.CONFIG.AUTOWANT_MANUALLY_ADDED), + "keep_torrent_files": checked(headphones.CONFIG.KEEP_TORRENT_FILES), + "prefer_torrents_0": radio(headphones.CONFIG.PREFER_TORRENTS, 0), + "prefer_torrents_1": radio(headphones.CONFIG.PREFER_TORRENTS, 1), + "prefer_torrents_2": radio(headphones.CONFIG.PREFER_TORRENTS, 2), + "magnet_links_0": radio(headphones.CONFIG.MAGNET_LINKS, 0), + "magnet_links_1": radio(headphones.CONFIG.MAGNET_LINKS, 1), + "magnet_links_2": radio(headphones.CONFIG.MAGNET_LINKS, 2), + "log_dir": headphones.CONFIG.LOG_DIR, + "cache_dir": headphones.CONFIG.CACHE_DIR, + "interface_list": interface_list, + "music_encoder": checked(headphones.CONFIG.MUSIC_ENCODER), + "encoder": headphones.CONFIG.ENCODER, + "xldprofile": headphones.CONFIG.XLDPROFILE, + "bitrate": int(headphones.CONFIG.BITRATE), + "encoder_path": headphones.CONFIG.ENCODER_PATH, + "advancedencoder": headphones.CONFIG.ADVANCEDENCODER, + "encoderoutputformat": headphones.CONFIG.ENCODEROUTPUTFORMAT, + "samplingfrequency": headphones.CONFIG.SAMPLINGFREQUENCY, + "encodervbrcbr": headphones.CONFIG.ENCODERVBRCBR, + "encoderquality": headphones.CONFIG.ENCODERQUALITY, + "encoderlossless": checked(headphones.CONFIG.ENCODERLOSSLESS), + "encoder_multicore": checked(headphones.CONFIG.ENCODER_MULTICORE), + "encoder_multicore_count": int(headphones.CONFIG.ENCODER_MULTICORE_COUNT), + "delete_lossless_files": checked(headphones.CONFIG.DELETE_LOSSLESS_FILES), + "growl_enabled": checked(headphones.CONFIG.GROWL_ENABLED), + "growl_onsnatch": checked(headphones.CONFIG.GROWL_ONSNATCH), + "growl_host": headphones.CONFIG.GROWL_HOST, + "growl_password": headphones.CONFIG.GROWL_PASSWORD, + "prowl_enabled": checked(headphones.CONFIG.PROWL_ENABLED), + "prowl_onsnatch": checked(headphones.CONFIG.PROWL_ONSNATCH), + "prowl_keys": headphones.CONFIG.PROWL_KEYS, + "prowl_priority": headphones.CONFIG.PROWL_PRIORITY, + "xbmc_enabled": checked(headphones.CONFIG.XBMC_ENABLED), + "xbmc_host": headphones.CONFIG.XBMC_HOST, + "xbmc_username": headphones.CONFIG.XBMC_USERNAME, + "xbmc_password": headphones.CONFIG.XBMC_PASSWORD, + "xbmc_update": checked(headphones.CONFIG.XBMC_UPDATE), + "xbmc_notify": checked(headphones.CONFIG.XBMC_NOTIFY), + "lms_enabled": checked(headphones.CONFIG.LMS_ENABLED), + "lms_host": headphones.CONFIG.LMS_HOST, + "plex_enabled": checked(headphones.CONFIG.PLEX_ENABLED), + "plex_server_host": headphones.CONFIG.PLEX_SERVER_HOST, + "plex_client_host": headphones.CONFIG.PLEX_CLIENT_HOST, + "plex_username": headphones.CONFIG.PLEX_USERNAME, + "plex_password": headphones.CONFIG.PLEX_PASSWORD, + "plex_update": checked(headphones.CONFIG.PLEX_UPDATE), + "plex_notify": checked(headphones.CONFIG.PLEX_NOTIFY), + "nma_enabled": checked(headphones.CONFIG.NMA_ENABLED), + "nma_apikey": headphones.CONFIG.NMA_APIKEY, + "nma_priority": int(headphones.CONFIG.NMA_PRIORITY), + "nma_onsnatch": checked(headphones.CONFIG.NMA_ONSNATCH), + "pushalot_enabled": checked(headphones.CONFIG.PUSHALOT_ENABLED), + "pushalot_apikey": headphones.CONFIG.PUSHALOT_APIKEY, + "pushalot_onsnatch": checked(headphones.CONFIG.PUSHALOT_ONSNATCH), + "synoindex_enabled": checked(headphones.CONFIG.SYNOINDEX_ENABLED), + "pushover_enabled": checked(headphones.CONFIG.PUSHOVER_ENABLED), + "pushover_onsnatch": checked(headphones.CONFIG.PUSHOVER_ONSNATCH), + "pushover_keys": headphones.CONFIG.PUSHOVER_KEYS, + "pushover_apitoken": headphones.CONFIG.PUSHOVER_APITOKEN, + "pushover_priority": headphones.CONFIG.PUSHOVER_PRIORITY, + "pushbullet_enabled": checked(headphones.CONFIG.PUSHBULLET_ENABLED), + "pushbullet_onsnatch": checked(headphones.CONFIG.PUSHBULLET_ONSNATCH), + "pushbullet_apikey": headphones.CONFIG.PUSHBULLET_APIKEY, + "pushbullet_deviceid": headphones.CONFIG.PUSHBULLET_DEVICEID, + "subsonic_enabled": checked(headphones.CONFIG.SUBSONIC_ENABLED), + "subsonic_host": headphones.CONFIG.SUBSONIC_HOST, + "subsonic_username": headphones.CONFIG.SUBSONIC_USERNAME, + "subsonic_password": headphones.CONFIG.SUBSONIC_PASSWORD, + "twitter_enabled": checked(headphones.CONFIG.TWITTER_ENABLED), + "twitter_onsnatch": checked(headphones.CONFIG.TWITTER_ONSNATCH), + "osx_notify_enabled": checked(headphones.CONFIG.OSX_NOTIFY_ENABLED), + "osx_notify_onsnatch": checked(headphones.CONFIG.OSX_NOTIFY_ONSNATCH), + "osx_notify_app": headphones.CONFIG.OSX_NOTIFY_APP, + "boxcar_enabled": checked(headphones.CONFIG.BOXCAR_ENABLED), + "boxcar_onsnatch": checked(headphones.CONFIG.BOXCAR_ONSNATCH), + "boxcar_token": headphones.CONFIG.BOXCAR_TOKEN, + "mirrorlist": headphones.MIRRORLIST, + "mirror": headphones.CONFIG.MIRROR, + "customhost": headphones.CONFIG.CUSTOMHOST, + "customport": headphones.CONFIG.CUSTOMPORT, + "customsleep": headphones.CONFIG.CUSTOMSLEEP, + "hpuser": headphones.CONFIG.HPUSER, + "hppass": headphones.CONFIG.HPPASS, + "songkick_enabled": checked(headphones.CONFIG.SONGKICK_ENABLED), + "songkick_apikey": headphones.CONFIG.SONGKICK_APIKEY, + "songkick_location": headphones.CONFIG.SONGKICK_LOCATION, + "songkick_filter_enabled": checked(headphones.CONFIG.SONGKICK_FILTER_ENABLED), + "cache_sizemb": headphones.CONFIG.CACHE_SIZEMB, + "file_permissions": headphones.CONFIG.FILE_PERMISSIONS, + "folder_permissions": headphones.CONFIG.FOLDER_PERMISSIONS, + "mpc_enabled": checked(headphones.CONFIG.MPC_ENABLED) + } - # Need to convert EXTRAS to a dictionary we can pass to the config: it'll come in as a string like 2,5,6,8 (append new extras to the end) - extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "spokenword", "audiobook", "other", "djmix", "mixtape_street", "broadcast", "interview", "demo"] - if headphones.EXTRAS: - extras = map(int, headphones.EXTRAS.split(',')) + # Need to convert EXTRAS to a dictionary we can pass to the config: + # it'll come in as a string like 2,5,6,8 + extra_munges = { + "dj-mix": "dj_mix", + "mixtape/street": "mixtape_street" + } + + extras_list = [extra_munges.get(x, x) for x in headphones.POSSIBLE_EXTRAS] + if headphones.CONFIG.EXTRAS: + extras = map(int, headphones.CONFIG.EXTRAS.split(',')) else: extras = [] @@ -1172,260 +1162,91 @@ class WebInterface(object): extras_dict[extra] = "checked" else: extras_dict[extra] = "" - i+=1 + i += 1 config["extras"] = extras_dict return serve_template(templatename="config.html", title="Settings", config=config) config.exposed = True - def configUpdate(self, http_host='0.0.0.0', http_username=None, http_port=8181, http_password=None, launch_browser=0, api_enabled=0, api_key=None, - download_scan_interval=None, update_db_interval=None, mb_ignore_age=None, search_interval=None, libraryscan_interval=None, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None, - sab_category=None, nzbget_host=None, nzbget_username=None, nzbget_password=None, nzbget_category=None, nzbget_priority=0, transmission_host=None, transmission_username=None, transmission_password=None, - utorrent_host=None, utorrent_username=None, utorrent_password=None, utorrent_label=None,nzb_downloader=0, torrent_downloader=0, download_dir=None, blackhole_dir=None, usenet_retention=None, - use_headphones_indexer=0, newznab=0, newznab_host=None, newznab_apikey=None, newznab_enabled=0, nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, omgwtfnzbs=0, omgwtfnzbs_uid=None, omgwtfnzbs_apikey=None, - preferred_words=None, required_words=None, ignored_words=None, preferred_quality=0, preferred_bitrate=None, detect_bitrate=0, freeze_db=0, move_files=0, torrentblackhole_dir=None, download_torrent_dir=None, - numberofseeders=None, use_piratebay=0, piratebay_proxy_url=None, piratebay_ratio=None, use_kat=0, kat_proxy_url=None, kat_ratio=None, use_mininova=0, mininova_ratio=None, waffles=0, waffles_uid=None, waffles_passkey=None, waffles_ratio=None, whatcd=0, whatcd_username=None, whatcd_password=None, whatcd_ratio=None, - rutracker=0, rutracker_user=None, rutracker_password=None, rutracker_ratio=None, rename_files=0, correct_metadata=0, cleanup_files=0, keep_nfo=0, add_album_art=0, album_art_format=None, embed_album_art=0, embed_lyrics=0, replace_existing_folders=False, - destination_dir=None, lossless_destination_dir=None, folder_format=None, file_format=None, file_underscores=0, include_extras=0, single=0, ep=0, compilation=0, soundtrack=0, live=0, remix=0, spokenword=0, audiobook=0, other=0, djmix=0, mixtape_street=0, broadcast=0, interview=0, demo=0, - autowant_upcoming=False, autowant_all=False, autowant_manually_added=False, keep_torrent_files=False, prefer_torrents=0, open_magnet_links=0, interface=None, log_dir=None, cache_dir=None, music_encoder=0, encoder=None, xldprofile=None, - bitrate=None, samplingfrequency=None, encoderfolder=None, advancedencoder=None, encoderoutputformat=None, encodervbrcbr=None, encoderquality=None, encoderlossless=0, subsonic_enabled=False, subsonic_host=None, subsonic_username=None, subsonic_password=None, - delete_lossless_files=0, growl_enabled=0, growl_onsnatch=0, growl_host=None, growl_password=None, prowl_enabled=0, prowl_onsnatch=0, prowl_keys=None, prowl_priority=0, xbmc_enabled=0, xbmc_host=None, xbmc_username=None, xbmc_password=None, - xbmc_update=0, xbmc_notify=0, nma_enabled=False, nma_apikey=None, nma_priority=0, nma_onsnatch=0, pushalot_enabled=False, pushalot_apikey=None, pushalot_onsnatch=0, synoindex_enabled=False, lms_enabled=0, lms_host=None, - pushover_enabled=0, pushover_onsnatch=0, pushover_keys=None, pushover_priority=0, pushover_apitoken=None, pushbullet_enabled=0, pushbullet_onsnatch=0, pushbullet_apikey=None, pushbullet_deviceid=None, twitter_enabled=0, twitter_onsnatch=0, - osx_notify_enabled=0, osx_notify_onsnatch=0, osx_notify_app=None, boxcar_enabled=0, boxcar_onsnatch=0, boxcar_token=None, mirror=None, customhost=None, customport=None, customsleep=None, hpuser=None, hppass=None, - preferred_bitrate_high_buffer=None, preferred_bitrate_low_buffer=None, preferred_bitrate_allow_lossless=0, lossless_bitrate_from=None, lossless_bitrate_to=None, cache_sizemb=None, enable_https=0, https_cert=None, https_key=None, - file_permissions=None, folder_permissions=None, plex_enabled=0, plex_server_host=None, plex_client_host=None, plex_username=None, plex_password=None, plex_update=0, plex_notify=0, - songkick_enabled=0, songkick_apikey=None, songkick_location=None, songkick_filter_enabled=0, encoder_multicore=False, encoder_multicore_count=0, mpc_enabled=False, **kwargs ): - - headphones.HTTP_HOST = http_host - headphones.HTTP_PORT = http_port - headphones.HTTP_USERNAME = http_username - headphones.HTTP_PASSWORD = http_password - headphones.LAUNCH_BROWSER = launch_browser - headphones.ENABLE_HTTPS = enable_https - headphones.HTTPS_CERT = https_cert - headphones.HTTPS_KEY = https_key - headphones.API_ENABLED = api_enabled - headphones.API_KEY = api_key - headphones.DOWNLOAD_SCAN_INTERVAL = download_scan_interval - headphones.UPDATE_DB_INTERVAL = update_db_interval - headphones.MB_IGNORE_AGE = mb_ignore_age - headphones.SEARCH_INTERVAL = search_interval - headphones.LIBRARYSCAN_INTERVAL = libraryscan_interval - headphones.SAB_HOST = sab_host - headphones.SAB_USERNAME = sab_username - headphones.SAB_PASSWORD = sab_password - headphones.SAB_APIKEY = sab_apikey - headphones.SAB_CATEGORY = sab_category - headphones.NZBGET_HOST = nzbget_host - headphones.NZBGET_USERNAME = nzbget_username - headphones.NZBGET_PASSWORD = nzbget_password - headphones.NZBGET_CATEGORY = nzbget_category - headphones.NZBGET_PRIORITY = int(nzbget_priority) - headphones.TRANSMISSION_HOST = transmission_host - headphones.TRANSMISSION_USERNAME = transmission_username - headphones.TRANSMISSION_PASSWORD = transmission_password - headphones.UTORRENT_HOST = utorrent_host - headphones.UTORRENT_USERNAME = utorrent_username - headphones.UTORRENT_PASSWORD = utorrent_password - headphones.UTORRENT_LABEL = utorrent_label - headphones.NZB_DOWNLOADER = int(nzb_downloader) - headphones.TORRENT_DOWNLOADER = int(torrent_downloader) - headphones.DOWNLOAD_DIR = download_dir - headphones.BLACKHOLE_DIR = blackhole_dir - headphones.USENET_RETENTION = usenet_retention - headphones.HEADPHONES_INDEXER = use_headphones_indexer - headphones.NEWZNAB = newznab - headphones.NEWZNAB_HOST = newznab_host - headphones.NEWZNAB_APIKEY = newznab_apikey - headphones.NEWZNAB_ENABLED = newznab_enabled - headphones.NZBSORG = nzbsorg - headphones.NZBSORG_UID = nzbsorg_uid - headphones.NZBSORG_HASH = nzbsorg_hash - headphones.OMGWTFNZBS = omgwtfnzbs - headphones.OMGWTFNZBS_UID = omgwtfnzbs_uid - headphones.OMGWTFNZBS_APIKEY = omgwtfnzbs_apikey - headphones.PREFERRED_WORDS = preferred_words - headphones.IGNORED_WORDS = ignored_words - headphones.REQUIRED_WORDS = required_words - headphones.TORRENTBLACKHOLE_DIR = torrentblackhole_dir - headphones.NUMBEROFSEEDERS = numberofseeders - headphones.DOWNLOAD_TORRENT_DIR = download_torrent_dir - headphones.KAT = use_kat - headphones.KAT_PROXY_URL = kat_proxy_url - headphones.KAT_RATIO = kat_ratio - headphones.PIRATEBAY = use_piratebay - headphones.PIRATEBAY_PROXY_URL = piratebay_proxy_url - headphones.PIRATEBAY_RATIO = piratebay_ratio - headphones.MININOVA = use_mininova - headphones.MININOVA_RATIO = mininova_ratio - headphones.WAFFLES = waffles - headphones.WAFFLES_UID = waffles_uid - headphones.WAFFLES_PASSKEY = waffles_passkey - headphones.WAFFLES_RATIO = waffles_ratio - headphones.RUTRACKER = rutracker - headphones.RUTRACKER_USER = rutracker_user - headphones.RUTRACKER_PASSWORD = rutracker_password - headphones.RUTRACKER_RATIO = rutracker_ratio - headphones.WHATCD = whatcd - headphones.WHATCD_USERNAME = whatcd_username - headphones.WHATCD_PASSWORD = whatcd_password - headphones.WHATCD_RATIO = whatcd_ratio - headphones.PREFERRED_QUALITY = int(preferred_quality) - headphones.PREFERRED_BITRATE = preferred_bitrate - headphones.PREFERRED_BITRATE_HIGH_BUFFER = preferred_bitrate_high_buffer - headphones.PREFERRED_BITRATE_LOW_BUFFER = preferred_bitrate_low_buffer - headphones.PREFERRED_BITRATE_ALLOW_LOSSLESS = preferred_bitrate_allow_lossless - headphones.DETECT_BITRATE = detect_bitrate - headphones.LOSSLESS_BITRATE_FROM = lossless_bitrate_from - headphones.LOSSLESS_BITRATE_TO = lossless_bitrate_to - headphones.FREEZE_DB = freeze_db - headphones.MOVE_FILES = move_files - headphones.CORRECT_METADATA = correct_metadata - headphones.RENAME_FILES = rename_files - headphones.CLEANUP_FILES = cleanup_files - headphones.KEEP_NFO = keep_nfo - headphones.ADD_ALBUM_ART = add_album_art - headphones.ALBUM_ART_FORMAT = album_art_format - headphones.EMBED_ALBUM_ART = embed_album_art - headphones.EMBED_LYRICS = embed_lyrics - headphones.REPLACE_EXISTING_FOLDERS = replace_existing_folders - headphones.DESTINATION_DIR = destination_dir - headphones.LOSSLESS_DESTINATION_DIR = lossless_destination_dir - headphones.FOLDER_FORMAT = folder_format - headphones.FILE_FORMAT = file_format - headphones.FILE_UNDERSCORES = file_underscores - headphones.INCLUDE_EXTRAS = include_extras - headphones.AUTOWANT_UPCOMING = autowant_upcoming - headphones.AUTOWANT_ALL = autowant_all - headphones.AUTOWANT_MANUALLY_ADDED = autowant_manually_added - headphones.KEEP_TORRENT_FILES = keep_torrent_files - headphones.PREFER_TORRENTS = int(prefer_torrents) - headphones.OPEN_MAGNET_LINKS = open_magnet_links - headphones.INTERFACE = interface - headphones.LOG_DIR = log_dir - headphones.CACHE_DIR = cache_dir - headphones.MUSIC_ENCODER = music_encoder - headphones.ENCODER = encoder - headphones.XLDPROFILE = xldprofile - headphones.BITRATE = int(bitrate) - headphones.SAMPLINGFREQUENCY = int(samplingfrequency) - headphones.ENCODER_PATH = encoderfolder - headphones.ADVANCEDENCODER = advancedencoder - headphones.ENCODEROUTPUTFORMAT = encoderoutputformat - headphones.ENCODERVBRCBR = encodervbrcbr - headphones.ENCODERQUALITY = int(encoderquality) - headphones.ENCODERLOSSLESS = int(encoderlossless) - headphones.ENCODER_MULTICORE = encoder_multicore - headphones.ENCODER_MULTICORE_COUNT = max(0, int(encoder_multicore_count)) - headphones.DELETE_LOSSLESS_FILES = int(delete_lossless_files) - headphones.GROWL_ENABLED = growl_enabled - headphones.GROWL_ONSNATCH = growl_onsnatch - headphones.GROWL_HOST = growl_host - headphones.GROWL_PASSWORD = growl_password - headphones.PROWL_ENABLED = prowl_enabled - headphones.PROWL_ONSNATCH = prowl_onsnatch - headphones.PROWL_KEYS = prowl_keys - headphones.PROWL_PRIORITY = prowl_priority - headphones.XBMC_ENABLED = xbmc_enabled - headphones.XBMC_HOST = xbmc_host - headphones.XBMC_USERNAME = xbmc_username - headphones.XBMC_PASSWORD = xbmc_password - headphones.XBMC_UPDATE = xbmc_update - headphones.XBMC_NOTIFY = xbmc_notify - headphones.LMS_ENABLED = lms_enabled - headphones.LMS_HOST = lms_host - headphones.PLEX_ENABLED = plex_enabled - headphones.PLEX_SERVER_HOST = plex_server_host - headphones.PLEX_CLIENT_HOST = plex_client_host - headphones.PLEX_USERNAME = plex_username - headphones.PLEX_PASSWORD = plex_password - headphones.PLEX_UPDATE = plex_update - headphones.PLEX_NOTIFY = plex_notify - headphones.NMA_ENABLED = nma_enabled - headphones.NMA_APIKEY = nma_apikey - headphones.NMA_PRIORITY = nma_priority - headphones.NMA_ONSNATCH = nma_onsnatch - headphones.PUSHALOT_ENABLED = pushalot_enabled - headphones.PUSHALOT_APIKEY = pushalot_apikey - headphones.PUSHALOT_ONSNATCH = pushalot_onsnatch - headphones.SYNOINDEX_ENABLED = synoindex_enabled - headphones.PUSHOVER_ENABLED = pushover_enabled - headphones.PUSHOVER_ONSNATCH = pushover_onsnatch - headphones.PUSHOVER_KEYS = pushover_keys - headphones.PUSHOVER_PRIORITY = pushover_priority - headphones.PUSHOVER_APITOKEN = pushover_apitoken - headphones.PUSHBULLET_ENABLED = pushbullet_enabled - headphones.PUSHBULLET_ONSNATCH = pushbullet_onsnatch - headphones.PUSHBULLET_APIKEY = pushbullet_apikey - headphones.PUSHBULLET_DEVICEID = pushbullet_deviceid - headphones.SUBSONIC_ENABLED = subsonic_enabled - headphones.SUBSONIC_HOST = subsonic_host - headphones.SUBSONIC_USERNAME = subsonic_username - headphones.SUBSONIC_PASSWORD = subsonic_password - headphones.SONGKICK_ENABLED = songkick_enabled - headphones.SONGKICK_APIKEY = songkick_apikey - headphones.SONGKICK_LOCATION = songkick_location - headphones.SONGKICK_FILTER_ENABLED = songkick_filter_enabled - headphones.TWITTER_ENABLED = twitter_enabled - headphones.TWITTER_ONSNATCH = twitter_onsnatch - - headphones.OSX_NOTIFY_ENABLED = osx_notify_enabled - headphones.OSX_NOTIFY_ONSNATCH = osx_notify_onsnatch - headphones.OSX_NOTIFY_APP = osx_notify_app - - headphones.BOXCAR_ENABLED = boxcar_enabled - headphones.BOXCAR_ONSNATCH = boxcar_onsnatch - headphones.BOXCAR_TOKEN = boxcar_token - - headphones.MPC_ENABLED = mpc_enabled - - headphones.MIRROR = mirror - headphones.CUSTOMHOST = customhost - headphones.CUSTOMPORT = customport - headphones.CUSTOMSLEEP = customsleep - headphones.HPUSER = hpuser - headphones.HPPASS = hppass - headphones.CACHE_SIZEMB = int(cache_sizemb) - headphones.FILE_PERMISSIONS = file_permissions - headphones.FOLDER_PERMISSIONS = folder_permissions - + def configUpdate(self, **kwargs): # Handle the variable config options. Note - keys with False values aren't getting passed - headphones.EXTRA_NEWZNABS = [] + checked_configs = [ + "launch_browser", "enable_https", "api_enabled", "use_blackhole", "headphones_indexer", "use_newznab", "newznab_enabled", + "use_nzbsorg", "use_omgwtfnzbs", "use_kat", "use_piratebay", "use_mininova", "use_waffles", "use_rutracker", "use_whatcd", + "preferred_bitrate_allow_lossless", "detect_bitrate", "freeze_db", "cue_split", "move_files", "rename_files", "correct_metadata", + "cleanup_files", "keep_nfo", "add_album_art", "embed_album_art", "embed_lyrics", "replace_existing_folders", "file_underscores", + "include_extras", "autowant_upcoming", "autowant_all", "autowant_manually_added", "keep_torrent_files", "music_encoder", + "encoderlossless", "encoder_multicore", "delete_lossless_files", "growl_enabled", "growl_onsnatch", "prowl_enabled", + "prowl_onsnatch", "xbmc_enabled", "xbmc_update", "xbmc_notify", "lms_enabled", "plex_enabled", "plex_update", "plex_notify", + "nma_enabled", "nma_onsnatch", "pushalot_enabled", "pushalot_onsnatch", "synoindex_enabled", "pushover_enabled", + "pushover_onsnatch", "pushbullet_enabled", "pushbullet_onsnatch", "subsonic_enabled", "twitter_enabled", "twitter_onsnatch", + "osx_notify_enabled", "osx_notify_onsnatch", "boxcar_enabled", "boxcar_onsnatch", "songkick_enabled", "songkick_filter_enabled", + "mpc_enabled" + ] + for checked_config in checked_configs: + if checked_config not in kwargs: + # checked items should be zero or one. if they were not sent then the item was not checked + kwargs[checked_config] = 0 - for kwarg in kwargs: - if kwarg.startswith('newznab_host'): - newznab_number = kwarg[12:] - newznab_host = kwargs['newznab_host' + newznab_number] - newznab_api = kwargs['newznab_api' + newznab_number] - try: - newznab_enabled = int(kwargs['newznab_enabled' + newznab_number]) - except KeyError: - newznab_enabled = 0 + for plain_config, use_config in [(x[4:], x) for x in kwargs if x.startswith('use_')]: + # the use prefix is fairly nice in the html, but does not match the actual config + kwargs[plain_config] = kwargs[use_config] + del kwargs[use_config] - headphones.EXTRA_NEWZNABS.append((newznab_host, newznab_api, newznab_enabled)) + extra_newznabs = [] + for kwarg in [x for x in kwargs if x.startswith('newznab_host')]: + newznab_host_key = kwarg + newznab_number = kwarg[12:] + if len(newznab_number): + newznab_api_key = 'newznab_api' + newznab_number + newznab_enabled_key = 'newznab_enabled' + newznab_number + newznab_host = kwargs.get(newznab_host_key, '') + newznab_api = kwargs.get(newznab_api_key, '') + newznab_enabled = int(kwargs.get(newznab_enabled_key, 0)) + for key in [newznab_host_key, newznab_api_key, newznab_enabled_key]: + if key in kwargs: + del kwargs[key] + extra_newznabs.append((newznab_host, newznab_api, newznab_enabled)) # Convert the extras to list then string. Coming in as 0 or 1 (append new extras to the end) temp_extras_list = [] - extras_list = [single, ep, compilation, soundtrack, live, remix, spokenword, audiobook, other, djmix, mixtape_street, broadcast, interview, demo] + + extra_munges = { + "dj-mix": "dj_mix", + "mixtape/street": "mixtape_street" + } + + expected_extras = [extra_munges.get(x, x) for x in headphones.POSSIBLE_EXTRAS] + extras_list = [kwargs.get(x, 0) for x in expected_extras] i = 1 for extra in extras_list: if extra: temp_extras_list.append(i) - i+=1 + i += 1 - headphones.EXTRAS = ','.join(str(n) for n in temp_extras_list) + for extra in expected_extras: + temp = '%s_temp' % extra + if temp in kwargs: + del kwargs[temp] + if extra in kwargs: + del kwargs[extra] + + headphones.CONFIG.EXTRAS = ','.join(str(n) for n in temp_extras_list) + headphones.CONFIG.clear_extra_newznabs() + headphones.CONFIG.process_kwargs(kwargs) + for extra_newznab in extra_newznabs: + headphones.CONFIG.add_extra_newznab(extra_newznab) # Sanity checking - if headphones.SEARCH_INTERVAL < 360: + if headphones.CONFIG.SEARCH_INTERVAL < 360: logger.info("Search interval too low. Resetting to 6 hour minimum") - headphones.SEARCH_INTERVAL = 360 + headphones.CONFIG.SEARCH_INTERVAL = 360 # Write the config - headphones.config_write() + headphones.CONFIG.write() #reconfigure musicbrainz database connection with the new values mb.startmb() @@ -1456,7 +1277,6 @@ class WebInterface(object): myDB = db.DBConnection() cloudlist = myDB.select('SELECT * from lastfmcloud') return serve_template(templatename="extras.html", title="Extras", cloudlist=cloudlist) - return page extras.exposed = True def addReleaseById(self, rid, rgid=None): @@ -1521,7 +1341,7 @@ class WebInterface(object): if AlbumID and not image_dict: image_url = "http://coverartarchive.org/release/%s/front-500.jpg" % AlbumID thumb_url = "http://coverartarchive.org/release/%s/front-250.jpg" % AlbumID - image_dict = {'artwork' : image_url, 'thumbnail' : thumb_url} + image_dict = {'artwork': image_url, 'thumbnail': thumb_url} elif AlbumID and (not image_dict['artwork'] or not image_dict['thumbnail']): if not image_dict['artwork']: image_dict['artwork'] = "http://coverartarchive.org/release/%s/front-500.jpg" % AlbumID @@ -1542,7 +1362,7 @@ class WebInterface(object): cherrypy.response.headers['Cache-Control'] = "max-age=0,no-cache,no-store" tweet = notifiers.TwitterNotifier() result = tweet._get_credentials(key) - logger.info(u"result: "+str(result)) + logger.info(u"result: " + str(result)) if result: return "Key verification successful" else: @@ -1561,7 +1381,7 @@ class WebInterface(object): def osxnotifyregister(self, app): cherrypy.response.headers['Cache-Control'] = "max-age=0,no-cache,no-store" - from lib.osxnotify import registerapp as osxnotify + from osxnotify import registerapp as osxnotify result, msg = osxnotify.registerapp(app) if result: osx_notify = notifiers.OSX_NOTIFY() @@ -1572,12 +1392,13 @@ class WebInterface(object): return msg osxnotifyregister.exposed = True + class Artwork(object): def index(self): return "Artwork" index.exposed = True - def default(self,ArtistOrAlbum="",ID=None): + def default(self, ArtistOrAlbum="", ID=None): from headphones import cache ArtistID = None AlbumID = None @@ -1586,23 +1407,23 @@ class Artwork(object): elif ArtistOrAlbum == "album": AlbumID = ID - relpath = cache.getArtwork(ArtistID,AlbumID) + relpath = cache.getArtwork(ArtistID, AlbumID) if not relpath: relpath = "data/interfaces/default/images/no-cover-art.png" basedir = os.path.dirname(sys.argv[0]) - path = os.path.join(basedir,relpath) + path = os.path.join(basedir, relpath) cherrypy.response.headers['Content-type'] = 'image/png' cherrypy.response.headers['Cache-Control'] = 'no-cache' else: - relpath = relpath.replace('cache/','',1) - path = os.path.join(headphones.CACHE_DIR,relpath) + relpath = relpath.replace('cache/', '', 1) + path = os.path.join(headphones.CONFIG.CACHE_DIR, relpath) fileext = os.path.splitext(relpath)[1][1::] cherrypy.response.headers['Content-type'] = 'image/' + fileext cherrypy.response.headers['Cache-Control'] = 'max-age=31556926' path = os.path.normpath(path) - f = open(path,'rb') + f = open(path, 'rb') return f.read() default.exposed = True @@ -1610,7 +1431,8 @@ class Artwork(object): def index(self): return "Here be thumbs" index.exposed = True - def default(self,ArtistOrAlbum="",ID=None): + + def default(self, ArtistOrAlbum="", ID=None): from headphones import cache ArtistID = None AlbumID = None @@ -1619,23 +1441,23 @@ class Artwork(object): elif ArtistOrAlbum == "album": AlbumID = ID - relpath = cache.getThumb(ArtistID,AlbumID) + relpath = cache.getThumb(ArtistID, AlbumID) if not relpath: relpath = "data/interfaces/default/images/no-cover-artist.png" basedir = os.path.dirname(sys.argv[0]) - path = os.path.join(basedir,relpath) + path = os.path.join(basedir, relpath) cherrypy.response.headers['Content-type'] = 'image/png' cherrypy.response.headers['Cache-Control'] = 'no-cache' else: - relpath = relpath.replace('cache/','',1) - path = os.path.join(headphones.CACHE_DIR,relpath) + relpath = relpath.replace('cache/', '', 1) + path = os.path.join(headphones.CONFIG.CACHE_DIR, relpath) fileext = os.path.splitext(relpath)[1][1::] cherrypy.response.headers['Content-type'] = 'image/' + fileext cherrypy.response.headers['Cache-Control'] = 'max-age=31556926' path = os.path.normpath(path) - f = open(path,'rb') + f = open(path, 'rb') return f.read() default.exposed = True diff --git a/headphones/webstart.py b/headphones/webstart.py index f6ac65ae..ffeaf279 100644 --- a/headphones/webstart.py +++ b/headphones/webstart.py @@ -22,6 +22,7 @@ from headphones import logger from headphones.webserve import WebInterface from headphones.helpers import create_https_certificates + def initialize(options=None): if options is None: options = {} @@ -36,12 +37,12 @@ def initialize(options=None): if not (https_cert and os.path.exists(https_cert)) or not (https_key and os.path.exists(https_key)): if not create_https_certificates(https_cert, https_key): logger.warn(u"Unable to create cert/key files, disabling HTTPS") - headphones.ENABLE_HTTPS = False + headphones.CONFIG.ENABLE_HTTPS = False enable_https = False if not (os.path.exists(https_cert) and os.path.exists(https_key)): logger.warn(u"Disabled HTTPS because of missing CERT and KEY files") - headphones.ENABLE_HTTPS = False + headphones.CONFIG.ENABLE_HTTPS = False enable_https = False options_dict = { @@ -52,7 +53,7 @@ def initialize(options=None): 'tools.encode.encoding': 'utf-8', 'tools.decode.on': True, 'log.screen': False, - 'engine.autoreload_on': False, + 'engine.autoreload.on': False, } if enable_https: @@ -71,30 +72,30 @@ def initialize(options=None): 'tools.staticdir.root': os.path.join(headphones.PROG_DIR, 'data'), 'tools.proxy.on': options['http_proxy'] # pay attention to X-Forwarded-Proto header }, - '/interfaces':{ + '/interfaces': { 'tools.staticdir.on': True, 'tools.staticdir.dir': "interfaces" }, - '/images':{ + '/images': { 'tools.staticdir.on': True, 'tools.staticdir.dir': "images" }, - '/css':{ + '/css': { 'tools.staticdir.on': True, 'tools.staticdir.dir': "css" }, - '/js':{ + '/js': { 'tools.staticdir.on': True, 'tools.staticdir.dir': "js" }, - '/favicon.ico':{ + '/favicon.ico': { 'tools.staticfile.on': True, 'tools.staticfile.filename': os.path.join(os.path.abspath( os.curdir), "images" + os.sep + "favicon.ico") }, - '/cache':{ + '/cache': { 'tools.staticdir.on': True, - 'tools.staticdir.dir': headphones.CACHE_DIR + 'tools.staticdir.dir': headphones.CONFIG.CACHE_DIR } } @@ -104,23 +105,21 @@ def initialize(options=None): conf['/'].update({ 'tools.auth_basic.on': True, 'tools.auth_basic.realm': 'Headphones web server', - 'tools.auth_basic.checkpassword': cherrypy.lib.auth_basic \ - .checkpassword_dict({ - options['http_username']: options['http_password'] - }) + 'tools.auth_basic.checkpassword': cherrypy.lib.auth_basic.checkpassword_dict({ + options['http_username']: options['http_password'] + }) }) - conf['/api'] = { 'tools.auth_basic.on': False } - + conf['/api'] = {'tools.auth_basic.on': False} # Prevent time-outs cherrypy.engine.timeout_monitor.unsubscribe() - cherrypy.tree.mount(WebInterface(), options['http_root'], config=conf) + cherrypy.tree.mount(WebInterface(), str(options['http_root']), config=conf) try: - cherrypy.process.servers.check_port(options['http_host'], options['http_port']) + cherrypy.process.servers.check_port(str(options['http_host']), options['http_port']) cherrypy.server.start() except IOError: sys.stderr.write('Failed to start on port: %i. Is something else running?\n' % (options['http_port'])) sys.exit(1) - cherrypy.server.wait() \ No newline at end of file + cherrypy.server.wait() diff --git a/lib/apscheduler/__init__.py b/lib/apscheduler/__init__.py index 6b502147..283002b7 100644 --- a/lib/apscheduler/__init__.py +++ b/lib/apscheduler/__init__.py @@ -1,3 +1,5 @@ -version_info = (2, 0, 0, 'rc', 2) -version = '.'.join(str(n) for n in version_info[:3]) -release = version + ''.join(str(n) for n in version_info[3:]) +version_info = (3, 0, 1) +version = '3.0.1' +release = '3.0.1' + +__version__ = release # PEP 396 diff --git a/lib/apscheduler/events.py b/lib/apscheduler/events.py index 80bde8e6..9418263d 100644 --- a/lib/apscheduler/events.py +++ b/lib/apscheduler/events.py @@ -1,63 +1,72 @@ -__all__ = ('EVENT_SCHEDULER_START', 'EVENT_SCHEDULER_SHUTDOWN', - 'EVENT_JOBSTORE_ADDED', 'EVENT_JOBSTORE_REMOVED', - 'EVENT_JOBSTORE_JOB_ADDED', 'EVENT_JOBSTORE_JOB_REMOVED', - 'EVENT_JOB_EXECUTED', 'EVENT_JOB_ERROR', 'EVENT_JOB_MISSED', - 'EVENT_ALL', 'SchedulerEvent', 'JobStoreEvent', 'JobEvent') +__all__ = ('EVENT_SCHEDULER_START', 'EVENT_SCHEDULER_SHUTDOWN', 'EVENT_EXECUTOR_ADDED', 'EVENT_EXECUTOR_REMOVED', + 'EVENT_JOBSTORE_ADDED', 'EVENT_JOBSTORE_REMOVED', 'EVENT_ALL_JOBS_REMOVED', 'EVENT_JOB_ADDED', + 'EVENT_JOB_REMOVED', 'EVENT_JOB_MODIFIED', 'EVENT_JOB_EXECUTED', 'EVENT_JOB_ERROR', 'EVENT_JOB_MISSED', + 'SchedulerEvent', 'JobEvent', 'JobExecutionEvent') -EVENT_SCHEDULER_START = 1 # The scheduler was started -EVENT_SCHEDULER_SHUTDOWN = 2 # The scheduler was shut down -EVENT_JOBSTORE_ADDED = 4 # A job store was added to the scheduler -EVENT_JOBSTORE_REMOVED = 8 # A job store was removed from the scheduler -EVENT_JOBSTORE_JOB_ADDED = 16 # A job was added to a job store -EVENT_JOBSTORE_JOB_REMOVED = 32 # A job was removed from a job store -EVENT_JOB_EXECUTED = 64 # A job was executed successfully -EVENT_JOB_ERROR = 128 # A job raised an exception during execution -EVENT_JOB_MISSED = 256 # A job's execution was missed -EVENT_ALL = (EVENT_SCHEDULER_START | EVENT_SCHEDULER_SHUTDOWN | - EVENT_JOBSTORE_ADDED | EVENT_JOBSTORE_REMOVED | - EVENT_JOBSTORE_JOB_ADDED | EVENT_JOBSTORE_JOB_REMOVED | - EVENT_JOB_EXECUTED | EVENT_JOB_ERROR | EVENT_JOB_MISSED) +EVENT_SCHEDULER_START = 1 +EVENT_SCHEDULER_SHUTDOWN = 2 +EVENT_EXECUTOR_ADDED = 4 +EVENT_EXECUTOR_REMOVED = 8 +EVENT_JOBSTORE_ADDED = 16 +EVENT_JOBSTORE_REMOVED = 32 +EVENT_ALL_JOBS_REMOVED = 64 +EVENT_JOB_ADDED = 128 +EVENT_JOB_REMOVED = 256 +EVENT_JOB_MODIFIED = 512 +EVENT_JOB_EXECUTED = 1024 +EVENT_JOB_ERROR = 2048 +EVENT_JOB_MISSED = 4096 +EVENT_ALL = (EVENT_SCHEDULER_START | EVENT_SCHEDULER_SHUTDOWN | EVENT_JOBSTORE_ADDED | EVENT_JOBSTORE_REMOVED | + EVENT_JOB_ADDED | EVENT_JOB_REMOVED | EVENT_JOB_MODIFIED | EVENT_JOB_EXECUTED | + EVENT_JOB_ERROR | EVENT_JOB_MISSED) class SchedulerEvent(object): """ An event that concerns the scheduler itself. - :var code: the type code of this event + :ivar code: the type code of this event + :ivar alias: alias of the job store or executor that was added or removed (if applicable) """ - def __init__(self, code): + + def __init__(self, code, alias=None): + super(SchedulerEvent, self).__init__() self.code = code - - -class JobStoreEvent(SchedulerEvent): - """ - An event that concerns job stores. - - :var alias: the alias of the job store involved - :var job: the new job if a job was added - """ - def __init__(self, code, alias, job=None): - SchedulerEvent.__init__(self, code) self.alias = alias - if job: - self.job = job + + def __repr__(self): + return '<%s (code=%d)>' % (self.__class__.__name__, self.code) class JobEvent(SchedulerEvent): + """ + An event that concerns a job. + + :ivar code: the type code of this event + :ivar job_id: identifier of the job in question + :ivar jobstore: alias of the job store containing the job in question + """ + + def __init__(self, code, job_id, jobstore): + super(JobEvent, self).__init__(code) + self.code = code + self.job_id = job_id + self.jobstore = jobstore + + +class JobExecutionEvent(JobEvent): """ An event that concerns the execution of individual jobs. - :var job: the job instance in question - :var scheduled_run_time: the time when the job was scheduled to be run - :var retval: the return value of the successfully executed job - :var exception: the exception raised by the job - :var traceback: the traceback object associated with the exception + :ivar scheduled_run_time: the time when the job was scheduled to be run + :ivar retval: the return value of the successfully executed job + :ivar exception: the exception raised by the job + :ivar traceback: a formatted traceback for the exception """ - def __init__(self, code, job, scheduled_run_time, retval=None, - exception=None, traceback=None): - SchedulerEvent.__init__(self, code) - self.job = job + + def __init__(self, code, job_id, jobstore, scheduled_run_time, retval=None, exception=None, traceback=None): + super(JobExecutionEvent, self).__init__(code, job_id, jobstore) self.scheduled_run_time = scheduled_run_time self.retval = retval self.exception = exception diff --git a/lib/apscheduler/executors/__init__.py b/lib/apscheduler/executors/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lib/apscheduler/executors/asyncio.py b/lib/apscheduler/executors/asyncio.py new file mode 100644 index 00000000..fade99f8 --- /dev/null +++ b/lib/apscheduler/executors/asyncio.py @@ -0,0 +1,28 @@ +from __future__ import absolute_import +import sys + +from apscheduler.executors.base import BaseExecutor, run_job + + +class AsyncIOExecutor(BaseExecutor): + """ + Runs jobs in the default executor of the event loop. + + Plugin alias: ``asyncio`` + """ + + def start(self, scheduler, alias): + super(AsyncIOExecutor, self).start(scheduler, alias) + self._eventloop = scheduler._eventloop + + def _do_submit_job(self, job, run_times): + def callback(f): + try: + events = f.result() + except: + self._run_job_error(job.id, *sys.exc_info()[1:]) + else: + self._run_job_success(job.id, events) + + f = self._eventloop.run_in_executor(None, run_job, job, job._jobstore_alias, run_times, self._logger.name) + f.add_done_callback(callback) diff --git a/lib/apscheduler/executors/base.py b/lib/apscheduler/executors/base.py new file mode 100644 index 00000000..5a0a19eb --- /dev/null +++ b/lib/apscheduler/executors/base.py @@ -0,0 +1,119 @@ +from abc import ABCMeta, abstractmethod +from collections import defaultdict +from datetime import datetime, timedelta +from traceback import format_tb +import logging +import sys + +from pytz import utc +import six + +from apscheduler.events import JobExecutionEvent, EVENT_JOB_MISSED, EVENT_JOB_ERROR, EVENT_JOB_EXECUTED + + +class MaxInstancesReachedError(Exception): + def __init__(self, job): + super(MaxInstancesReachedError, self).__init__( + 'Job "%s" has already reached its maximum number of instances (%d)' % (job.id, job.max_instances)) + + +class BaseExecutor(six.with_metaclass(ABCMeta, object)): + """Abstract base class that defines the interface that every executor must implement.""" + + _scheduler = None + _lock = None + _logger = logging.getLogger('apscheduler.executors') + + def __init__(self): + super(BaseExecutor, self).__init__() + self._instances = defaultdict(lambda: 0) + + def start(self, scheduler, alias): + """ + Called by the scheduler when the scheduler is being started or when the executor is being added to an already + running scheduler. + + :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting this executor + :param str|unicode alias: alias of this executor as it was assigned to the scheduler + """ + + self._scheduler = scheduler + self._lock = scheduler._create_lock() + self._logger = logging.getLogger('apscheduler.executors.%s' % alias) + + def shutdown(self, wait=True): + """ + Shuts down this executor. + + :param bool wait: ``True`` to wait until all submitted jobs have been executed + """ + + def submit_job(self, job, run_times): + """ + Submits job for execution. + + :param Job job: job to execute + :param list[datetime] run_times: list of datetimes specifying when the job should have been run + :raises MaxInstancesReachedError: if the maximum number of allowed instances for this job has been reached + """ + + assert self._lock is not None, 'This executor has not been started yet' + with self._lock: + if self._instances[job.id] >= job.max_instances: + raise MaxInstancesReachedError(job) + + self._do_submit_job(job, run_times) + self._instances[job.id] += 1 + + @abstractmethod + def _do_submit_job(self, job, run_times): + """Performs the actual task of scheduling `run_job` to be called.""" + + def _run_job_success(self, job_id, events): + """Called by the executor with the list of generated events when `run_job` has been successfully called.""" + + with self._lock: + self._instances[job_id] -= 1 + + for event in events: + self._scheduler._dispatch_event(event) + + def _run_job_error(self, job_id, exc, traceback=None): + """Called by the executor with the exception if there is an error calling `run_job`.""" + + with self._lock: + self._instances[job_id] -= 1 + + exc_info = (exc.__class__, exc, traceback) + self._logger.error('Error running job %s', job_id, exc_info=exc_info) + + +def run_job(job, jobstore_alias, run_times, logger_name): + """Called by executors to run the job. Returns a list of scheduler events to be dispatched by the scheduler.""" + + events = [] + logger = logging.getLogger(logger_name) + for run_time in run_times: + # See if the job missed its run time window, and handle possible misfires accordingly + if job.misfire_grace_time is not None: + difference = datetime.now(utc) - run_time + grace_time = timedelta(seconds=job.misfire_grace_time) + if difference > grace_time: + events.append(JobExecutionEvent(EVENT_JOB_MISSED, job.id, jobstore_alias, run_time)) + logger.warning('Run time of job "%s" was missed by %s', job, difference) + continue + + logger.info('Running job "%s" (scheduled at %s)', job, run_time) + try: + retval = job.func(*job.args, **job.kwargs) + except: + exc, tb = sys.exc_info()[1:] + formatted_tb = ''.join(format_tb(tb)) + events.append(JobExecutionEvent(EVENT_JOB_ERROR, job.id, jobstore_alias, run_time, exception=exc, + traceback=formatted_tb)) + logger.exception('Job "%s" raised an exception', job) + else: + events.append(JobExecutionEvent(EVENT_JOB_EXECUTED, job.id, jobstore_alias, run_time, retval=retval)) + logger.info('Job "%s" executed successfully', job) + + return events diff --git a/lib/apscheduler/executors/debug.py b/lib/apscheduler/executors/debug.py new file mode 100644 index 00000000..1f6f6b1a --- /dev/null +++ b/lib/apscheduler/executors/debug.py @@ -0,0 +1,19 @@ +import sys + +from apscheduler.executors.base import BaseExecutor, run_job + + +class DebugExecutor(BaseExecutor): + """ + A special executor that executes the target callable directly instead of deferring it to a thread or process. + + Plugin alias: ``debug`` + """ + + def _do_submit_job(self, job, run_times): + try: + events = run_job(job, job._jobstore_alias, run_times, self._logger.name) + except: + self._run_job_error(job.id, *sys.exc_info()[1:]) + else: + self._run_job_success(job.id, events) diff --git a/lib/apscheduler/executors/gevent.py b/lib/apscheduler/executors/gevent.py new file mode 100644 index 00000000..9f4db2fc --- /dev/null +++ b/lib/apscheduler/executors/gevent.py @@ -0,0 +1,29 @@ +from __future__ import absolute_import +import sys + +from apscheduler.executors.base import BaseExecutor, run_job + + +try: + import gevent +except ImportError: # pragma: nocover + raise ImportError('GeventExecutor requires gevent installed') + + +class GeventExecutor(BaseExecutor): + """ + Runs jobs as greenlets. + + Plugin alias: ``gevent`` + """ + + def _do_submit_job(self, job, run_times): + def callback(greenlet): + try: + events = greenlet.get() + except: + self._run_job_error(job.id, *sys.exc_info()[1:]) + else: + self._run_job_success(job.id, events) + + gevent.spawn(run_job, job, job._jobstore_alias, run_times, self._logger.name).link(callback) diff --git a/lib/apscheduler/executors/pool.py b/lib/apscheduler/executors/pool.py new file mode 100644 index 00000000..2f4ef455 --- /dev/null +++ b/lib/apscheduler/executors/pool.py @@ -0,0 +1,54 @@ +from abc import abstractmethod +import concurrent.futures + +from apscheduler.executors.base import BaseExecutor, run_job + + +class BasePoolExecutor(BaseExecutor): + @abstractmethod + def __init__(self, pool): + super(BasePoolExecutor, self).__init__() + self._pool = pool + + def _do_submit_job(self, job, run_times): + def callback(f): + exc, tb = (f.exception_info() if hasattr(f, 'exception_info') else + (f.exception(), getattr(f.exception(), '__traceback__', None))) + if exc: + self._run_job_error(job.id, exc, tb) + else: + self._run_job_success(job.id, f.result()) + + f = self._pool.submit(run_job, job, job._jobstore_alias, run_times, self._logger.name) + f.add_done_callback(callback) + + def shutdown(self, wait=True): + self._pool.shutdown(wait) + + +class ThreadPoolExecutor(BasePoolExecutor): + """ + An executor that runs jobs in a concurrent.futures thread pool. + + Plugin alias: ``threadpool`` + + :param max_workers: the maximum number of spawned threads. + """ + + def __init__(self, max_workers=10): + pool = concurrent.futures.ThreadPoolExecutor(int(max_workers)) + super(ThreadPoolExecutor, self).__init__(pool) + + +class ProcessPoolExecutor(BasePoolExecutor): + """ + An executor that runs jobs in a concurrent.futures process pool. + + Plugin alias: ``processpool`` + + :param max_workers: the maximum number of spawned processes. + """ + + def __init__(self, max_workers=10): + pool = concurrent.futures.ProcessPoolExecutor(int(max_workers)) + super(ProcessPoolExecutor, self).__init__(pool) diff --git a/lib/apscheduler/executors/twisted.py b/lib/apscheduler/executors/twisted.py new file mode 100644 index 00000000..29217221 --- /dev/null +++ b/lib/apscheduler/executors/twisted.py @@ -0,0 +1,25 @@ +from __future__ import absolute_import + +from apscheduler.executors.base import BaseExecutor, run_job + + +class TwistedExecutor(BaseExecutor): + """ + Runs jobs in the reactor's thread pool. + + Plugin alias: ``twisted`` + """ + + def start(self, scheduler, alias): + super(TwistedExecutor, self).start(scheduler, alias) + self._reactor = scheduler._reactor + + def _do_submit_job(self, job, run_times): + def callback(success, result): + if success: + self._run_job_success(job.id, result) + else: + self._run_job_error(job.id, result.value, result.tb) + + self._reactor.getThreadPool().callInThreadWithCallback(callback, run_job, job, job._jobstore_alias, run_times, + self._logger.name) diff --git a/lib/apscheduler/job.py b/lib/apscheduler/job.py index 868e7234..f5639dae 100644 --- a/lib/apscheduler/job.py +++ b/lib/apscheduler/job.py @@ -1,134 +1,252 @@ -""" -Jobs represent scheduled tasks. -""" +from collections import Iterable, Mapping +from uuid import uuid4 -from threading import Lock -from datetime import timedelta +import six -from apscheduler.util import to_unicode, ref_to_obj, get_callable_name,\ - obj_to_ref - - -class MaxInstancesReachedError(Exception): - pass +from apscheduler.triggers.base import BaseTrigger +from apscheduler.util import ref_to_obj, obj_to_ref, datetime_repr, repr_escape, get_callable_name, check_callable_args, \ + convert_to_datetime class Job(object): """ - Encapsulates the actual Job along with its metadata. Job instances - are created by the scheduler when adding jobs, and it should not be - directly instantiated. + Contains the options given when scheduling callables and its current schedule and other state. + This class should never be instantiated by the user. - :param trigger: trigger that determines the execution times - :param func: callable to call when the trigger is triggered - :param args: list of positional arguments to call func with - :param kwargs: dict of keyword arguments to call func with - :param name: name of the job (optional) - :param misfire_grace_time: seconds after the designated run time that - the job is still allowed to be run - :param coalesce: run once instead of many times if the scheduler determines - that the job should be run more than once in succession - :param max_runs: maximum number of times this job is allowed to be - triggered - :param max_instances: maximum number of concurrently running - instances allowed for this job + :var str id: the unique identifier of this job + :var str name: the description of this job + :var func: the callable to execute + :var tuple|list args: positional arguments to the callable + :var dict kwargs: keyword arguments to the callable + :var bool coalesce: whether to only run the job once when several run times are due + :var trigger: the trigger object that controls the schedule of this job + :var str executor: the name of the executor that will run this job + :var int misfire_grace_time: the time (in seconds) how much this job's execution is allowed to be late + :var int max_instances: the maximum number of concurrently executing instances allowed for this job + :var datetime.datetime next_run_time: the next scheduled run time of this job """ - id = None - next_run_time = None - def __init__(self, trigger, func, args, kwargs, misfire_grace_time, - coalesce, name=None, max_runs=None, max_instances=1): - if not trigger: - raise ValueError('The trigger must not be None') - if not hasattr(func, '__call__'): - raise TypeError('func must be callable') - if not hasattr(args, '__getitem__'): - raise TypeError('args must be a list-like object') - if not hasattr(kwargs, '__getitem__'): - raise TypeError('kwargs must be a dict-like object') - if misfire_grace_time <= 0: - raise ValueError('misfire_grace_time must be a positive value') - if max_runs is not None and max_runs <= 0: - raise ValueError('max_runs must be a positive value') - if max_instances <= 0: - raise ValueError('max_instances must be a positive value') + __slots__ = ('_scheduler', '_jobstore_alias', 'id', 'trigger', 'executor', 'func', 'func_ref', 'args', 'kwargs', + 'name', 'misfire_grace_time', 'coalesce', 'max_instances', 'next_run_time') - self._lock = Lock() + def __init__(self, scheduler, id=None, **kwargs): + super(Job, self).__init__() + self._scheduler = scheduler + self._jobstore_alias = None + self._modify(id=id or uuid4().hex, **kwargs) - self.trigger = trigger - self.func = func - self.args = args - self.kwargs = kwargs - self.name = to_unicode(name or get_callable_name(func)) - self.misfire_grace_time = misfire_grace_time - self.coalesce = coalesce - self.max_runs = max_runs - self.max_instances = max_instances - self.runs = 0 - self.instances = 0 - - def compute_next_run_time(self, now): - if self.runs == self.max_runs: - self.next_run_time = None - else: - self.next_run_time = self.trigger.get_next_fire_time(now) - - return self.next_run_time - - def get_run_times(self, now): + def modify(self, **changes): """ - Computes the scheduled run times between ``next_run_time`` and ``now``. + Makes the given changes to this job and saves it in the associated job store. + Accepted keyword arguments are the same as the variables on this class. + + .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.modify_job` """ + + self._scheduler.modify_job(self.id, self._jobstore_alias, **changes) + + def reschedule(self, trigger, **trigger_args): + """ + Shortcut for switching the trigger on this job. + + .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.reschedule_job` + """ + + self._scheduler.reschedule_job(self.id, self._jobstore_alias, trigger, **trigger_args) + + def pause(self): + """ + Temporarily suspend the execution of this job. + + .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.pause_job` + """ + + self._scheduler.pause_job(self.id, self._jobstore_alias) + + def resume(self): + """ + Resume the schedule of this job if previously paused. + + .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.resume_job` + """ + + self._scheduler.resume_job(self.id, self._jobstore_alias) + + def remove(self): + """ + Unschedules this job and removes it from its associated job store. + + .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.remove_job` + """ + + self._scheduler.remove_job(self.id, self._jobstore_alias) + + @property + def pending(self): + """Returns ``True`` if the referenced job is still waiting to be added to its designated job store.""" + + return self._jobstore_alias is None + + # + # Private API + # + + def _get_run_times(self, now): + """ + Computes the scheduled run times between ``next_run_time`` and ``now`` (inclusive). + + :type now: datetime.datetime + :rtype: list[datetime.datetime] + """ + run_times = [] - run_time = self.next_run_time - increment = timedelta(microseconds=1) - while ((not self.max_runs or self.runs < self.max_runs) and - run_time and run_time <= now): - run_times.append(run_time) - run_time = self.trigger.get_next_fire_time(run_time + increment) + next_run_time = self.next_run_time + while next_run_time and next_run_time <= now: + run_times.append(next_run_time) + next_run_time = self.trigger.get_next_fire_time(next_run_time, now) return run_times - def add_instance(self): - self._lock.acquire() - try: - if self.instances == self.max_instances: - raise MaxInstancesReachedError - self.instances += 1 - finally: - self._lock.release() + def _modify(self, **changes): + """Validates the changes to the Job and makes the modifications if and only if all of them validate.""" - def remove_instance(self): - self._lock.acquire() - try: - assert self.instances > 0, 'Already at 0 instances' - self.instances -= 1 - finally: - self._lock.release() + approved = {} + + if 'id' in changes: + value = changes.pop('id') + if not isinstance(value, six.string_types): + raise TypeError("id must be a nonempty string") + if hasattr(self, 'id'): + raise ValueError('The job ID may not be changed') + approved['id'] = value + + if 'func' in changes or 'args' in changes or 'kwargs' in changes: + func = changes.pop('func') if 'func' in changes else self.func + args = changes.pop('args') if 'args' in changes else self.args + kwargs = changes.pop('kwargs') if 'kwargs' in changes else self.kwargs + + if isinstance(func, str): + func_ref = func + func = ref_to_obj(func) + elif callable(func): + try: + func_ref = obj_to_ref(func) + except ValueError: + # If this happens, this Job won't be serializable + func_ref = None + else: + raise TypeError('func must be a callable or a textual reference to one') + + if not hasattr(self, 'name') and changes.get('name', None) is None: + changes['name'] = get_callable_name(func) + + if isinstance(args, six.string_types) or not isinstance(args, Iterable): + raise TypeError('args must be a non-string iterable') + if isinstance(kwargs, six.string_types) or not isinstance(kwargs, Mapping): + raise TypeError('kwargs must be a dict-like object') + + check_callable_args(func, args, kwargs) + + approved['func'] = func + approved['func_ref'] = func_ref + approved['args'] = args + approved['kwargs'] = kwargs + + if 'name' in changes: + value = changes.pop('name') + if not value or not isinstance(value, six.string_types): + raise TypeError("name must be a nonempty string") + approved['name'] = value + + if 'misfire_grace_time' in changes: + value = changes.pop('misfire_grace_time') + if value is not None and (not isinstance(value, six.integer_types) or value <= 0): + raise TypeError('misfire_grace_time must be either None or a positive integer') + approved['misfire_grace_time'] = value + + if 'coalesce' in changes: + value = bool(changes.pop('coalesce')) + approved['coalesce'] = value + + if 'max_instances' in changes: + value = changes.pop('max_instances') + if not isinstance(value, six.integer_types) or value <= 0: + raise TypeError('max_instances must be a positive integer') + approved['max_instances'] = value + + if 'trigger' in changes: + trigger = changes.pop('trigger') + if not isinstance(trigger, BaseTrigger): + raise TypeError('Expected a trigger instance, got %s instead' % trigger.__class__.__name__) + + approved['trigger'] = trigger + + if 'executor' in changes: + value = changes.pop('executor') + if not isinstance(value, six.string_types): + raise TypeError('executor must be a string') + approved['executor'] = value + + if 'next_run_time' in changes: + value = changes.pop('next_run_time') + approved['next_run_time'] = convert_to_datetime(value, self._scheduler.timezone, 'next_run_time') + + if changes: + raise AttributeError('The following are not modifiable attributes of Job: %s' % ', '.join(changes)) + + for key, value in six.iteritems(approved): + setattr(self, key, value) def __getstate__(self): - # Prevents the unwanted pickling of transient or unpicklable variables - state = self.__dict__.copy() - state.pop('instances', None) - state.pop('func', None) - state.pop('_lock', None) - state['func_ref'] = obj_to_ref(self.func) - return state + # Don't allow this Job to be serialized if the function reference could not be determined + if not self.func_ref: + raise ValueError('This Job cannot be serialized since the reference to its callable (%r) could not be ' + 'determined. Consider giving a textual reference (module:function name) instead.' % + (self.func,)) + + return { + 'version': 1, + 'id': self.id, + 'func': self.func_ref, + 'trigger': self.trigger, + 'executor': self.executor, + 'args': self.args, + 'kwargs': self.kwargs, + 'name': self.name, + 'misfire_grace_time': self.misfire_grace_time, + 'coalesce': self.coalesce, + 'max_instances': self.max_instances, + 'next_run_time': self.next_run_time + } def __setstate__(self, state): - state['instances'] = 0 - state['func'] = ref_to_obj(state.pop('func_ref')) - state['_lock'] = Lock() - self.__dict__ = state + if state.get('version', 1) > 1: + raise ValueError('Job has version %s, but only version 1 can be handled' % state['version']) + + self.id = state['id'] + self.func_ref = state['func'] + self.func = ref_to_obj(self.func_ref) + self.trigger = state['trigger'] + self.executor = state['executor'] + self.args = state['args'] + self.kwargs = state['kwargs'] + self.name = state['name'] + self.misfire_grace_time = state['misfire_grace_time'] + self.coalesce = state['coalesce'] + self.max_instances = state['max_instances'] + self.next_run_time = state['next_run_time'] def __eq__(self, other): if isinstance(other, Job): - return self.id is not None and other.id == self.id or self is other + return self.id == other.id return NotImplemented def __repr__(self): - return '' % (self.name, repr(self.trigger)) + return '' % (repr_escape(self.id), repr_escape(self.name)) def __str__(self): - return '%s (trigger: %s, next run at: %s)' % (self.name, - str(self.trigger), str(self.next_run_time)) + return '%s (trigger: %s, next run at: %s)' % (repr_escape(self.name), repr_escape(str(self.trigger)), + datetime_repr(self.next_run_time)) + + def __unicode__(self): + return six.u('%s (trigger: %s, next run at: %s)') % (self.name, self.trigger, datetime_repr(self.next_run_time)) diff --git a/lib/apscheduler/jobstores/base.py b/lib/apscheduler/jobstores/base.py index f0a16ddb..d9f7147b 100644 --- a/lib/apscheduler/jobstores/base.py +++ b/lib/apscheduler/jobstores/base.py @@ -1,25 +1,127 @@ -""" -Abstract base class that provides the interface needed by all job stores. -Job store methods are also documented here. -""" +from abc import ABCMeta, abstractmethod +import logging + +import six -class JobStore(object): - def add_job(self, job): - """Adds the given job from this store.""" - raise NotImplementedError +class JobLookupError(KeyError): + """Raised when the job store cannot find a job for update or removal.""" - def update_job(self, job): - """Persists the running state of the given job.""" - raise NotImplementedError + def __init__(self, job_id): + super(JobLookupError, self).__init__(six.u('No job by the id of %s was found') % job_id) - def remove_job(self, job): - """Removes the given jobs from this store.""" - raise NotImplementedError - def load_jobs(self): - """Loads jobs from this store into memory.""" - raise NotImplementedError +class ConflictingIdError(KeyError): + """Raised when the uniqueness of job IDs is being violated.""" - def close(self): + def __init__(self, job_id): + super(ConflictingIdError, self).__init__(six.u('Job identifier (%s) conflicts with an existing job') % job_id) + + +class TransientJobError(ValueError): + """Raised when an attempt to add transient (with no func_ref) job to a persistent job store is detected.""" + + def __init__(self, job_id): + super(TransientJobError, self).__init__( + six.u('Job (%s) cannot be added to this job store because a reference to the callable could not be ' + 'determined.') % job_id) + + +class BaseJobStore(six.with_metaclass(ABCMeta)): + """Abstract base class that defines the interface that every job store must implement.""" + + _scheduler = None + _alias = None + _logger = logging.getLogger('apscheduler.jobstores') + + def start(self, scheduler, alias): + """ + Called by the scheduler when the scheduler is being started or when the job store is being added to an already + running scheduler. + + :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting this job store + :param str|unicode alias: alias of this job store as it was assigned to the scheduler + """ + + self._scheduler = scheduler + self._alias = alias + self._logger = logging.getLogger('apscheduler.jobstores.%s' % alias) + + def shutdown(self): """Frees any resources still bound to this job store.""" + + @abstractmethod + def lookup_job(self, job_id): + """ + Returns a specific job, or ``None`` if it isn't found.. + + The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of the returned job to + point to the scheduler and itself, respectively. + + :param str|unicode job_id: identifier of the job + :rtype: Job + """ + + @abstractmethod + def get_due_jobs(self, now): + """ + Returns the list of jobs that have ``next_run_time`` earlier or equal to ``now``. + The returned jobs must be sorted by next run time (ascending). + + :param datetime.datetime now: the current (timezone aware) datetime + :rtype: list[Job] + """ + + @abstractmethod + def get_next_run_time(self): + """ + Returns the earliest run time of all the jobs stored in this job store, or ``None`` if there are no active jobs. + + :rtype: datetime.datetime + """ + + @abstractmethod + def get_all_jobs(self): + """ + Returns a list of all jobs in this job store. The returned jobs should be sorted by next run time (ascending). + Paused jobs (next_run_time is None) should be sorted last. + + The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of the returned jobs to + point to the scheduler and itself, respectively. + + :rtype: list[Job] + """ + + @abstractmethod + def add_job(self, job): + """ + Adds the given job to this store. + + :param Job job: the job to add + :raises ConflictingIdError: if there is another job in this store with the same ID + """ + + @abstractmethod + def update_job(self, job): + """ + Replaces the job in the store with the given newer version. + + :param Job job: the job to update + :raises JobLookupError: if the job does not exist + """ + + @abstractmethod + def remove_job(self, job_id): + """ + Removes the given job from this store. + + :param str|unicode job_id: identifier of the job + :raises JobLookupError: if the job does not exist + """ + + @abstractmethod + def remove_all_jobs(self): + """Removes all jobs from this store.""" + + def __repr__(self): + return '<%s>' % self.__class__.__name__ diff --git a/lib/apscheduler/jobstores/memory.py b/lib/apscheduler/jobstores/memory.py new file mode 100644 index 00000000..645391f3 --- /dev/null +++ b/lib/apscheduler/jobstores/memory.py @@ -0,0 +1,107 @@ +from __future__ import absolute_import + +from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError +from apscheduler.util import datetime_to_utc_timestamp + + +class MemoryJobStore(BaseJobStore): + """ + Stores jobs in an array in RAM. Provides no persistence support. + + Plugin alias: ``memory`` + """ + + def __init__(self): + super(MemoryJobStore, self).__init__() + self._jobs = [] # list of (job, timestamp), sorted by next_run_time and job id (ascending) + self._jobs_index = {} # id -> (job, timestamp) lookup table + + def lookup_job(self, job_id): + return self._jobs_index.get(job_id, (None, None))[0] + + def get_due_jobs(self, now): + now_timestamp = datetime_to_utc_timestamp(now) + pending = [] + for job, timestamp in self._jobs: + if timestamp is None or timestamp > now_timestamp: + break + pending.append(job) + + return pending + + def get_next_run_time(self): + return self._jobs[0][0].next_run_time if self._jobs else None + + def get_all_jobs(self): + return [j[0] for j in self._jobs] + + def add_job(self, job): + if job.id in self._jobs_index: + raise ConflictingIdError(job.id) + + timestamp = datetime_to_utc_timestamp(job.next_run_time) + index = self._get_job_index(timestamp, job.id) + self._jobs.insert(index, (job, timestamp)) + self._jobs_index[job.id] = (job, timestamp) + + def update_job(self, job): + old_job, old_timestamp = self._jobs_index.get(job.id, (None, None)) + if old_job is None: + raise JobLookupError(job.id) + + # If the next run time has not changed, simply replace the job in its present index. + # Otherwise, reinsert the job to the list to preserve the ordering. + old_index = self._get_job_index(old_timestamp, old_job.id) + new_timestamp = datetime_to_utc_timestamp(job.next_run_time) + if old_timestamp == new_timestamp: + self._jobs[old_index] = (job, new_timestamp) + else: + del self._jobs[old_index] + new_index = self._get_job_index(new_timestamp, job.id) + self._jobs.insert(new_index, (job, new_timestamp)) + + self._jobs_index[old_job.id] = (job, new_timestamp) + + def remove_job(self, job_id): + job, timestamp = self._jobs_index.get(job_id, (None, None)) + if job is None: + raise JobLookupError(job_id) + + index = self._get_job_index(timestamp, job_id) + del self._jobs[index] + del self._jobs_index[job.id] + + def remove_all_jobs(self): + self._jobs = [] + self._jobs_index = {} + + def shutdown(self): + self.remove_all_jobs() + + def _get_job_index(self, timestamp, job_id): + """ + Returns the index of the given job, or if it's not found, the index where the job should be inserted based on + the given timestamp. + + :type timestamp: int + :type job_id: str + """ + + lo, hi = 0, len(self._jobs) + timestamp = float('inf') if timestamp is None else timestamp + while lo < hi: + mid = (lo + hi) // 2 + mid_job, mid_timestamp = self._jobs[mid] + mid_timestamp = float('inf') if mid_timestamp is None else mid_timestamp + if mid_timestamp > timestamp: + hi = mid + elif mid_timestamp < timestamp: + lo = mid + 1 + elif mid_job.id > job_id: + hi = mid + elif mid_job.id < job_id: + lo = mid + 1 + else: + return mid + + return lo diff --git a/lib/apscheduler/jobstores/mongodb.py b/lib/apscheduler/jobstores/mongodb.py new file mode 100644 index 00000000..ff762f7e --- /dev/null +++ b/lib/apscheduler/jobstores/mongodb.py @@ -0,0 +1,124 @@ +from __future__ import absolute_import + +from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError +from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime +from apscheduler.job import Job + +try: + import cPickle as pickle +except ImportError: # pragma: nocover + import pickle + +try: + from bson.binary import Binary + from pymongo.errors import DuplicateKeyError + from pymongo import MongoClient, ASCENDING +except ImportError: # pragma: nocover + raise ImportError('MongoDBJobStore requires PyMongo installed') + + +class MongoDBJobStore(BaseJobStore): + """ + Stores jobs in a MongoDB database. Any leftover keyword arguments are directly passed to pymongo's `MongoClient + `_. + + Plugin alias: ``mongodb`` + + :param str database: database to store jobs in + :param str collection: collection to store jobs in + :param client: a :class:`~pymongo.mongo_client.MongoClient` instance to use instead of providing connection + arguments + :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available + """ + + def __init__(self, database='apscheduler', collection='jobs', client=None, + pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args): + super(MongoDBJobStore, self).__init__() + self.pickle_protocol = pickle_protocol + + if not database: + raise ValueError('The "database" parameter must not be empty') + if not collection: + raise ValueError('The "collection" parameter must not be empty') + + if client: + self.connection = maybe_ref(client) + else: + connect_args.setdefault('w', 1) + self.connection = MongoClient(**connect_args) + + self.collection = self.connection[database][collection] + self.collection.ensure_index('next_run_time', sparse=True) + + def lookup_job(self, job_id): + document = self.collection.find_one(job_id, ['job_state']) + return self._reconstitute_job(document['job_state']) if document else None + + def get_due_jobs(self, now): + timestamp = datetime_to_utc_timestamp(now) + return self._get_jobs({'next_run_time': {'$lte': timestamp}}) + + def get_next_run_time(self): + document = self.collection.find_one({'next_run_time': {'$ne': None}}, fields=['next_run_time'], + sort=[('next_run_time', ASCENDING)]) + return utc_timestamp_to_datetime(document['next_run_time']) if document else None + + def get_all_jobs(self): + return self._get_jobs({}) + + def add_job(self, job): + try: + self.collection.insert({ + '_id': job.id, + 'next_run_time': datetime_to_utc_timestamp(job.next_run_time), + 'job_state': Binary(pickle.dumps(job.__getstate__(), self.pickle_protocol)) + }) + except DuplicateKeyError: + raise ConflictingIdError(job.id) + + def update_job(self, job): + changes = { + 'next_run_time': datetime_to_utc_timestamp(job.next_run_time), + 'job_state': Binary(pickle.dumps(job.__getstate__(), self.pickle_protocol)) + } + result = self.collection.update({'_id': job.id}, {'$set': changes}) + if result and result['n'] == 0: + raise JobLookupError(id) + + def remove_job(self, job_id): + result = self.collection.remove(job_id) + if result and result['n'] == 0: + raise JobLookupError(job_id) + + def remove_all_jobs(self): + self.collection.remove() + + def shutdown(self): + self.connection.disconnect() + + def _reconstitute_job(self, job_state): + job_state = pickle.loads(job_state) + job = Job.__new__(Job) + job.__setstate__(job_state) + job._scheduler = self._scheduler + job._jobstore_alias = self._alias + return job + + def _get_jobs(self, conditions): + jobs = [] + failed_job_ids = [] + for document in self.collection.find(conditions, ['_id', 'job_state'], sort=[('next_run_time', ASCENDING)]): + try: + jobs.append(self._reconstitute_job(document['job_state'])) + except: + self._logger.exception('Unable to restore job "%s" -- removing it', document['_id']) + failed_job_ids.append(document['_id']) + + # Remove all the jobs we failed to restore + if failed_job_ids: + self.collection.remove({'_id': {'$in': failed_job_ids}}) + + return jobs + + def __repr__(self): + return '<%s (client=%s)>' % (self.__class__.__name__, self.connection) diff --git a/lib/apscheduler/jobstores/mongodb_store.py b/lib/apscheduler/jobstores/mongodb_store.py deleted file mode 100644 index 3f522c25..00000000 --- a/lib/apscheduler/jobstores/mongodb_store.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -Stores jobs in a MongoDB database. -""" -import logging - -from apscheduler.jobstores.base import JobStore -from apscheduler.job import Job - -try: - import cPickle as pickle -except ImportError: # pragma: nocover - import pickle - -try: - from bson.binary import Binary - from pymongo.connection import Connection -except ImportError: # pragma: nocover - raise ImportError('MongoDBJobStore requires PyMongo installed') - -logger = logging.getLogger(__name__) - - -class MongoDBJobStore(JobStore): - def __init__(self, database='apscheduler', collection='jobs', - connection=None, pickle_protocol=pickle.HIGHEST_PROTOCOL, - **connect_args): - self.jobs = [] - self.pickle_protocol = pickle_protocol - - if not database: - raise ValueError('The "database" parameter must not be empty') - if not collection: - raise ValueError('The "collection" parameter must not be empty') - - if connection: - self.connection = connection - else: - self.connection = Connection(**connect_args) - - self.collection = self.connection[database][collection] - - def add_job(self, job): - job_dict = job.__getstate__() - job_dict['trigger'] = Binary(pickle.dumps(job.trigger, - self.pickle_protocol)) - job_dict['args'] = Binary(pickle.dumps(job.args, - self.pickle_protocol)) - job_dict['kwargs'] = Binary(pickle.dumps(job.kwargs, - self.pickle_protocol)) - job.id = self.collection.insert(job_dict) - self.jobs.append(job) - - def remove_job(self, job): - self.collection.remove(job.id) - self.jobs.remove(job) - - def load_jobs(self): - jobs = [] - for job_dict in self.collection.find(): - try: - job = Job.__new__(Job) - job_dict['id'] = job_dict.pop('_id') - job_dict['trigger'] = pickle.loads(job_dict['trigger']) - job_dict['args'] = pickle.loads(job_dict['args']) - job_dict['kwargs'] = pickle.loads(job_dict['kwargs']) - job.__setstate__(job_dict) - jobs.append(job) - except Exception: - job_name = job_dict.get('name', '(unknown)') - logger.exception('Unable to restore job "%s"', job_name) - self.jobs = jobs - - def update_job(self, job): - spec = {'_id': job.id} - document = {'$set': {'next_run_time': job.next_run_time}, - '$inc': {'runs': 1}} - self.collection.update(spec, document) - - def close(self): - self.connection.disconnect() - - def __repr__(self): - connection = self.collection.database.connection - return '<%s (connection=%s)>' % (self.__class__.__name__, connection) diff --git a/lib/apscheduler/jobstores/ram_store.py b/lib/apscheduler/jobstores/ram_store.py deleted file mode 100644 index 60458fba..00000000 --- a/lib/apscheduler/jobstores/ram_store.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -Stores jobs in an array in RAM. Provides no persistence support. -""" - -from apscheduler.jobstores.base import JobStore - - -class RAMJobStore(JobStore): - def __init__(self): - self.jobs = [] - - def add_job(self, job): - self.jobs.append(job) - - def update_job(self, job): - pass - - def remove_job(self, job): - self.jobs.remove(job) - - def load_jobs(self): - pass - - def __repr__(self): - return '<%s>' % (self.__class__.__name__) diff --git a/lib/apscheduler/jobstores/redis.py b/lib/apscheduler/jobstores/redis.py new file mode 100644 index 00000000..2b4ffd52 --- /dev/null +++ b/lib/apscheduler/jobstores/redis.py @@ -0,0 +1,138 @@ +from __future__ import absolute_import + +import six + +from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError +from apscheduler.util import datetime_to_utc_timestamp, utc_timestamp_to_datetime +from apscheduler.job import Job + +try: + import cPickle as pickle +except ImportError: # pragma: nocover + import pickle + +try: + from redis import StrictRedis +except ImportError: # pragma: nocover + raise ImportError('RedisJobStore requires redis installed') + + +class RedisJobStore(BaseJobStore): + """ + Stores jobs in a Redis database. Any leftover keyword arguments are directly passed to redis's StrictRedis. + + Plugin alias: ``redis`` + + :param int db: the database number to store jobs in + :param str jobs_key: key to store jobs in + :param str run_times_key: key to store the jobs' run times in + :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available + """ + + def __init__(self, db=0, jobs_key='apscheduler.jobs', run_times_key='apscheduler.run_times', + pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args): + super(RedisJobStore, self).__init__() + + if db is None: + raise ValueError('The "db" parameter must not be empty') + if not jobs_key: + raise ValueError('The "jobs_key" parameter must not be empty') + if not run_times_key: + raise ValueError('The "run_times_key" parameter must not be empty') + + self.pickle_protocol = pickle_protocol + self.jobs_key = jobs_key + self.run_times_key = run_times_key + self.redis = StrictRedis(db=int(db), **connect_args) + + def lookup_job(self, job_id): + job_state = self.redis.hget(self.jobs_key, job_id) + return self._reconstitute_job(job_state) if job_state else None + + def get_due_jobs(self, now): + timestamp = datetime_to_utc_timestamp(now) + job_ids = self.redis.zrangebyscore(self.run_times_key, 0, timestamp) + if job_ids: + job_states = self.redis.hmget(self.jobs_key, *job_ids) + return self._reconstitute_jobs(six.moves.zip(job_ids, job_states)) + return [] + + def get_next_run_time(self): + next_run_time = self.redis.zrange(self.run_times_key, 0, 0, withscores=True) + if next_run_time: + return utc_timestamp_to_datetime(next_run_time[0][1]) + + def get_all_jobs(self): + job_states = self.redis.hgetall(self.jobs_key) + jobs = self._reconstitute_jobs(six.iteritems(job_states)) + return sorted(jobs, key=lambda job: job.next_run_time) + + def add_job(self, job): + if self.redis.hexists(self.jobs_key, job.id): + raise ConflictingIdError(job.id) + + with self.redis.pipeline() as pipe: + pipe.multi() + pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(), self.pickle_protocol)) + pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id) + pipe.execute() + + def update_job(self, job): + if not self.redis.hexists(self.jobs_key, job.id): + raise JobLookupError(job.id) + + with self.redis.pipeline() as pipe: + pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(), self.pickle_protocol)) + if job.next_run_time: + pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id) + else: + pipe.zrem(self.run_times_key, job.id) + pipe.execute() + + def remove_job(self, job_id): + if not self.redis.hexists(self.jobs_key, job_id): + raise JobLookupError(job_id) + + with self.redis.pipeline() as pipe: + pipe.hdel(self.jobs_key, job_id) + pipe.zrem(self.run_times_key, job_id) + pipe.execute() + + def remove_all_jobs(self): + with self.redis.pipeline() as pipe: + pipe.delete(self.jobs_key) + pipe.delete(self.run_times_key) + pipe.execute() + + def shutdown(self): + self.redis.connection_pool.disconnect() + + def _reconstitute_job(self, job_state): + job_state = pickle.loads(job_state) + job = Job.__new__(Job) + job.__setstate__(job_state) + job._scheduler = self._scheduler + job._jobstore_alias = self._alias + return job + + def _reconstitute_jobs(self, job_states): + jobs = [] + failed_job_ids = [] + for job_id, job_state in job_states: + try: + jobs.append(self._reconstitute_job(job_state)) + except: + self._logger.exception('Unable to restore job "%s" -- removing it', job_id) + failed_job_ids.append(job_id) + + # Remove all the jobs we failed to restore + if failed_job_ids: + with self.redis.pipeline() as pipe: + pipe.hdel(self.jobs_key, *failed_job_ids) + pipe.zrem(self.run_times_key, *failed_job_ids) + pipe.execute() + + return jobs + + def __repr__(self): + return '<%s>' % self.__class__.__name__ diff --git a/lib/apscheduler/jobstores/shelve_store.py b/lib/apscheduler/jobstores/shelve_store.py deleted file mode 100644 index 87c95f8f..00000000 --- a/lib/apscheduler/jobstores/shelve_store.py +++ /dev/null @@ -1,65 +0,0 @@ -""" -Stores jobs in a file governed by the :mod:`shelve` module. -""" - -import shelve -import pickle -import random -import logging - -from apscheduler.jobstores.base import JobStore -from apscheduler.job import Job -from apscheduler.util import itervalues - -logger = logging.getLogger(__name__) - - -class ShelveJobStore(JobStore): - MAX_ID = 1000000 - - def __init__(self, path, pickle_protocol=pickle.HIGHEST_PROTOCOL): - self.jobs = [] - self.path = path - self.pickle_protocol = pickle_protocol - self.store = shelve.open(path, 'c', self.pickle_protocol) - - def _generate_id(self): - id = None - while not id: - id = str(random.randint(1, self.MAX_ID)) - if not id in self.store: - return id - - def add_job(self, job): - job.id = self._generate_id() - self.jobs.append(job) - self.store[job.id] = job.__getstate__() - - def update_job(self, job): - job_dict = self.store[job.id] - job_dict['next_run_time'] = job.next_run_time - job_dict['runs'] = job.runs - self.store[job.id] = job_dict - - def remove_job(self, job): - del self.store[job.id] - self.jobs.remove(job) - - def load_jobs(self): - jobs = [] - for job_dict in itervalues(self.store): - try: - job = Job.__new__(Job) - job.__setstate__(job_dict) - jobs.append(job) - except Exception: - job_name = job_dict.get('name', '(unknown)') - logger.exception('Unable to restore job "%s"', job_name) - - self.jobs = jobs - - def close(self): - self.store.close() - - def __repr__(self): - return '<%s (path=%s)>' % (self.__class__.__name__, self.path) diff --git a/lib/apscheduler/jobstores/sqlalchemy.py b/lib/apscheduler/jobstores/sqlalchemy.py new file mode 100644 index 00000000..f8a3c151 --- /dev/null +++ b/lib/apscheduler/jobstores/sqlalchemy.py @@ -0,0 +1,137 @@ +from __future__ import absolute_import + +from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError +from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime +from apscheduler.job import Job + +try: + import cPickle as pickle +except ImportError: # pragma: nocover + import pickle + +try: + from sqlalchemy import create_engine, Table, Column, MetaData, Unicode, Float, LargeBinary, select + from sqlalchemy.exc import IntegrityError +except ImportError: # pragma: nocover + raise ImportError('SQLAlchemyJobStore requires SQLAlchemy installed') + + +class SQLAlchemyJobStore(BaseJobStore): + """ + Stores jobs in a database table using SQLAlchemy. The table will be created if it doesn't exist in the database. + + Plugin alias: ``sqlalchemy`` + + :param str url: connection string (see `SQLAlchemy documentation + `_ + on this) + :param engine: an SQLAlchemy Engine to use instead of creating a new one based on ``url`` + :param str tablename: name of the table to store jobs in + :param metadata: a :class:`~sqlalchemy.MetaData` instance to use instead of creating a new one + :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available + """ + + def __init__(self, url=None, engine=None, tablename='apscheduler_jobs', metadata=None, + pickle_protocol=pickle.HIGHEST_PROTOCOL): + super(SQLAlchemyJobStore, self).__init__() + self.pickle_protocol = pickle_protocol + metadata = maybe_ref(metadata) or MetaData() + + if engine: + self.engine = maybe_ref(engine) + elif url: + self.engine = create_engine(url) + else: + raise ValueError('Need either "engine" or "url" defined') + + # 191 = max key length in MySQL for InnoDB/utf8mb4 tables, 25 = precision that translates to an 8-byte float + self.jobs_t = Table( + tablename, metadata, + Column('id', Unicode(191, _warn_on_bytestring=False), primary_key=True), + Column('next_run_time', Float(25), index=True), + Column('job_state', LargeBinary, nullable=False) + ) + + self.jobs_t.create(self.engine, True) + + def lookup_job(self, job_id): + selectable = select([self.jobs_t.c.job_state]).where(self.jobs_t.c.id == job_id) + job_state = self.engine.execute(selectable).scalar() + return self._reconstitute_job(job_state) if job_state else None + + def get_due_jobs(self, now): + timestamp = datetime_to_utc_timestamp(now) + return self._get_jobs(self.jobs_t.c.next_run_time <= timestamp) + + def get_next_run_time(self): + selectable = select([self.jobs_t.c.next_run_time]).where(self.jobs_t.c.next_run_time != None).\ + order_by(self.jobs_t.c.next_run_time).limit(1) + next_run_time = self.engine.execute(selectable).scalar() + return utc_timestamp_to_datetime(next_run_time) + + def get_all_jobs(self): + return self._get_jobs() + + def add_job(self, job): + insert = self.jobs_t.insert().values(**{ + 'id': job.id, + 'next_run_time': datetime_to_utc_timestamp(job.next_run_time), + 'job_state': pickle.dumps(job.__getstate__(), self.pickle_protocol) + }) + try: + self.engine.execute(insert) + except IntegrityError: + raise ConflictingIdError(job.id) + + def update_job(self, job): + update = self.jobs_t.update().values(**{ + 'next_run_time': datetime_to_utc_timestamp(job.next_run_time), + 'job_state': pickle.dumps(job.__getstate__(), self.pickle_protocol) + }).where(self.jobs_t.c.id == job.id) + result = self.engine.execute(update) + if result.rowcount == 0: + raise JobLookupError(id) + + def remove_job(self, job_id): + delete = self.jobs_t.delete().where(self.jobs_t.c.id == job_id) + result = self.engine.execute(delete) + if result.rowcount == 0: + raise JobLookupError(job_id) + + def remove_all_jobs(self): + delete = self.jobs_t.delete() + self.engine.execute(delete) + + def shutdown(self): + self.engine.dispose() + + def _reconstitute_job(self, job_state): + job_state = pickle.loads(job_state) + job_state['jobstore'] = self + job = Job.__new__(Job) + job.__setstate__(job_state) + job._scheduler = self._scheduler + job._jobstore_alias = self._alias + return job + + def _get_jobs(self, *conditions): + jobs = [] + selectable = select([self.jobs_t.c.id, self.jobs_t.c.job_state]).order_by(self.jobs_t.c.next_run_time) + selectable = selectable.where(*conditions) if conditions else selectable + failed_job_ids = set() + for row in self.engine.execute(selectable): + try: + jobs.append(self._reconstitute_job(row.job_state)) + except: + self._logger.exception('Unable to restore job "%s" -- removing it', row.id) + failed_job_ids.add(row.id) + + # Remove all the jobs we failed to restore + if failed_job_ids: + delete = self.jobs_t.delete().where(self.jobs_t.c.id.in_(failed_job_ids)) + self.engine.execute(delete) + + return jobs + + def __repr__(self): + return '<%s (url=%s)>' % (self.__class__.__name__, self.engine.url) diff --git a/lib/apscheduler/jobstores/sqlalchemy_store.py b/lib/apscheduler/jobstores/sqlalchemy_store.py deleted file mode 100644 index 8ece7e24..00000000 --- a/lib/apscheduler/jobstores/sqlalchemy_store.py +++ /dev/null @@ -1,87 +0,0 @@ -""" -Stores jobs in a database table using SQLAlchemy. -""" -import pickle -import logging - -from apscheduler.jobstores.base import JobStore -from apscheduler.job import Job - -try: - from sqlalchemy import * -except ImportError: # pragma: nocover - raise ImportError('SQLAlchemyJobStore requires SQLAlchemy installed') - -logger = logging.getLogger(__name__) - - -class SQLAlchemyJobStore(JobStore): - def __init__(self, url=None, engine=None, tablename='apscheduler_jobs', - metadata=None, pickle_protocol=pickle.HIGHEST_PROTOCOL): - self.jobs = [] - self.pickle_protocol = pickle_protocol - - if engine: - self.engine = engine - elif url: - self.engine = create_engine(url) - else: - raise ValueError('Need either "engine" or "url" defined') - - self.jobs_t = Table(tablename, metadata or MetaData(), - Column('id', Integer, - Sequence(tablename + '_id_seq', optional=True), - primary_key=True), - Column('trigger', PickleType(pickle_protocol, mutable=False), - nullable=False), - Column('func_ref', String(1024), nullable=False), - Column('args', PickleType(pickle_protocol, mutable=False), - nullable=False), - Column('kwargs', PickleType(pickle_protocol, mutable=False), - nullable=False), - Column('name', Unicode(1024), unique=True), - Column('misfire_grace_time', Integer, nullable=False), - Column('coalesce', Boolean, nullable=False), - Column('max_runs', Integer), - Column('max_instances', Integer), - Column('next_run_time', DateTime, nullable=False), - Column('runs', BigInteger)) - - self.jobs_t.create(self.engine, True) - - def add_job(self, job): - job_dict = job.__getstate__() - result = self.engine.execute(self.jobs_t.insert().values(**job_dict)) - job.id = result.inserted_primary_key[0] - self.jobs.append(job) - - def remove_job(self, job): - delete = self.jobs_t.delete().where(self.jobs_t.c.id == job.id) - self.engine.execute(delete) - self.jobs.remove(job) - - def load_jobs(self): - jobs = [] - for row in self.engine.execute(select([self.jobs_t])): - try: - job = Job.__new__(Job) - job_dict = dict(row.items()) - job.__setstate__(job_dict) - jobs.append(job) - except Exception: - job_name = job_dict.get('name', '(unknown)') - logger.exception('Unable to restore job "%s"', job_name) - self.jobs = jobs - - def update_job(self, job): - job_dict = job.__getstate__() - update = self.jobs_t.update().where(self.jobs_t.c.id == job.id).\ - values(next_run_time=job_dict['next_run_time'], - runs=job_dict['runs']) - self.engine.execute(update) - - def close(self): - self.engine.dispose() - - def __repr__(self): - return '<%s (url=%s)>' % (self.__class__.__name__, self.engine.url) diff --git a/lib/apscheduler/scheduler.py b/lib/apscheduler/scheduler.py deleted file mode 100644 index ee08ad8b..00000000 --- a/lib/apscheduler/scheduler.py +++ /dev/null @@ -1,559 +0,0 @@ -""" -This module is the main part of the library. It houses the Scheduler class -and related exceptions. -""" - -from threading import Thread, Event, Lock -from datetime import datetime, timedelta -from logging import getLogger -import os -import sys - -from apscheduler.util import * -from apscheduler.triggers import SimpleTrigger, IntervalTrigger, CronTrigger -from apscheduler.jobstores.ram_store import RAMJobStore -from apscheduler.job import Job, MaxInstancesReachedError -from apscheduler.events import * -from apscheduler.threadpool import ThreadPool - -logger = getLogger(__name__) - - -class SchedulerAlreadyRunningError(Exception): - """ - Raised when attempting to start or configure the scheduler when it's - already running. - """ - - def __str__(self): - return 'Scheduler is already running' - - -class Scheduler(object): - """ - This class is responsible for scheduling jobs and triggering - their execution. - """ - - _stopped = False - _thread = None - - def __init__(self, gconfig={}, **options): - self._wakeup = Event() - self._jobstores = {} - self._jobstores_lock = Lock() - self._listeners = [] - self._listeners_lock = Lock() - self._pending_jobs = [] - self.configure(gconfig, **options) - - def configure(self, gconfig={}, **options): - """ - Reconfigures the scheduler with the given options. Can only be done - when the scheduler isn't running. - """ - if self.running: - raise SchedulerAlreadyRunningError - - # Set general options - config = combine_opts(gconfig, 'apscheduler.', options) - self.misfire_grace_time = int(config.pop('misfire_grace_time', 1)) - self.coalesce = asbool(config.pop('coalesce', True)) - self.daemonic = asbool(config.pop('daemonic', True)) - - # Configure the thread pool - if 'threadpool' in config: - self._threadpool = maybe_ref(config['threadpool']) - else: - threadpool_opts = combine_opts(config, 'threadpool.') - self._threadpool = ThreadPool(**threadpool_opts) - - # Configure job stores - jobstore_opts = combine_opts(config, 'jobstore.') - jobstores = {} - for key, value in jobstore_opts.items(): - store_name, option = key.split('.', 1) - opts_dict = jobstores.setdefault(store_name, {}) - opts_dict[option] = value - - for alias, opts in jobstores.items(): - classname = opts.pop('class') - cls = maybe_ref(classname) - jobstore = cls(**opts) - self.add_jobstore(jobstore, alias, True) - - def start(self): - """ - Starts the scheduler in a new thread. - """ - if self.running: - raise SchedulerAlreadyRunningError - - # Create a RAMJobStore as the default if there is no default job store - if not 'default' in self._jobstores: - self.add_jobstore(RAMJobStore(), 'default', True) - - # Schedule all pending jobs - for job, jobstore in self._pending_jobs: - self._real_add_job(job, jobstore, False) - del self._pending_jobs[:] - - self._stopped = False - self._thread = Thread(target=self._main_loop, name='APScheduler') - self._thread.setDaemon(self.daemonic) - self._thread.start() - - def shutdown(self, wait=True, shutdown_threadpool=True): - """ - Shuts down the scheduler and terminates the thread. - Does not interrupt any currently running jobs. - - :param wait: ``True`` to wait until all currently executing jobs have - finished (if ``shutdown_threadpool`` is also ``True``) - :param shutdown_threadpool: ``True`` to shut down the thread pool - """ - if not self.running: - return - - self._stopped = True - self._wakeup.set() - - # Shut down the thread pool - if shutdown_threadpool: - self._threadpool.shutdown(wait) - - # Wait until the scheduler thread terminates - self._thread.join() - - @property - def running(self): - return not self._stopped and self._thread and self._thread.isAlive() - - def add_jobstore(self, jobstore, alias, quiet=False): - """ - Adds a job store to this scheduler. - - :param jobstore: job store to be added - :param alias: alias for the job store - :param quiet: True to suppress scheduler thread wakeup - :type jobstore: instance of - :class:`~apscheduler.jobstores.base.JobStore` - :type alias: str - """ - self._jobstores_lock.acquire() - try: - if alias in self._jobstores: - raise KeyError('Alias "%s" is already in use' % alias) - self._jobstores[alias] = jobstore - jobstore.load_jobs() - finally: - self._jobstores_lock.release() - - # Notify listeners that a new job store has been added - self._notify_listeners(JobStoreEvent(EVENT_JOBSTORE_ADDED, alias)) - - # Notify the scheduler so it can scan the new job store for jobs - if not quiet: - self._wakeup.set() - - def remove_jobstore(self, alias): - """ - Removes the job store by the given alias from this scheduler. - - :type alias: str - """ - self._jobstores_lock.acquire() - try: - try: - del self._jobstores[alias] - except KeyError: - raise KeyError('No such job store: %s' % alias) - finally: - self._jobstores_lock.release() - - # Notify listeners that a job store has been removed - self._notify_listeners(JobStoreEvent(EVENT_JOBSTORE_REMOVED, alias)) - - def add_listener(self, callback, mask=EVENT_ALL): - """ - Adds a listener for scheduler events. When a matching event occurs, - ``callback`` is executed with the event object as its sole argument. - If the ``mask`` parameter is not provided, the callback will receive - events of all types. - - :param callback: any callable that takes one argument - :param mask: bitmask that indicates which events should be listened to - """ - self._listeners_lock.acquire() - try: - self._listeners.append((callback, mask)) - finally: - self._listeners_lock.release() - - def remove_listener(self, callback): - """ - Removes a previously added event listener. - """ - self._listeners_lock.acquire() - try: - for i, (cb, _) in enumerate(self._listeners): - if callback == cb: - del self._listeners[i] - finally: - self._listeners_lock.release() - - def _notify_listeners(self, event): - self._listeners_lock.acquire() - try: - listeners = tuple(self._listeners) - finally: - self._listeners_lock.release() - - for cb, mask in listeners: - if event.code & mask: - try: - cb(event) - except: - logger.exception('Error notifying listener') - - def _real_add_job(self, job, jobstore, wakeup): - job.compute_next_run_time(datetime.now()) - if not job.next_run_time: - raise ValueError('Not adding job since it would never be run') - - self._jobstores_lock.acquire() - try: - try: - store = self._jobstores[jobstore] - except KeyError: - raise KeyError('No such job store: %s' % jobstore) - store.add_job(job) - finally: - self._jobstores_lock.release() - - # Notify listeners that a new job has been added - event = JobStoreEvent(EVENT_JOBSTORE_JOB_ADDED, jobstore, job) - self._notify_listeners(event) - - logger.info('Added job "%s" to job store "%s"', job, jobstore) - - # Notify the scheduler about the new job - if wakeup: - self._wakeup.set() - - def add_job(self, trigger, func, args, kwargs, jobstore='default', - **options): - """ - Adds the given job to the job list and notifies the scheduler thread. - - :param trigger: alias of the job store to store the job in - :param func: callable to run at the given time - :param args: list of positional arguments to call func with - :param kwargs: dict of keyword arguments to call func with - :param jobstore: alias of the job store to store the job in - :rtype: :class:`~apscheduler.job.Job` - """ - job = Job(trigger, func, args or [], kwargs or {}, - options.pop('misfire_grace_time', self.misfire_grace_time), - options.pop('coalesce', self.coalesce), **options) - if not self.running: - self._pending_jobs.append((job, jobstore)) - logger.info('Adding job tentatively -- it will be properly ' - 'scheduled when the scheduler starts') - else: - self._real_add_job(job, jobstore, True) - return job - - def _remove_job(self, job, alias, jobstore): - jobstore.remove_job(job) - - # Notify listeners that a job has been removed - event = JobStoreEvent(EVENT_JOBSTORE_JOB_REMOVED, alias, job) - self._notify_listeners(event) - - logger.info('Removed job "%s"', job) - - def add_date_job(self, func, date, args=None, kwargs=None, **options): - """ - Schedules a job to be completed on a specific date and time. - - :param func: callable to run at the given time - :param date: the date/time to run the job at - :param name: name of the job - :param jobstore: stored the job in the named (or given) job store - :param misfire_grace_time: seconds after the designated run time that - the job is still allowed to be run - :type date: :class:`datetime.date` - :rtype: :class:`~apscheduler.job.Job` - """ - trigger = SimpleTrigger(date) - return self.add_job(trigger, func, args, kwargs, **options) - - def add_interval_job(self, func, weeks=0, days=0, hours=0, minutes=0, - seconds=0, start_date=None, args=None, kwargs=None, - **options): - """ - Schedules a job to be completed on specified intervals. - - :param func: callable to run - :param weeks: number of weeks to wait - :param days: number of days to wait - :param hours: number of hours to wait - :param minutes: number of minutes to wait - :param seconds: number of seconds to wait - :param start_date: when to first execute the job and start the - counter (default is after the given interval) - :param args: list of positional arguments to call func with - :param kwargs: dict of keyword arguments to call func with - :param name: name of the job - :param jobstore: alias of the job store to add the job to - :param misfire_grace_time: seconds after the designated run time that - the job is still allowed to be run - :rtype: :class:`~apscheduler.job.Job` - """ - interval = timedelta(weeks=weeks, days=days, hours=hours, - minutes=minutes, seconds=seconds) - trigger = IntervalTrigger(interval, start_date) - return self.add_job(trigger, func, args, kwargs, **options) - - def add_cron_job(self, func, year='*', month='*', day='*', week='*', - day_of_week='*', hour='*', minute='*', second='*', - start_date=None, args=None, kwargs=None, **options): - """ - Schedules a job to be completed on times that match the given - expressions. - - :param func: callable to run - :param year: year to run on - :param month: month to run on (0 = January) - :param day: day of month to run on - :param week: week of the year to run on - :param day_of_week: weekday to run on (0 = Monday) - :param hour: hour to run on - :param second: second to run on - :param args: list of positional arguments to call func with - :param kwargs: dict of keyword arguments to call func with - :param name: name of the job - :param jobstore: alias of the job store to add the job to - :param misfire_grace_time: seconds after the designated run time that - the job is still allowed to be run - :return: the scheduled job - :rtype: :class:`~apscheduler.job.Job` - """ - trigger = CronTrigger(year=year, month=month, day=day, week=week, - day_of_week=day_of_week, hour=hour, - minute=minute, second=second, - start_date=start_date) - return self.add_job(trigger, func, args, kwargs, **options) - - def cron_schedule(self, **options): - """ - Decorator version of :meth:`add_cron_job`. - This decorator does not wrap its host function. - Unscheduling decorated functions is possible by passing the ``job`` - attribute of the scheduled function to :meth:`unschedule_job`. - """ - def inner(func): - func.job = self.add_cron_job(func, **options) - return func - return inner - - def interval_schedule(self, **options): - """ - Decorator version of :meth:`add_interval_job`. - This decorator does not wrap its host function. - Unscheduling decorated functions is possible by passing the ``job`` - attribute of the scheduled function to :meth:`unschedule_job`. - """ - def inner(func): - func.job = self.add_interval_job(func, **options) - return func - return inner - - def get_jobs(self): - """ - Returns a list of all scheduled jobs. - - :return: list of :class:`~apscheduler.job.Job` objects - """ - self._jobstores_lock.acquire() - try: - jobs = [] - for jobstore in itervalues(self._jobstores): - jobs.extend(jobstore.jobs) - return jobs - finally: - self._jobstores_lock.release() - - def unschedule_job(self, job): - """ - Removes a job, preventing it from being run any more. - """ - self._jobstores_lock.acquire() - try: - for alias, jobstore in iteritems(self._jobstores): - if job in list(jobstore.jobs): - self._remove_job(job, alias, jobstore) - return - finally: - self._jobstores_lock.release() - - raise KeyError('Job "%s" is not scheduled in any job store' % job) - - def unschedule_func(self, func): - """ - Removes all jobs that would execute the given function. - """ - found = False - self._jobstores_lock.acquire() - try: - for alias, jobstore in iteritems(self._jobstores): - for job in list(jobstore.jobs): - if job.func == func: - self._remove_job(job, alias, jobstore) - found = True - finally: - self._jobstores_lock.release() - - if not found: - raise KeyError('The given function is not scheduled in this ' - 'scheduler') - - def print_jobs(self, out=None): - """ - Prints out a textual listing of all jobs currently scheduled on this - scheduler. - - :param out: a file-like object to print to (defaults to **sys.stdout** - if nothing is given) - """ - out = out or sys.stdout - job_strs = [] - self._jobstores_lock.acquire() - try: - for alias, jobstore in iteritems(self._jobstores): - job_strs.append('Jobstore %s:' % alias) - if jobstore.jobs: - for job in jobstore.jobs: - job_strs.append(' %s' % job) - else: - job_strs.append(' No scheduled jobs') - finally: - self._jobstores_lock.release() - - out.write(os.linesep.join(job_strs)) - - def _run_job(self, job, run_times): - """ - Acts as a harness that runs the actual job code in a thread. - """ - for run_time in run_times: - # See if the job missed its run time window, and handle possible - # misfires accordingly - difference = datetime.now() - run_time - grace_time = timedelta(seconds=job.misfire_grace_time) - if difference > grace_time: - # Notify listeners about a missed run - event = JobEvent(EVENT_JOB_MISSED, job, run_time) - self._notify_listeners(event) - logger.warning('Run time of job "%s" was missed by %s', - job, difference) - else: - try: - job.add_instance() - except MaxInstancesReachedError: - event = JobEvent(EVENT_JOB_MISSED, job, run_time) - self._notify_listeners(event) - logger.warning('Execution of job "%s" skipped: ' - 'maximum number of running instances ' - 'reached (%d)', job, job.max_instances) - break - - logger.info('Running job "%s" (scheduled at %s)', job, - run_time) - - try: - retval = job.func(*job.args, **job.kwargs) - except: - # Notify listeners about the exception - exc, tb = sys.exc_info()[1:] - event = JobEvent(EVENT_JOB_ERROR, job, run_time, - exception=exc, traceback=tb) - self._notify_listeners(event) - - logger.exception('Job "%s" raised an exception', job) - else: - # Notify listeners about successful execution - event = JobEvent(EVENT_JOB_EXECUTED, job, run_time, - retval=retval) - self._notify_listeners(event) - - logger.info('Job "%s" executed successfully', job) - - job.remove_instance() - - # If coalescing is enabled, don't attempt any further runs - if job.coalesce: - break - - def _process_jobs(self, now): - """ - Iterates through jobs in every jobstore, starts pending jobs - and figures out the next wakeup time. - """ - next_wakeup_time = None - self._jobstores_lock.acquire() - try: - for alias, jobstore in iteritems(self._jobstores): - for job in tuple(jobstore.jobs): - run_times = job.get_run_times(now) - if run_times: - self._threadpool.submit(self._run_job, job, run_times) - - # Increase the job's run count - if job.coalesce: - job.runs += 1 - else: - job.runs += len(run_times) - - # Update the job, but don't keep finished jobs around - if job.compute_next_run_time(now + timedelta(microseconds=1)): - jobstore.update_job(job) - else: - self._remove_job(job, alias, jobstore) - - if not next_wakeup_time: - next_wakeup_time = job.next_run_time - elif job.next_run_time: - next_wakeup_time = min(next_wakeup_time, - job.next_run_time) - return next_wakeup_time - finally: - self._jobstores_lock.release() - - def _main_loop(self): - """Executes jobs on schedule.""" - - logger.info('Scheduler started') - self._notify_listeners(SchedulerEvent(EVENT_SCHEDULER_START)) - - self._wakeup.clear() - while not self._stopped: - logger.debug('Looking for jobs to run') - now = datetime.now() - next_wakeup_time = self._process_jobs(now) - - # Sleep until the next job is scheduled to be run, - # a new job is added or the scheduler is stopped - if next_wakeup_time is not None: - wait_seconds = time_difference(next_wakeup_time, now) - logger.debug('Next wakeup is due at %s (in %f seconds)', - next_wakeup_time, wait_seconds) - self._wakeup.wait(wait_seconds) - else: - logger.debug('No jobs; waiting until a job is added') - self._wakeup.wait() - self._wakeup.clear() - - logger.info('Scheduler has been shut down') - self._notify_listeners(SchedulerEvent(EVENT_SCHEDULER_SHUTDOWN)) diff --git a/lib/apscheduler/schedulers/__init__.py b/lib/apscheduler/schedulers/__init__.py new file mode 100644 index 00000000..bd8a7900 --- /dev/null +++ b/lib/apscheduler/schedulers/__init__.py @@ -0,0 +1,12 @@ +class SchedulerAlreadyRunningError(Exception): + """Raised when attempting to start or configure the scheduler when it's already running.""" + + def __str__(self): + return 'Scheduler is already running' + + +class SchedulerNotRunningError(Exception): + """Raised when attempting to shutdown the scheduler when it's not running.""" + + def __str__(self): + return 'Scheduler is not running' diff --git a/lib/apscheduler/schedulers/asyncio.py b/lib/apscheduler/schedulers/asyncio.py new file mode 100644 index 00000000..b91ee97a --- /dev/null +++ b/lib/apscheduler/schedulers/asyncio.py @@ -0,0 +1,68 @@ +from __future__ import absolute_import +from functools import wraps + +from apscheduler.schedulers.base import BaseScheduler +from apscheduler.util import maybe_ref + +try: + import asyncio +except ImportError: # pragma: nocover + try: + import trollius as asyncio + except ImportError: + raise ImportError('AsyncIOScheduler requires either Python 3.4 or the asyncio package installed') + + +def run_in_event_loop(func): + @wraps(func) + def wrapper(self, *args, **kwargs): + self._eventloop.call_soon_threadsafe(func, self, *args, **kwargs) + return wrapper + + +class AsyncIOScheduler(BaseScheduler): + """ + A scheduler that runs on an asyncio (:pep:`3156`) event loop. + + Extra options: + + ============== ============================================================= + ``event_loop`` AsyncIO event loop to use (defaults to the global event loop) + ============== ============================================================= + """ + + _eventloop = None + _timeout = None + + def start(self): + super(AsyncIOScheduler, self).start() + self.wakeup() + + @run_in_event_loop + def shutdown(self, wait=True): + super(AsyncIOScheduler, self).shutdown(wait) + self._stop_timer() + + def _configure(self, config): + self._eventloop = maybe_ref(config.pop('event_loop', None)) or asyncio.get_event_loop() + super(AsyncIOScheduler, self)._configure(config) + + def _start_timer(self, wait_seconds): + self._stop_timer() + if wait_seconds is not None: + self._timeout = self._eventloop.call_later(wait_seconds, self.wakeup) + + def _stop_timer(self): + if self._timeout: + self._timeout.cancel() + del self._timeout + + @run_in_event_loop + def wakeup(self): + self._stop_timer() + wait_seconds = self._process_jobs() + self._start_timer(wait_seconds) + + def _create_default_executor(self): + from apscheduler.executors.asyncio import AsyncIOExecutor + return AsyncIOExecutor() diff --git a/lib/apscheduler/schedulers/background.py b/lib/apscheduler/schedulers/background.py new file mode 100644 index 00000000..86ff2ba3 --- /dev/null +++ b/lib/apscheduler/schedulers/background.py @@ -0,0 +1,39 @@ +from __future__ import absolute_import +from threading import Thread, Event + +from apscheduler.schedulers.base import BaseScheduler +from apscheduler.schedulers.blocking import BlockingScheduler +from apscheduler.util import asbool + + +class BackgroundScheduler(BlockingScheduler): + """ + A scheduler that runs in the background using a separate thread + (:meth:`~apscheduler.schedulers.base.BaseScheduler.start` will return immediately). + + Extra options: + + ========== ============================================================================================ + ``daemon`` Set the ``daemon`` option in the background thread (defaults to ``True``, + see `the documentation `_ + for further details) + ========== ============================================================================================ + """ + + _thread = None + + def _configure(self, config): + self._daemon = asbool(config.pop('daemon', True)) + super(BackgroundScheduler, self)._configure(config) + + def start(self): + BaseScheduler.start(self) + self._event = Event() + self._thread = Thread(target=self._main_loop, name='APScheduler') + self._thread.daemon = self._daemon + self._thread.start() + + def shutdown(self, wait=True): + super(BackgroundScheduler, self).shutdown(wait) + self._thread.join() + del self._thread diff --git a/lib/apscheduler/schedulers/base.py b/lib/apscheduler/schedulers/base.py new file mode 100644 index 00000000..cdc0bf06 --- /dev/null +++ b/lib/apscheduler/schedulers/base.py @@ -0,0 +1,845 @@ +from __future__ import print_function +from abc import ABCMeta, abstractmethod +from collections import MutableMapping +from threading import RLock +from datetime import datetime +from logging import getLogger +import sys + +from pkg_resources import iter_entry_points +from tzlocal import get_localzone +import six + +from apscheduler.schedulers import SchedulerAlreadyRunningError, SchedulerNotRunningError +from apscheduler.executors.base import MaxInstancesReachedError, BaseExecutor +from apscheduler.executors.pool import ThreadPoolExecutor +from apscheduler.jobstores.base import ConflictingIdError, JobLookupError, BaseJobStore +from apscheduler.jobstores.memory import MemoryJobStore +from apscheduler.job import Job +from apscheduler.triggers.base import BaseTrigger +from apscheduler.util import asbool, asint, astimezone, maybe_ref, timedelta_seconds, undefined +from apscheduler.events import ( + SchedulerEvent, JobEvent, EVENT_SCHEDULER_START, EVENT_SCHEDULER_SHUTDOWN, EVENT_JOBSTORE_ADDED, + EVENT_JOBSTORE_REMOVED, EVENT_ALL, EVENT_JOB_MODIFIED, EVENT_JOB_REMOVED, EVENT_JOB_ADDED, EVENT_EXECUTOR_ADDED, + EVENT_EXECUTOR_REMOVED, EVENT_ALL_JOBS_REMOVED) + + +class BaseScheduler(six.with_metaclass(ABCMeta)): + """ + Abstract base class for all schedulers. Takes the following keyword arguments: + + :param str|logging.Logger logger: logger to use for the scheduler's logging (defaults to apscheduler.scheduler) + :param str|datetime.tzinfo timezone: the default time zone (defaults to the local timezone) + :param dict job_defaults: default values for newly added jobs + :param dict jobstores: a dictionary of job store alias -> job store instance or configuration dict + :param dict executors: a dictionary of executor alias -> executor instance or configuration dict + + .. seealso:: :ref:`scheduler-config` + """ + + _trigger_plugins = dict((ep.name, ep) for ep in iter_entry_points('apscheduler.triggers')) + _trigger_classes = {} + _executor_plugins = dict((ep.name, ep) for ep in iter_entry_points('apscheduler.executors')) + _executor_classes = {} + _jobstore_plugins = dict((ep.name, ep) for ep in iter_entry_points('apscheduler.jobstores')) + _jobstore_classes = {} + _stopped = True + + # + # Public API + # + + def __init__(self, gconfig={}, **options): + super(BaseScheduler, self).__init__() + self._executors = {} + self._executors_lock = self._create_lock() + self._jobstores = {} + self._jobstores_lock = self._create_lock() + self._listeners = [] + self._listeners_lock = self._create_lock() + self._pending_jobs = [] + self.configure(gconfig, **options) + + def configure(self, gconfig={}, prefix='apscheduler.', **options): + """ + Reconfigures the scheduler with the given options. Can only be done when the scheduler isn't running. + + :param dict gconfig: a "global" configuration dictionary whose values can be overridden by keyword arguments to + this method + :param str|unicode prefix: pick only those keys from ``gconfig`` that are prefixed with this string + (pass an empty string or ``None`` to use all keys) + :raises SchedulerAlreadyRunningError: if the scheduler is already running + """ + + if self.running: + raise SchedulerAlreadyRunningError + + # If a non-empty prefix was given, strip it from the keys in the global configuration dict + if prefix: + prefixlen = len(prefix) + gconfig = dict((key[prefixlen:], value) for key, value in six.iteritems(gconfig) if key.startswith(prefix)) + + # Create a structure from the dotted options (e.g. "a.b.c = d" -> {'a': {'b': {'c': 'd'}}}) + config = {} + for key, value in six.iteritems(gconfig): + parts = key.split('.') + parent = config + key = parts.pop(0) + while parts: + parent = parent.setdefault(key, {}) + key = parts.pop(0) + parent[key] = value + + # Override any options with explicit keyword arguments + config.update(options) + self._configure(config) + + @abstractmethod + def start(self): + """ + Starts the scheduler. The details of this process depend on the implementation. + + :raises SchedulerAlreadyRunningError: if the scheduler is already running + """ + + if self.running: + raise SchedulerAlreadyRunningError + + with self._executors_lock: + # Create a default executor if nothing else is configured + if 'default' not in self._executors: + self.add_executor(self._create_default_executor(), 'default') + + # Start all the executors + for alias, executor in six.iteritems(self._executors): + executor.start(self, alias) + + with self._jobstores_lock: + # Create a default job store if nothing else is configured + if 'default' not in self._jobstores: + self.add_jobstore(self._create_default_jobstore(), 'default') + + # Start all the job stores + for alias, store in six.iteritems(self._jobstores): + store.start(self, alias) + + # Schedule all pending jobs + for job, jobstore_alias, replace_existing in self._pending_jobs: + self._real_add_job(job, jobstore_alias, replace_existing, False) + del self._pending_jobs[:] + + self._stopped = False + self._logger.info('Scheduler started') + + # Notify listeners that the scheduler has been started + self._dispatch_event(SchedulerEvent(EVENT_SCHEDULER_START)) + + @abstractmethod + def shutdown(self, wait=True): + """ + Shuts down the scheduler. Does not interrupt any currently running jobs. + + :param bool wait: ``True`` to wait until all currently executing jobs have finished + :raises SchedulerNotRunningError: if the scheduler has not been started yet + """ + + if not self.running: + raise SchedulerNotRunningError + + self._stopped = True + + # Shut down all executors + for executor in six.itervalues(self._executors): + executor.shutdown(wait) + + # Shut down all job stores + for jobstore in six.itervalues(self._jobstores): + jobstore.shutdown() + + self._logger.info('Scheduler has been shut down') + self._dispatch_event(SchedulerEvent(EVENT_SCHEDULER_SHUTDOWN)) + + @property + def running(self): + return not self._stopped + + def add_executor(self, executor, alias='default', **executor_opts): + """ + Adds an executor to this scheduler. Any extra keyword arguments will be passed to the executor plugin's + constructor, assuming that the first argument is the name of an executor plugin. + + :param str|unicode|apscheduler.executors.base.BaseExecutor executor: either an executor instance or the name of + an executor plugin + :param str|unicode alias: alias for the scheduler + :raises ValueError: if there is already an executor by the given alias + """ + + with self._executors_lock: + if alias in self._executors: + raise ValueError('This scheduler already has an executor by the alias of "%s"' % alias) + + if isinstance(executor, BaseExecutor): + self._executors[alias] = executor + elif isinstance(executor, six.string_types): + self._executors[alias] = executor = self._create_plugin_instance('executor', executor, executor_opts) + else: + raise TypeError('Expected an executor instance or a string, got %s instead' % + executor.__class__.__name__) + + # Start the executor right away if the scheduler is running + if self.running: + executor.start(self) + + self._dispatch_event(SchedulerEvent(EVENT_EXECUTOR_ADDED, alias)) + + def remove_executor(self, alias, shutdown=True): + """ + Removes the executor by the given alias from this scheduler. + + :param str|unicode alias: alias of the executor + :param bool shutdown: ``True`` to shut down the executor after removing it + """ + + with self._jobstores_lock: + executor = self._lookup_executor(alias) + del self._executors[alias] + + if shutdown: + executor.shutdown() + + self._dispatch_event(SchedulerEvent(EVENT_EXECUTOR_REMOVED, alias)) + + def add_jobstore(self, jobstore, alias='default', **jobstore_opts): + """ + Adds a job store to this scheduler. Any extra keyword arguments will be passed to the job store plugin's + constructor, assuming that the first argument is the name of a job store plugin. + + :param str|unicode|apscheduler.jobstores.base.BaseJobStore jobstore: job store to be added + :param str|unicode alias: alias for the job store + :raises ValueError: if there is already a job store by the given alias + """ + + with self._jobstores_lock: + if alias in self._jobstores: + raise ValueError('This scheduler already has a job store by the alias of "%s"' % alias) + + if isinstance(jobstore, BaseJobStore): + self._jobstores[alias] = jobstore + elif isinstance(jobstore, six.string_types): + self._jobstores[alias] = jobstore = self._create_plugin_instance('jobstore', jobstore, jobstore_opts) + else: + raise TypeError('Expected a job store instance or a string, got %s instead' % + jobstore.__class__.__name__) + + # Start the job store right away if the scheduler is running + if self.running: + jobstore.start(self, alias) + + # Notify listeners that a new job store has been added + self._dispatch_event(SchedulerEvent(EVENT_JOBSTORE_ADDED, alias)) + + # Notify the scheduler so it can scan the new job store for jobs + if self.running: + self.wakeup() + + def remove_jobstore(self, alias, shutdown=True): + """ + Removes the job store by the given alias from this scheduler. + + :param str|unicode alias: alias of the job store + :param bool shutdown: ``True`` to shut down the job store after removing it + """ + + with self._jobstores_lock: + jobstore = self._lookup_jobstore(alias) + del self._jobstores[alias] + + if shutdown: + jobstore.shutdown() + + self._dispatch_event(SchedulerEvent(EVENT_JOBSTORE_REMOVED, alias)) + + def add_listener(self, callback, mask=EVENT_ALL): + """ + add_listener(callback, mask=EVENT_ALL) + + Adds a listener for scheduler events. When a matching event occurs, ``callback`` is executed with the event + object as its sole argument. If the ``mask`` parameter is not provided, the callback will receive events of all + types. + + :param callback: any callable that takes one argument + :param int mask: bitmask that indicates which events should be listened to + + .. seealso:: :mod:`apscheduler.events` + .. seealso:: :ref:`scheduler-events` + """ + + with self._listeners_lock: + self._listeners.append((callback, mask)) + + def remove_listener(self, callback): + """Removes a previously added event listener.""" + + with self._listeners_lock: + for i, (cb, _) in enumerate(self._listeners): + if callback == cb: + del self._listeners[i] + + def add_job(self, func, trigger=None, args=None, kwargs=None, id=None, name=None, misfire_grace_time=undefined, + coalesce=undefined, max_instances=undefined, next_run_time=undefined, jobstore='default', + executor='default', replace_existing=False, **trigger_args): + """ + add_job(func, trigger=None, args=None, kwargs=None, id=None, name=None, misfire_grace_time=undefined, \ + coalesce=undefined, max_instances=undefined, next_run_time=undefined, jobstore='default', \ + executor='default', replace_existing=False, **trigger_args) + + Adds the given job to the job list and wakes up the scheduler if it's already running. + + Any option that defaults to ``undefined`` will be replaced with the corresponding default value when the job is + scheduled (which happens when the scheduler is started, or immediately if the scheduler is already running). + + The ``func`` argument can be given either as a callable object or a textual reference in the + ``package.module:some.object`` format, where the first half (separated by ``:``) is an importable module and the + second half is a reference to the callable object, relative to the module. + + The ``trigger`` argument can either be: + #. the alias name of the trigger (e.g. ``date``, ``interval`` or ``cron``), in which case any extra keyword + arguments to this method are passed on to the trigger's constructor + #. an instance of a trigger class + + :param func: callable (or a textual reference to one) to run at the given time + :param str|apscheduler.triggers.base.BaseTrigger trigger: trigger that determines when ``func`` is called + :param list|tuple args: list of positional arguments to call func with + :param dict kwargs: dict of keyword arguments to call func with + :param str|unicode id: explicit identifier for the job (for modifying it later) + :param str|unicode name: textual description of the job + :param int misfire_grace_time: seconds after the designated run time that the job is still allowed to be run + :param bool coalesce: run once instead of many times if the scheduler determines that the job should be run more + than once in succession + :param int max_instances: maximum number of concurrently running instances allowed for this job + :param datetime next_run_time: when to first run the job, regardless of the trigger (pass ``None`` to add the + job as paused) + :param str|unicode jobstore: alias of the job store to store the job in + :param str|unicode executor: alias of the executor to run the job with + :param bool replace_existing: ``True`` to replace an existing job with the same ``id`` (but retain the + number of runs from the existing one) + :rtype: Job + """ + + job_kwargs = { + 'trigger': self._create_trigger(trigger, trigger_args), + 'executor': executor, + 'func': func, + 'args': tuple(args) if args is not None else (), + 'kwargs': dict(kwargs) if kwargs is not None else {}, + 'id': id, + 'name': name, + 'misfire_grace_time': misfire_grace_time, + 'coalesce': coalesce, + 'max_instances': max_instances, + 'next_run_time': next_run_time + } + job_kwargs = dict((key, value) for key, value in six.iteritems(job_kwargs) if value is not undefined) + job = Job(self, **job_kwargs) + + # Don't really add jobs to job stores before the scheduler is up and running + with self._jobstores_lock: + if not self.running: + self._pending_jobs.append((job, jobstore, replace_existing)) + self._logger.info('Adding job tentatively -- it will be properly scheduled when the scheduler starts') + else: + self._real_add_job(job, jobstore, replace_existing, True) + + return job + + def scheduled_job(self, trigger, args=None, kwargs=None, id=None, name=None, misfire_grace_time=undefined, + coalesce=undefined, max_instances=undefined, next_run_time=undefined, jobstore='default', + executor='default', **trigger_args): + """ + scheduled_job(trigger, args=None, kwargs=None, id=None, name=None, misfire_grace_time=undefined, \ + coalesce=undefined, max_instances=undefined, next_run_time=undefined, jobstore='default', \ + executor='default',**trigger_args) + + A decorator version of :meth:`add_job`, except that ``replace_existing`` is always ``True``. + + .. important:: The ``id`` argument must be given if scheduling a job in a persistent job store. The scheduler + cannot, however, enforce this requirement. + """ + + def inner(func): + self.add_job(func, trigger, args, kwargs, id, name, misfire_grace_time, coalesce, max_instances, + next_run_time, jobstore, executor, True, **trigger_args) + return func + return inner + + def modify_job(self, job_id, jobstore=None, **changes): + """ + Modifies the properties of a single job. Modifications are passed to this method as extra keyword arguments. + + :param str|unicode job_id: the identifier of the job + :param str|unicode jobstore: alias of the job store that contains the job + """ + with self._jobstores_lock: + job, jobstore = self._lookup_job(job_id, jobstore) + job._modify(**changes) + if jobstore: + self._lookup_jobstore(jobstore).update_job(job) + + self._dispatch_event(JobEvent(EVENT_JOB_MODIFIED, job_id, jobstore)) + + # Wake up the scheduler since the job's next run time may have been changed + self.wakeup() + + def reschedule_job(self, job_id, jobstore=None, trigger=None, **trigger_args): + """ + Constructs a new trigger for a job and updates its next run time. + Extra keyword arguments are passed directly to the trigger's constructor. + + :param str|unicode job_id: the identifier of the job + :param str|unicode jobstore: alias of the job store that contains the job + :param trigger: alias of the trigger type or a trigger instance + """ + + trigger = self._create_trigger(trigger, trigger_args) + now = datetime.now(self.timezone) + next_run_time = trigger.get_next_fire_time(None, now) + self.modify_job(job_id, jobstore, trigger=trigger, next_run_time=next_run_time) + + def pause_job(self, job_id, jobstore=None): + """ + Causes the given job not to be executed until it is explicitly resumed. + + :param str|unicode job_id: the identifier of the job + :param str|unicode jobstore: alias of the job store that contains the job + """ + + self.modify_job(job_id, jobstore, next_run_time=None) + + def resume_job(self, job_id, jobstore=None): + """ + Resumes the schedule of the given job, or removes the job if its schedule is finished. + + :param str|unicode job_id: the identifier of the job + :param str|unicode jobstore: alias of the job store that contains the job + """ + + with self._jobstores_lock: + job, jobstore = self._lookup_job(job_id, jobstore) + now = datetime.now(self.timezone) + next_run_time = job.trigger.get_next_fire_time(None, now) + if next_run_time: + self.modify_job(job_id, jobstore, next_run_time=next_run_time) + else: + self.remove_job(job.id, jobstore) + + def get_jobs(self, jobstore=None, pending=None): + """ + Returns a list of pending jobs (if the scheduler hasn't been started yet) and scheduled jobs, either from a + specific job store or from all of them. + + :param str|unicode jobstore: alias of the job store + :param bool pending: ``False`` to leave out pending jobs (jobs that are waiting for the scheduler start to be + added to their respective job stores), ``True`` to only include pending jobs, anything else + to return both + :rtype: list[Job] + """ + + with self._jobstores_lock: + jobs = [] + + if pending is not False: + for job, alias, replace_existing in self._pending_jobs: + if jobstore is None or alias == jobstore: + jobs.append(job) + + if pending is not True: + for alias, store in six.iteritems(self._jobstores): + if jobstore is None or alias == jobstore: + jobs.extend(store.get_all_jobs()) + + return jobs + + def get_job(self, job_id, jobstore=None): + """ + Returns the Job that matches the given ``job_id``. + + :param str|unicode job_id: the identifier of the job + :param str|unicode jobstore: alias of the job store that most likely contains the job + :return: the Job by the given ID, or ``None`` if it wasn't found + :rtype: Job + """ + + with self._jobstores_lock: + try: + return self._lookup_job(job_id, jobstore)[0] + except JobLookupError: + return + + def remove_job(self, job_id, jobstore=None): + """ + Removes a job, preventing it from being run any more. + + :param str|unicode job_id: the identifier of the job + :param str|unicode jobstore: alias of the job store that contains the job + :raises JobLookupError: if the job was not found + """ + + with self._jobstores_lock: + # Check if the job is among the pending jobs + for i, (job, jobstore_alias, replace_existing) in enumerate(self._pending_jobs): + if job.id == job_id: + del self._pending_jobs[i] + jobstore = jobstore_alias + break + else: + # Otherwise, try to remove it from each store until it succeeds or we run out of stores to check + for alias, store in six.iteritems(self._jobstores): + if jobstore in (None, alias): + try: + store.remove_job(job_id) + except JobLookupError: + continue + + jobstore = alias + break + + if jobstore is None: + raise JobLookupError(job_id) + + # Notify listeners that a job has been removed + event = JobEvent(EVENT_JOB_REMOVED, job_id, jobstore) + self._dispatch_event(event) + + self._logger.info('Removed job %s', job_id) + + def remove_all_jobs(self, jobstore=None): + """ + Removes all jobs from the specified job store, or all job stores if none is given. + + :param str|unicode jobstore: alias of the job store + """ + + with self._jobstores_lock: + if jobstore: + self._pending_jobs = [pending for pending in self._pending_jobs if pending[1] != jobstore] + else: + self._pending_jobs = [] + + for alias, store in six.iteritems(self._jobstores): + if jobstore in (None, alias): + store.remove_all_jobs() + + self._dispatch_event(SchedulerEvent(EVENT_ALL_JOBS_REMOVED, jobstore)) + + def print_jobs(self, jobstore=None, out=None): + """ + print_jobs(jobstore=None, out=sys.stdout) + + Prints out a textual listing of all jobs currently scheduled on either all job stores or just a specific one. + + :param str|unicode jobstore: alias of the job store, ``None`` to list jobs from all stores + :param file out: a file-like object to print to (defaults to **sys.stdout** if nothing is given) + """ + + out = out or sys.stdout + with self._jobstores_lock: + if self._pending_jobs: + print(six.u('Pending jobs:'), file=out) + for job, jobstore_alias, replace_existing in self._pending_jobs: + if jobstore in (None, jobstore_alias): + print(six.u(' %s') % job, file=out) + + for alias, store in six.iteritems(self._jobstores): + if jobstore in (None, alias): + print(six.u('Jobstore %s:') % alias, file=out) + jobs = store.get_all_jobs() + if jobs: + for job in jobs: + print(six.u(' %s') % job, file=out) + else: + print(six.u(' No scheduled jobs'), file=out) + + @abstractmethod + def wakeup(self): + """ + Notifies the scheduler that there may be jobs due for execution. + Triggers :meth:`_process_jobs` to be run in an implementation specific manner. + """ + + # + # Private API + # + + def _configure(self, config): + # Set general options + self._logger = maybe_ref(config.pop('logger', None)) or getLogger('apscheduler.scheduler') + self.timezone = astimezone(config.pop('timezone', None)) or get_localzone() + + # Set the job defaults + job_defaults = config.get('job_defaults', {}) + self._job_defaults = { + 'misfire_grace_time': asint(job_defaults.get('misfire_grace_time', 1)), + 'coalesce': asbool(job_defaults.get('coalesce', True)), + 'max_instances': asint(job_defaults.get('max_instances', 1)) + } + + # Configure executors + self._executors.clear() + for alias, value in six.iteritems(config.get('executors', {})): + if isinstance(value, BaseExecutor): + self.add_executor(value, alias) + elif isinstance(value, MutableMapping): + executor_class = value.pop('class', None) + plugin = value.pop('type', None) + if plugin: + executor = self._create_plugin_instance('executor', plugin, value) + elif executor_class: + cls = maybe_ref(executor_class) + executor = cls(**value) + else: + raise ValueError('Cannot create executor "%s" -- either "type" or "class" must be defined' % alias) + + self.add_executor(executor, alias) + else: + raise TypeError("Expected executor instance or dict for executors['%s'], got %s instead" % ( + alias, value.__class__.__name__)) + + # Configure job stores + self._jobstores.clear() + for alias, value in six.iteritems(config.get('jobstores', {})): + if isinstance(value, BaseJobStore): + self.add_jobstore(value, alias) + elif isinstance(value, MutableMapping): + jobstore_class = value.pop('class', None) + plugin = value.pop('type', None) + if plugin: + jobstore = self._create_plugin_instance('jobstore', plugin, value) + elif jobstore_class: + cls = maybe_ref(jobstore_class) + jobstore = cls(**value) + else: + raise ValueError('Cannot create job store "%s" -- either "type" or "class" must be defined' % alias) + + self.add_jobstore(jobstore, alias) + else: + raise TypeError("Expected job store instance or dict for jobstores['%s'], got %s instead" % ( + alias, value.__class__.__name__)) + + def _create_default_executor(self): + """Creates a default executor store, specific to the particular scheduler type.""" + + return ThreadPoolExecutor() + + def _create_default_jobstore(self): + """Creates a default job store, specific to the particular scheduler type.""" + + return MemoryJobStore() + + def _lookup_executor(self, alias): + """ + Returns the executor instance by the given name from the list of executors that were added to this scheduler. + + :type alias: str + :raises KeyError: if no executor by the given alias is not found + """ + + try: + return self._executors[alias] + except KeyError: + raise KeyError('No such executor: %s' % alias) + + def _lookup_jobstore(self, alias): + """ + Returns the job store instance by the given name from the list of job stores that were added to this scheduler. + + :type alias: str + :raises KeyError: if no job store by the given alias is not found + """ + + try: + return self._jobstores[alias] + except KeyError: + raise KeyError('No such job store: %s' % alias) + + def _lookup_job(self, job_id, jobstore_alias): + """ + Finds a job by its ID. + + :type job_id: str + :param str jobstore_alias: alias of a job store to look in + :return tuple[Job, str]: a tuple of job, jobstore alias (jobstore alias is None in case of a pending job) + :raises JobLookupError: if no job by the given ID is found. + """ + + # Check if the job is among the pending jobs + for job, alias, replace_existing in self._pending_jobs: + if job.id == job_id: + return job, None + + # Look in all job stores + for alias, store in six.iteritems(self._jobstores): + if jobstore_alias in (None, alias): + job = store.lookup_job(job_id) + if job is not None: + return job, alias + + raise JobLookupError(job_id) + + def _dispatch_event(self, event): + """ + Dispatches the given event to interested listeners. + + :param SchedulerEvent event: the event to send + """ + + with self._listeners_lock: + listeners = tuple(self._listeners) + + for cb, mask in listeners: + if event.code & mask: + try: + cb(event) + except: + self._logger.exception('Error notifying listener') + + def _real_add_job(self, job, jobstore_alias, replace_existing, wakeup): + """ + :param Job job: the job to add + :param bool replace_existing: ``True`` to use update_job() in case the job already exists in the store + :param bool wakeup: ``True`` to wake up the scheduler after adding the job + """ + + # Fill in undefined values with defaults + replacements = {} + for key, value in six.iteritems(self._job_defaults): + if not hasattr(job, key): + replacements[key] = value + + # Calculate the next run time if there is none defined + if not hasattr(job, 'next_run_time'): + now = datetime.now(self.timezone) + replacements['next_run_time'] = job.trigger.get_next_fire_time(None, now) + + # Apply any replacements + job._modify(**replacements) + + # Add the job to the given job store + store = self._lookup_jobstore(jobstore_alias) + try: + store.add_job(job) + except ConflictingIdError: + if replace_existing: + store.update_job(job) + else: + raise + + # Mark the job as no longer pending + job._jobstore_alias = jobstore_alias + + # Notify listeners that a new job has been added + event = JobEvent(EVENT_JOB_ADDED, job.id, jobstore_alias) + self._dispatch_event(event) + + self._logger.info('Added job "%s" to job store "%s"', job.name, jobstore_alias) + + # Notify the scheduler about the new job + if wakeup: + self.wakeup() + + def _create_plugin_instance(self, type_, alias, constructor_kwargs): + """Creates an instance of the given plugin type, loading the plugin first if necessary.""" + + plugin_container, class_container, base_class = { + 'trigger': (self._trigger_plugins, self._trigger_classes, BaseTrigger), + 'jobstore': (self._jobstore_plugins, self._jobstore_classes, BaseJobStore), + 'executor': (self._executor_plugins, self._executor_classes, BaseExecutor) + }[type_] + + try: + plugin_cls = class_container[alias] + except KeyError: + if alias in plugin_container: + plugin_cls = class_container[alias] = plugin_container[alias].load() + if not issubclass(plugin_cls, base_class): + raise TypeError('The {0} entry point does not point to a {0} class'.format(type_)) + else: + raise LookupError('No {0} by the name "{1}" was found'.format(type_, alias)) + + return plugin_cls(**constructor_kwargs) + + def _create_trigger(self, trigger, trigger_args): + if isinstance(trigger, BaseTrigger): + return trigger + elif trigger is None: + trigger = 'date' + elif not isinstance(trigger, six.string_types): + raise TypeError('Expected a trigger instance or string, got %s instead' % trigger.__class__.__name__) + + # Use the scheduler's time zone if nothing else is specified + trigger_args.setdefault('timezone', self.timezone) + + # Instantiate the trigger class + return self._create_plugin_instance('trigger', trigger, trigger_args) + + def _create_lock(self): + """Creates a reentrant lock object.""" + + return RLock() + + def _process_jobs(self): + """ + Iterates through jobs in every jobstore, starts jobs that are due and figures out how long to wait for the next + round. + """ + + self._logger.debug('Looking for jobs to run') + now = datetime.now(self.timezone) + next_wakeup_time = None + + with self._jobstores_lock: + for jobstore_alias, jobstore in six.iteritems(self._jobstores): + for job in jobstore.get_due_jobs(now): + # Look up the job's executor + try: + executor = self._lookup_executor(job.executor) + except: + self._logger.error( + 'Executor lookup ("%s") failed for job "%s" -- removing it from the job store', + job.executor, job) + self.remove_job(job.id, jobstore_alias) + continue + + run_times = job._get_run_times(now) + run_times = run_times[-1:] if run_times and job.coalesce else run_times + if run_times: + try: + executor.submit_job(job, run_times) + except MaxInstancesReachedError: + self._logger.warning( + 'Execution of job "%s" skipped: maximum number of running instances reached (%d)', + job, job.max_instances) + except: + self._logger.exception('Error submitting job "%s" to executor "%s"', job, job.executor) + + # Update the job if it has a next execution time. Otherwise remove it from the job store. + job_next_run = job.trigger.get_next_fire_time(run_times[-1], now) + if job_next_run: + job._modify(next_run_time=job_next_run) + jobstore.update_job(job) + else: + self.remove_job(job.id, jobstore_alias) + + # Set a new next wakeup time if there isn't one yet or the jobstore has an even earlier one + jobstore_next_run_time = jobstore.get_next_run_time() + if jobstore_next_run_time and (next_wakeup_time is None or jobstore_next_run_time < next_wakeup_time): + next_wakeup_time = jobstore_next_run_time + + # Determine the delay until this method should be called again + if next_wakeup_time is not None: + wait_seconds = max(timedelta_seconds(next_wakeup_time - now), 0) + self._logger.debug('Next wakeup is due at %s (in %f seconds)', next_wakeup_time, wait_seconds) + else: + wait_seconds = None + self._logger.debug('No jobs; waiting until a job is added') + + return wait_seconds diff --git a/lib/apscheduler/schedulers/blocking.py b/lib/apscheduler/schedulers/blocking.py new file mode 100644 index 00000000..2720822c --- /dev/null +++ b/lib/apscheduler/schedulers/blocking.py @@ -0,0 +1,32 @@ +from __future__ import absolute_import +from threading import Event + +from apscheduler.schedulers.base import BaseScheduler + + +class BlockingScheduler(BaseScheduler): + """ + A scheduler that runs in the foreground (:meth:`~apscheduler.schedulers.base.BaseScheduler.start` will block). + """ + + MAX_WAIT_TIME = 4294967 # Maximum value accepted by Event.wait() on Windows + + _event = None + + def start(self): + super(BlockingScheduler, self).start() + self._event = Event() + self._main_loop() + + def shutdown(self, wait=True): + super(BlockingScheduler, self).shutdown(wait) + self._event.set() + + def _main_loop(self): + while self.running: + wait_seconds = self._process_jobs() + self._event.wait(wait_seconds if wait_seconds is not None else self.MAX_WAIT_TIME) + self._event.clear() + + def wakeup(self): + self._event.set() diff --git a/lib/apscheduler/schedulers/gevent.py b/lib/apscheduler/schedulers/gevent.py new file mode 100644 index 00000000..9cce6589 --- /dev/null +++ b/lib/apscheduler/schedulers/gevent.py @@ -0,0 +1,35 @@ +from __future__ import absolute_import + +from apscheduler.schedulers.blocking import BlockingScheduler +from apscheduler.schedulers.base import BaseScheduler + +try: + from gevent.event import Event + from gevent.lock import RLock + import gevent +except ImportError: # pragma: nocover + raise ImportError('GeventScheduler requires gevent installed') + + +class GeventScheduler(BlockingScheduler): + """A scheduler that runs as a Gevent greenlet.""" + + _greenlet = None + + def start(self): + BaseScheduler.start(self) + self._event = Event() + self._greenlet = gevent.spawn(self._main_loop) + return self._greenlet + + def shutdown(self, wait=True): + super(GeventScheduler, self).shutdown(wait) + self._greenlet.join() + del self._greenlet + + def _create_lock(self): + return RLock() + + def _create_default_executor(self): + from apscheduler.executors.gevent import GeventExecutor + return GeventExecutor() diff --git a/lib/apscheduler/schedulers/qt.py b/lib/apscheduler/schedulers/qt.py new file mode 100644 index 00000000..dde5afaa --- /dev/null +++ b/lib/apscheduler/schedulers/qt.py @@ -0,0 +1,46 @@ +from __future__ import absolute_import + +from apscheduler.schedulers.base import BaseScheduler + +try: + from PyQt5.QtCore import QObject, QTimer +except ImportError: # pragma: nocover + try: + from PyQt4.QtCore import QObject, QTimer + except ImportError: + try: + from PySide.QtCore import QObject, QTimer # flake8: noqa + except ImportError: + raise ImportError('QtScheduler requires either PyQt5, PyQt4 or PySide installed') + + +class QtScheduler(BaseScheduler): + """A scheduler that runs in a Qt event loop.""" + + _timer = None + + def start(self): + super(QtScheduler, self).start() + self.wakeup() + + def shutdown(self, wait=True): + super(QtScheduler, self).shutdown(wait) + self._stop_timer() + + def _start_timer(self, wait_seconds): + self._stop_timer() + if wait_seconds is not None: + self._timer = QTimer.singleShot(wait_seconds * 1000, self._process_jobs) + + def _stop_timer(self): + if self._timer: + if self._timer.isActive(): + self._timer.stop() + del self._timer + + def wakeup(self): + self._start_timer(0) + + def _process_jobs(self): + wait_seconds = super(QtScheduler, self)._process_jobs() + self._start_timer(wait_seconds) diff --git a/lib/apscheduler/schedulers/tornado.py b/lib/apscheduler/schedulers/tornado.py new file mode 100644 index 00000000..78093308 --- /dev/null +++ b/lib/apscheduler/schedulers/tornado.py @@ -0,0 +1,60 @@ +from __future__ import absolute_import +from datetime import timedelta +from functools import wraps + +from apscheduler.schedulers.base import BaseScheduler +from apscheduler.util import maybe_ref + +try: + from tornado.ioloop import IOLoop +except ImportError: # pragma: nocover + raise ImportError('TornadoScheduler requires tornado installed') + + +def run_in_ioloop(func): + @wraps(func) + def wrapper(self, *args, **kwargs): + self._ioloop.add_callback(func, self, *args, **kwargs) + return wrapper + + +class TornadoScheduler(BaseScheduler): + """ + A scheduler that runs on a Tornado IOLoop. + + =========== =============================================================== + ``io_loop`` Tornado IOLoop instance to use (defaults to the global IO loop) + =========== =============================================================== + """ + + _ioloop = None + _timeout = None + + def start(self): + super(TornadoScheduler, self).start() + self.wakeup() + + @run_in_ioloop + def shutdown(self, wait=True): + super(TornadoScheduler, self).shutdown(wait) + self._stop_timer() + + def _configure(self, config): + self._ioloop = maybe_ref(config.pop('io_loop', None)) or IOLoop.current() + super(TornadoScheduler, self)._configure(config) + + def _start_timer(self, wait_seconds): + self._stop_timer() + if wait_seconds is not None: + self._timeout = self._ioloop.add_timeout(timedelta(seconds=wait_seconds), self.wakeup) + + def _stop_timer(self): + if self._timeout: + self._ioloop.remove_timeout(self._timeout) + del self._timeout + + @run_in_ioloop + def wakeup(self): + self._stop_timer() + wait_seconds = self._process_jobs() + self._start_timer(wait_seconds) diff --git a/lib/apscheduler/schedulers/twisted.py b/lib/apscheduler/schedulers/twisted.py new file mode 100644 index 00000000..166b6130 --- /dev/null +++ b/lib/apscheduler/schedulers/twisted.py @@ -0,0 +1,65 @@ +from __future__ import absolute_import +from functools import wraps + +from apscheduler.schedulers.base import BaseScheduler +from apscheduler.util import maybe_ref + +try: + from twisted.internet import reactor as default_reactor +except ImportError: # pragma: nocover + raise ImportError('TwistedScheduler requires Twisted installed') + + +def run_in_reactor(func): + @wraps(func) + def wrapper(self, *args, **kwargs): + self._reactor.callFromThread(func, self, *args, **kwargs) + return wrapper + + +class TwistedScheduler(BaseScheduler): + """ + A scheduler that runs on a Twisted reactor. + + Extra options: + + =========== ======================================================== + ``reactor`` Reactor instance to use (defaults to the global reactor) + =========== ======================================================== + """ + + _reactor = None + _delayedcall = None + + def _configure(self, config): + self._reactor = maybe_ref(config.pop('reactor', default_reactor)) + super(TwistedScheduler, self)._configure(config) + + def start(self): + super(TwistedScheduler, self).start() + self.wakeup() + + @run_in_reactor + def shutdown(self, wait=True): + super(TwistedScheduler, self).shutdown(wait) + self._stop_timer() + + def _start_timer(self, wait_seconds): + self._stop_timer() + if wait_seconds is not None: + self._delayedcall = self._reactor.callLater(wait_seconds, self.wakeup) + + def _stop_timer(self): + if self._delayedcall and self._delayedcall.active(): + self._delayedcall.cancel() + del self._delayedcall + + @run_in_reactor + def wakeup(self): + self._stop_timer() + wait_seconds = self._process_jobs() + self._start_timer(wait_seconds) + + def _create_default_executor(self): + from apscheduler.executors.twisted import TwistedExecutor + return TwistedExecutor() diff --git a/lib/apscheduler/threadpool.py b/lib/apscheduler/threadpool.py deleted file mode 100644 index 8ec47da0..00000000 --- a/lib/apscheduler/threadpool.py +++ /dev/null @@ -1,133 +0,0 @@ -""" -Generic thread pool class. Modeled after Java's ThreadPoolExecutor. -Please note that this ThreadPool does *not* fully implement the PEP 3148 -ThreadPool! -""" - -from threading import Thread, Lock, currentThread -from weakref import ref -import logging -import atexit - -try: - from queue import Queue, Empty -except ImportError: - from Queue import Queue, Empty - -logger = logging.getLogger(__name__) -_threadpools = set() - - -# Worker threads are daemonic in order to let the interpreter exit without -# an explicit shutdown of the thread pool. The following trick is necessary -# to allow worker threads to finish cleanly. -def _shutdown_all(): - for pool_ref in tuple(_threadpools): - pool = pool_ref() - if pool: - pool.shutdown() - -atexit.register(_shutdown_all) - - -class ThreadPool(object): - def __init__(self, core_threads=0, max_threads=20, keepalive=1): - """ - :param core_threads: maximum number of persistent threads in the pool - :param max_threads: maximum number of total threads in the pool - :param thread_class: callable that creates a Thread object - :param keepalive: seconds to keep non-core worker threads waiting - for new tasks - """ - self.core_threads = core_threads - self.max_threads = max(max_threads, core_threads, 1) - self.keepalive = keepalive - self._queue = Queue() - self._threads_lock = Lock() - self._threads = set() - self._shutdown = False - - _threadpools.add(ref(self)) - logger.info('Started thread pool with %d core threads and %s maximum ' - 'threads', core_threads, max_threads or 'unlimited') - - def _adjust_threadcount(self): - self._threads_lock.acquire() - try: - if self.num_threads < self.max_threads: - self._add_thread(self.num_threads < self.core_threads) - finally: - self._threads_lock.release() - - def _add_thread(self, core): - t = Thread(target=self._run_jobs, args=(core,)) - t.setDaemon(True) - t.start() - self._threads.add(t) - - def _run_jobs(self, core): - logger.debug('Started worker thread') - block = True - timeout = None - if not core: - block = self.keepalive > 0 - timeout = self.keepalive - - while True: - try: - func, args, kwargs = self._queue.get(block, timeout) - except Empty: - break - - if self._shutdown: - break - - try: - func(*args, **kwargs) - except: - logger.exception('Error in worker thread') - - self._threads_lock.acquire() - self._threads.remove(currentThread()) - self._threads_lock.release() - - logger.debug('Exiting worker thread') - - @property - def num_threads(self): - return len(self._threads) - - def submit(self, func, *args, **kwargs): - if self._shutdown: - raise RuntimeError('Cannot schedule new tasks after shutdown') - - self._queue.put((func, args, kwargs)) - self._adjust_threadcount() - - def shutdown(self, wait=True): - if self._shutdown: - return - - logging.info('Shutting down thread pool') - self._shutdown = True - _threadpools.remove(ref(self)) - - self._threads_lock.acquire() - for _ in range(self.num_threads): - self._queue.put((None, None, None)) - self._threads_lock.release() - - if wait: - self._threads_lock.acquire() - threads = tuple(self._threads) - self._threads_lock.release() - for thread in threads: - thread.join() - - def __repr__(self): - if self.max_threads: - threadcount = '%d/%d' % (self.num_threads, self.max_threads) - else: - threadcount = '%d' % self.num_threads - - return '' % (id(self), threadcount) diff --git a/lib/apscheduler/triggers/__init__.py b/lib/apscheduler/triggers/__init__.py index 74a97884..e69de29b 100644 --- a/lib/apscheduler/triggers/__init__.py +++ b/lib/apscheduler/triggers/__init__.py @@ -1,3 +0,0 @@ -from apscheduler.triggers.cron import CronTrigger -from apscheduler.triggers.interval import IntervalTrigger -from apscheduler.triggers.simple import SimpleTrigger diff --git a/lib/apscheduler/triggers/base.py b/lib/apscheduler/triggers/base.py new file mode 100644 index 00000000..3520d316 --- /dev/null +++ b/lib/apscheduler/triggers/base.py @@ -0,0 +1,16 @@ +from abc import ABCMeta, abstractmethod + +import six + + +class BaseTrigger(six.with_metaclass(ABCMeta)): + """Abstract base class that defines the interface that every trigger must implement.""" + + @abstractmethod + def get_next_fire_time(self, previous_fire_time, now): + """ + Returns the next datetime to fire on, If no such datetime can be calculated, returns ``None``. + + :param datetime.datetime previous_fire_time: the previous time the trigger was fired + :param datetime.datetime now: current datetime + """ diff --git a/lib/apscheduler/triggers/cron/__init__.py b/lib/apscheduler/triggers/cron/__init__.py index 3f8d9a8f..8df901ea 100644 --- a/lib/apscheduler/triggers/cron/__init__.py +++ b/lib/apscheduler/triggers/cron/__init__.py @@ -1,32 +1,71 @@ -from datetime import date, datetime +from datetime import datetime, timedelta -from apscheduler.triggers.cron.fields import * -from apscheduler.util import datetime_ceil, convert_to_datetime +from tzlocal import get_localzone +import six + +from apscheduler.triggers.base import BaseTrigger +from apscheduler.triggers.cron.fields import BaseField, WeekField, DayOfMonthField, DayOfWeekField, DEFAULT_VALUES +from apscheduler.util import datetime_ceil, convert_to_datetime, datetime_repr, astimezone -class CronTrigger(object): - FIELD_NAMES = ('year', 'month', 'day', 'week', 'day_of_week', 'hour', - 'minute', 'second') - FIELDS_MAP = {'year': BaseField, - 'month': BaseField, - 'week': WeekField, - 'day': DayOfMonthField, - 'day_of_week': DayOfWeekField, - 'hour': BaseField, - 'minute': BaseField, - 'second': BaseField} +class CronTrigger(BaseTrigger): + """ + Triggers when current time matches all specified time constraints, similarly to how the UNIX cron scheduler works. - def __init__(self, **values): - self.start_date = values.pop('start_date', None) - if self.start_date: - self.start_date = convert_to_datetime(self.start_date) + :param int|str year: 4-digit year + :param int|str month: month (1-12) + :param int|str day: day of the (1-31) + :param int|str week: ISO week (1-53) + :param int|str day_of_week: number or name of weekday (0-6 or mon,tue,wed,thu,fri,sat,sun) + :param int|str hour: hour (0-23) + :param int|str minute: minute (0-59) + :param int|str second: second (0-59) + :param datetime|str start_date: earliest possible date/time to trigger on (inclusive) + :param datetime|str end_date: latest possible date/time to trigger on (inclusive) + :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations + (defaults to scheduler timezone) + .. note:: The first weekday is always **monday**. + """ + + FIELD_NAMES = ('year', 'month', 'day', 'week', 'day_of_week', 'hour', 'minute', 'second') + FIELDS_MAP = { + 'year': BaseField, + 'month': BaseField, + 'week': WeekField, + 'day': DayOfMonthField, + 'day_of_week': DayOfWeekField, + 'hour': BaseField, + 'minute': BaseField, + 'second': BaseField + } + + __slots__ = 'timezone', 'start_date', 'end_date', 'fields' + + def __init__(self, year=None, month=None, day=None, week=None, day_of_week=None, hour=None, minute=None, + second=None, start_date=None, end_date=None, timezone=None): + if timezone: + self.timezone = astimezone(timezone) + elif start_date and start_date.tzinfo: + self.timezone = start_date.tzinfo + elif end_date and end_date.tzinfo: + self.timezone = end_date.tzinfo + else: + self.timezone = get_localzone() + + self.start_date = convert_to_datetime(start_date, self.timezone, 'start_date') + self.end_date = convert_to_datetime(end_date, self.timezone, 'end_date') + + values = dict((key, value) for (key, value) in six.iteritems(locals()) + if key in self.FIELD_NAMES and value is not None) self.fields = [] + assign_defaults = False for field_name in self.FIELD_NAMES: if field_name in values: exprs = values.pop(field_name) is_default = False - elif not values: + assign_defaults = not values + elif assign_defaults: exprs = DEFAULT_VALUES[field_name] is_default = True else: @@ -39,18 +78,16 @@ class CronTrigger(object): def _increment_field_value(self, dateval, fieldnum): """ - Increments the designated field and resets all less significant fields - to their minimum values. + Increments the designated field and resets all less significant fields to their minimum values. :type dateval: datetime :type fieldnum: int - :type amount: int + :return: a tuple containing the new date, and the number of the field that was actually incremented :rtype: tuple - :return: a tuple containing the new date, and the number of the field - that was actually incremented """ - i = 0 + values = {} + i = 0 while i < len(self.fields): field = self.fields[i] if not field.REAL: @@ -77,7 +114,8 @@ class CronTrigger(object): values[field.name] = value + 1 i += 1 - return datetime(**values), fieldnum + difference = datetime(**values) - dateval.replace(tzinfo=None) + return self.timezone.normalize(dateval + difference), fieldnum def _set_field_value(self, dateval, fieldnum, new_value): values = {} @@ -90,13 +128,17 @@ class CronTrigger(object): else: values[field.name] = new_value - return datetime(**values) + difference = datetime(**values) - dateval.replace(tzinfo=None) + return self.timezone.normalize(dateval + difference) + + def get_next_fire_time(self, previous_fire_time, now): + if previous_fire_time: + start_date = max(now, previous_fire_time + timedelta(microseconds=1)) + else: + start_date = max(now, self.start_date) if self.start_date else now - def get_next_fire_time(self, start_date): - if self.start_date: - start_date = max(start_date, self.start_date) - next_date = datetime_ceil(start_date) fieldnum = 0 + next_date = datetime_ceil(start_date).astimezone(self.timezone) while 0 <= fieldnum < len(self.fields): field = self.fields[fieldnum] curr_value = field.get_value(next_date) @@ -104,32 +146,31 @@ class CronTrigger(object): if next_value is None: # No valid value was found - next_date, fieldnum = self._increment_field_value(next_date, - fieldnum - 1) + next_date, fieldnum = self._increment_field_value(next_date, fieldnum - 1) elif next_value > curr_value: # A valid, but higher than the starting value, was found if field.REAL: - next_date = self._set_field_value(next_date, fieldnum, - next_value) + next_date = self._set_field_value(next_date, fieldnum, next_value) fieldnum += 1 else: - next_date, fieldnum = self._increment_field_value(next_date, - fieldnum) + next_date, fieldnum = self._increment_field_value(next_date, fieldnum) else: # A valid value was found, no changes necessary fieldnum += 1 + # Return if the date has rolled past the end date + if self.end_date and next_date > self.end_date: + return None + if fieldnum >= 0: return next_date def __str__(self): - options = ["%s='%s'" % (f.name, str(f)) for f in self.fields - if not f.is_default] + options = ["%s='%s'" % (f.name, f) for f in self.fields if not f.is_default] return 'cron[%s]' % (', '.join(options)) def __repr__(self): - options = ["%s='%s'" % (f.name, str(f)) for f in self.fields - if not f.is_default] + options = ["%s='%s'" % (f.name, f) for f in self.fields if not f.is_default] if self.start_date: - options.append("start_date='%s'" % self.start_date.isoformat(' ')) + options.append("start_date='%s'" % datetime_repr(self.start_date)) return '<%s (%s)>' % (self.__class__.__name__, ', '.join(options)) diff --git a/lib/apscheduler/triggers/cron/expressions.py b/lib/apscheduler/triggers/cron/expressions.py index 018c7a30..55272db4 100644 --- a/lib/apscheduler/triggers/cron/expressions.py +++ b/lib/apscheduler/triggers/cron/expressions.py @@ -7,8 +7,8 @@ import re from apscheduler.util import asint -__all__ = ('AllExpression', 'RangeExpression', 'WeekdayRangeExpression', - 'WeekdayPositionExpression') +__all__ = ('AllExpression', 'RangeExpression', 'WeekdayRangeExpression', 'WeekdayPositionExpression', + 'LastDayOfMonthExpression') WEEKDAYS = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'] @@ -57,8 +57,7 @@ class RangeExpression(AllExpression): if last is None and step is None: last = first if last is not None and first > last: - raise ValueError('The minimum value in a range must not be ' - 'higher than the maximum') + raise ValueError('The minimum value in a range must not be higher than the maximum') self.first = first self.last = last @@ -102,8 +101,7 @@ class RangeExpression(AllExpression): class WeekdayRangeExpression(RangeExpression): - value_re = re.compile(r'(?P[a-z]+)(?:-(?P[a-z]+))?', - re.IGNORECASE) + value_re = re.compile(r'(?P[a-z]+)(?:-(?P[a-z]+))?', re.IGNORECASE) def __init__(self, first, last=None): try: @@ -135,8 +133,7 @@ class WeekdayRangeExpression(RangeExpression): class WeekdayPositionExpression(AllExpression): options = ['1st', '2nd', '3rd', '4th', '5th', 'last'] - value_re = re.compile(r'(?P%s) +(?P(?:\d+|\w+))' - % '|'.join(options), re.IGNORECASE) + value_re = re.compile(r'(?P%s) +(?P(?:\d+|\w+))' % '|'.join(options), re.IGNORECASE) def __init__(self, option_name, weekday_name): try: @@ -169,10 +166,23 @@ class WeekdayPositionExpression(AllExpression): return target_day def __str__(self): - return '%s %s' % (self.options[self.option_num], - WEEKDAYS[self.weekday]) + return '%s %s' % (self.options[self.option_num], WEEKDAYS[self.weekday]) def __repr__(self): - return "%s('%s', '%s')" % (self.__class__.__name__, - self.options[self.option_num], - WEEKDAYS[self.weekday]) + return "%s('%s', '%s')" % (self.__class__.__name__, self.options[self.option_num], WEEKDAYS[self.weekday]) + + +class LastDayOfMonthExpression(AllExpression): + value_re = re.compile(r'last', re.IGNORECASE) + + def __init__(self): + pass + + def get_next_value(self, date, field): + return monthrange(date.year, date.month)[1] + + def __str__(self): + return 'last' + + def __repr__(self): + return "%s()" % self.__class__.__name__ diff --git a/lib/apscheduler/triggers/cron/fields.py b/lib/apscheduler/triggers/cron/fields.py index ef970cc9..e220599f 100644 --- a/lib/apscheduler/triggers/cron/fields.py +++ b/lib/apscheduler/triggers/cron/fields.py @@ -5,18 +5,18 @@ fields. from calendar import monthrange -from apscheduler.triggers.cron.expressions import * - -__all__ = ('MIN_VALUES', 'MAX_VALUES', 'DEFAULT_VALUES', 'BaseField', - 'WeekField', 'DayOfMonthField', 'DayOfWeekField') +from apscheduler.triggers.cron.expressions import ( + AllExpression, RangeExpression, WeekdayPositionExpression, LastDayOfMonthExpression, WeekdayRangeExpression) -MIN_VALUES = {'year': 1970, 'month': 1, 'day': 1, 'week': 1, - 'day_of_week': 0, 'hour': 0, 'minute': 0, 'second': 0} -MAX_VALUES = {'year': 2 ** 63, 'month': 12, 'day:': 31, 'week': 53, - 'day_of_week': 6, 'hour': 23, 'minute': 59, 'second': 59} -DEFAULT_VALUES = {'year': '*', 'month': 1, 'day': 1, 'week': '*', - 'day_of_week': '*', 'hour': 0, 'minute': 0, 'second': 0} +__all__ = ('MIN_VALUES', 'MAX_VALUES', 'DEFAULT_VALUES', 'BaseField', 'WeekField', 'DayOfMonthField', 'DayOfWeekField') + + +MIN_VALUES = {'year': 1970, 'month': 1, 'day': 1, 'week': 1, 'day_of_week': 0, 'hour': 0, 'minute': 0, 'second': 0} +MAX_VALUES = {'year': 2 ** 63, 'month': 12, 'day:': 31, 'week': 53, 'day_of_week': 6, 'hour': 23, 'minute': 59, + 'second': 59} +DEFAULT_VALUES = {'year': '*', 'month': 1, 'day': 1, 'week': '*', 'day_of_week': '*', 'hour': 0, 'minute': 0, + 'second': 0} class BaseField(object): @@ -65,16 +65,14 @@ class BaseField(object): self.expressions.append(compiled_expr) return - raise ValueError('Unrecognized expression "%s" for field "%s"' % - (expr, self.name)) + raise ValueError('Unrecognized expression "%s" for field "%s"' % (expr, self.name)) def __str__(self): expr_strings = (str(e) for e in self.expressions) return ','.join(expr_strings) def __repr__(self): - return "%s('%s', '%s')" % (self.__class__.__name__, self.name, - str(self)) + return "%s('%s', '%s')" % (self.__class__.__name__, self.name, self) class WeekField(BaseField): @@ -85,7 +83,7 @@ class WeekField(BaseField): class DayOfMonthField(BaseField): - COMPILERS = BaseField.COMPILERS + [WeekdayPositionExpression] + COMPILERS = BaseField.COMPILERS + [WeekdayPositionExpression, LastDayOfMonthExpression] def get_max(self, dateval): return monthrange(dateval.year, dateval.month)[1] diff --git a/lib/apscheduler/triggers/date.py b/lib/apscheduler/triggers/date.py new file mode 100644 index 00000000..237e6b4e --- /dev/null +++ b/lib/apscheduler/triggers/date.py @@ -0,0 +1,30 @@ +from datetime import datetime + +from tzlocal import get_localzone + +from apscheduler.triggers.base import BaseTrigger +from apscheduler.util import convert_to_datetime, datetime_repr, astimezone + + +class DateTrigger(BaseTrigger): + """ + Triggers once on the given datetime. If ``run_date`` is left empty, current time is used. + + :param datetime|str run_date: the date/time to run the job at + :param datetime.tzinfo|str timezone: time zone for ``run_date`` if it doesn't have one already + """ + + __slots__ = 'timezone', 'run_date' + + def __init__(self, run_date=None, timezone=None): + timezone = astimezone(timezone) or get_localzone() + self.run_date = convert_to_datetime(run_date or datetime.now(), timezone, 'run_date') + + def get_next_fire_time(self, previous_fire_time, now): + return self.run_date if previous_fire_time is None else None + + def __str__(self): + return 'date[%s]' % datetime_repr(self.run_date) + + def __repr__(self): + return "<%s (run_date='%s')>" % (self.__class__.__name__, datetime_repr(self.run_date)) diff --git a/lib/apscheduler/triggers/interval.py b/lib/apscheduler/triggers/interval.py index dd16d777..df9e6fe7 100644 --- a/lib/apscheduler/triggers/interval.py +++ b/lib/apscheduler/triggers/interval.py @@ -1,39 +1,65 @@ -from datetime import datetime, timedelta +from datetime import timedelta, datetime from math import ceil -from apscheduler.util import convert_to_datetime, timedelta_seconds +from tzlocal import get_localzone + +from apscheduler.triggers.base import BaseTrigger +from apscheduler.util import convert_to_datetime, timedelta_seconds, datetime_repr, astimezone -class IntervalTrigger(object): - def __init__(self, interval, start_date=None): - if not isinstance(interval, timedelta): - raise TypeError('interval must be a timedelta') - if start_date: - start_date = convert_to_datetime(start_date) +class IntervalTrigger(BaseTrigger): + """ + Triggers on specified intervals, starting on ``start_date`` if specified, ``datetime.now()`` + interval + otherwise. - self.interval = interval + :param int weeks: number of weeks to wait + :param int days: number of days to wait + :param int hours: number of hours to wait + :param int minutes: number of minutes to wait + :param int seconds: number of seconds to wait + :param datetime|str start_date: starting point for the interval calculation + :param datetime|str end_date: latest possible date/time to trigger on + :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations + """ + + __slots__ = 'timezone', 'start_date', 'end_date', 'interval' + + def __init__(self, weeks=0, days=0, hours=0, minutes=0, seconds=0, start_date=None, end_date=None, timezone=None): + self.interval = timedelta(weeks=weeks, days=days, hours=hours, minutes=minutes, seconds=seconds) self.interval_length = timedelta_seconds(self.interval) if self.interval_length == 0: self.interval = timedelta(seconds=1) self.interval_length = 1 - if start_date is None: - self.start_date = datetime.now() + self.interval + if timezone: + self.timezone = astimezone(timezone) + elif start_date and start_date.tzinfo: + self.timezone = start_date.tzinfo + elif end_date and end_date.tzinfo: + self.timezone = end_date.tzinfo else: - self.start_date = convert_to_datetime(start_date) + self.timezone = get_localzone() - def get_next_fire_time(self, start_date): - if start_date < self.start_date: - return self.start_date + start_date = start_date or (datetime.now(self.timezone) + self.interval) + self.start_date = convert_to_datetime(start_date, self.timezone, 'start_date') + self.end_date = convert_to_datetime(end_date, self.timezone, 'end_date') - timediff_seconds = timedelta_seconds(start_date - self.start_date) - next_interval_num = int(ceil(timediff_seconds / self.interval_length)) - return self.start_date + self.interval * next_interval_num + def get_next_fire_time(self, previous_fire_time, now): + if previous_fire_time: + next_fire_time = previous_fire_time + self.interval + elif self.start_date > now: + next_fire_time = self.start_date + else: + timediff_seconds = timedelta_seconds(now - self.start_date) + next_interval_num = int(ceil(timediff_seconds / self.interval_length)) + next_fire_time = self.start_date + self.interval * next_interval_num + + if not self.end_date or next_fire_time <= self.end_date: + return self.timezone.normalize(next_fire_time) def __str__(self): return 'interval[%s]' % str(self.interval) def __repr__(self): - return "<%s (interval=%s, start_date=%s)>" % ( - self.__class__.__name__, repr(self.interval), - repr(self.start_date)) + return "<%s (interval=%r, start_date='%s')>" % (self.__class__.__name__, self.interval, + datetime_repr(self.start_date)) diff --git a/lib/apscheduler/triggers/simple.py b/lib/apscheduler/triggers/simple.py deleted file mode 100644 index ea61b3f1..00000000 --- a/lib/apscheduler/triggers/simple.py +++ /dev/null @@ -1,17 +0,0 @@ -from apscheduler.util import convert_to_datetime - - -class SimpleTrigger(object): - def __init__(self, run_date): - self.run_date = convert_to_datetime(run_date) - - def get_next_fire_time(self, start_date): - if self.run_date >= start_date: - return self.run_date - - def __str__(self): - return 'date[%s]' % str(self.run_date) - - def __repr__(self): - return '<%s (run_date=%s)>' % ( - self.__class__.__name__, repr(self.run_date)) diff --git a/lib/apscheduler/util.py b/lib/apscheduler/util.py index af28ae49..988f9427 100644 --- a/lib/apscheduler/util.py +++ b/lib/apscheduler/util.py @@ -1,26 +1,48 @@ -""" -This module contains several handy functions primarily meant for internal use. -""" +"""This module contains several handy functions primarily meant for internal use.""" -from datetime import date, datetime, timedelta -from time import mktime +from __future__ import division +from datetime import date, datetime, time, timedelta, tzinfo +from inspect import isfunction, ismethod, getargspec +from calendar import timegm import re -import sys -__all__ = ('asint', 'asbool', 'convert_to_datetime', 'timedelta_seconds', - 'time_difference', 'datetime_ceil', 'combine_opts', - 'get_callable_name', 'obj_to_ref', 'ref_to_obj', 'maybe_ref', - 'to_unicode', 'iteritems', 'itervalues', 'xrange') +from pytz import timezone, utc +import six + +try: + from inspect import signature +except ImportError: # pragma: nocover + try: + from funcsigs import signature + except ImportError: + signature = None + +__all__ = ('asint', 'asbool', 'astimezone', 'convert_to_datetime', 'datetime_to_utc_timestamp', + 'utc_timestamp_to_datetime', 'timedelta_seconds', 'datetime_ceil', 'get_callable_name', 'obj_to_ref', + 'ref_to_obj', 'maybe_ref', 'repr_escape', 'check_callable_args') + + +class _Undefined(object): + def __nonzero__(self): + return False + + def __bool__(self): + return False + + def __repr__(self): + return '' + +undefined = _Undefined() #: a unique object that only signifies that no value is defined def asint(text): """ - Safely converts a string to an integer, returning None if the string - is None. + Safely converts a string to an integer, returning None if the string is None. :type text: str :rtype: int """ + if text is not None: return int(text) @@ -31,6 +53,7 @@ def asbool(obj): :rtype: bool """ + if isinstance(obj, str): obj = obj.strip().lower() if obj in ('true', 'yes', 'on', 'y', 't', '1'): @@ -41,36 +64,99 @@ def asbool(obj): return bool(obj) +def astimezone(obj): + """ + Interprets an object as a timezone. + + :rtype: tzinfo + """ + + if isinstance(obj, six.string_types): + return timezone(obj) + if isinstance(obj, tzinfo): + if not hasattr(obj, 'localize') or not hasattr(obj, 'normalize'): + raise TypeError('Only timezones from the pytz library are supported') + if obj.zone == 'local': + raise ValueError('Unable to determine the name of the local timezone -- use an explicit timezone instead') + return obj + if obj is not None: + raise TypeError('Expected tzinfo, got %s instead' % obj.__class__.__name__) + + _DATE_REGEX = re.compile( r'(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})' r'(?: (?P\d{1,2}):(?P\d{1,2}):(?P\d{1,2})' r'(?:\.(?P\d{1,6}))?)?') -def convert_to_datetime(input): +def convert_to_datetime(input, tz, arg_name): """ - Converts the given object to a datetime object, if possible. - If an actual datetime object is passed, it is returned unmodified. - If the input is a string, it is parsed as a datetime. + Converts the given object to a timezone aware datetime object. + If a timezone aware datetime object is passed, it is returned unmodified. + If a native datetime object is passed, it is given the specified timezone. + If the input is a string, it is parsed as a datetime with the given timezone. Date strings are accepted in three different forms: date only (Y-m-d), date with time (Y-m-d H:M:S) or with date+time with microseconds (Y-m-d H:M:S.micro). + :param str|datetime input: the datetime or string to convert to a timezone aware datetime + :param datetime.tzinfo tz: timezone to interpret ``input`` in + :param str arg_name: the name of the argument (used in an error message) :rtype: datetime """ - if isinstance(input, datetime): - return input + + if input is None: + return + elif isinstance(input, datetime): + datetime_ = input elif isinstance(input, date): - return datetime.fromordinal(input.toordinal()) - elif isinstance(input, str): + datetime_ = datetime.combine(input, time()) + elif isinstance(input, six.string_types): m = _DATE_REGEX.match(input) if not m: raise ValueError('Invalid date string') values = [(k, int(v or 0)) for k, v in m.groupdict().items()] values = dict(values) - return datetime(**values) - raise TypeError('Unsupported input type: %s' % type(input)) + datetime_ = datetime(**values) + else: + raise TypeError('Unsupported type for %s: %s' % (arg_name, input.__class__.__name__)) + + if datetime_.tzinfo is not None: + return datetime_ + if tz is None: + raise ValueError('The "tz" argument must be specified if %s has no timezone information' % arg_name) + if isinstance(tz, six.string_types): + tz = timezone(tz) + + try: + return tz.localize(datetime_, is_dst=None) + except AttributeError: + raise TypeError('Only pytz timezones are supported (need the localize() and normalize() methods)') + + +def datetime_to_utc_timestamp(timeval): + """ + Converts a datetime instance to a timestamp. + + :type timeval: datetime + :rtype: float + """ + + if timeval is not None: + return timegm(timeval.utctimetuple()) + timeval.microsecond / 1000000 + + +def utc_timestamp_to_datetime(timestamp): + """ + Converts the given timestamp to a datetime instance. + + :type timestamp: float + :rtype: datetime + """ + + if timestamp is not None: + return datetime.fromtimestamp(timestamp, utc) def timedelta_seconds(delta): @@ -80,125 +166,220 @@ def timedelta_seconds(delta): :type delta: timedelta :rtype: float """ + return delta.days * 24 * 60 * 60 + delta.seconds + \ delta.microseconds / 1000000.0 -def time_difference(date1, date2): - """ - Returns the time difference in seconds between the given two - datetime objects. The difference is calculated as: date1 - date2. - - :param date1: the later datetime - :type date1: datetime - :param date2: the earlier datetime - :type date2: datetime - :rtype: float - """ - later = mktime(date1.timetuple()) + date1.microsecond / 1000000.0 - earlier = mktime(date2.timetuple()) + date2.microsecond / 1000000.0 - return later - earlier - - def datetime_ceil(dateval): """ Rounds the given datetime object upwards. :type dateval: datetime """ + if dateval.microsecond > 0: - return dateval + timedelta(seconds=1, - microseconds=-dateval.microsecond) + return dateval + timedelta(seconds=1, microseconds=-dateval.microsecond) return dateval -def combine_opts(global_config, prefix, local_config={}): - """ - Returns a subdictionary from keys and values of ``global_config`` where - the key starts with the given prefix, combined with options from - local_config. The keys in the subdictionary have the prefix removed. - - :type global_config: dict - :type prefix: str - :type local_config: dict - :rtype: dict - """ - prefixlen = len(prefix) - subconf = {} - for key, value in global_config.items(): - if key.startswith(prefix): - key = key[prefixlen:] - subconf[key] = value - subconf.update(local_config) - return subconf +def datetime_repr(dateval): + return dateval.strftime('%Y-%m-%d %H:%M:%S %Z') if dateval else 'None' def get_callable_name(func): """ Returns the best available display name for the given function/callable. + + :rtype: str """ - name = func.__module__ - if hasattr(func, '__self__') and func.__self__: - name += '.' + func.__self__.__name__ - elif hasattr(func, 'im_self') and func.im_self: # py2.4, 2.5 - name += '.' + func.im_self.__name__ - if hasattr(func, '__name__'): - name += '.' + func.__name__ - return name + + # the easy case (on Python 3.3+) + if hasattr(func, '__qualname__'): + return func.__qualname__ + + # class methods, bound and unbound methods + f_self = getattr(func, '__self__', None) or getattr(func, 'im_self', None) + if f_self and hasattr(func, '__name__'): + f_class = f_self if isinstance(f_self, type) else f_self.__class__ + else: + f_class = getattr(func, 'im_class', None) + + if f_class and hasattr(func, '__name__'): + return '%s.%s' % (f_class.__name__, func.__name__) + + # class or class instance + if hasattr(func, '__call__'): + # class + if hasattr(func, '__name__'): + return func.__name__ + + # instance of a class with a __call__ method + return func.__class__.__name__ + + raise TypeError('Unable to determine a name for %r -- maybe it is not a callable?' % func) def obj_to_ref(obj): """ Returns the path to the given object. - """ - ref = '%s:%s' % (obj.__module__, obj.__name__) - try: - obj2 = ref_to_obj(ref) - except AttributeError: - pass - else: - if obj2 == obj: - return ref - raise ValueError('Only module level objects are supported') + :rtype: str + """ + + try: + ref = '%s:%s' % (obj.__module__, get_callable_name(obj)) + obj2 = ref_to_obj(ref) + if obj != obj2: + raise ValueError + except Exception: + raise ValueError('Cannot determine the reference to %r' % obj) + + return ref def ref_to_obj(ref): """ Returns the object pointed to by ``ref``. + + :type ref: str """ + + if not isinstance(ref, six.string_types): + raise TypeError('References must be strings') + if ':' not in ref: + raise ValueError('Invalid reference') + modulename, rest = ref.split(':', 1) - obj = __import__(modulename) - for name in modulename.split('.')[1:] + rest.split('.'): - obj = getattr(obj, name) - return obj + try: + obj = __import__(modulename) + except ImportError: + raise LookupError('Error resolving reference %s: could not import module' % ref) + + try: + for name in modulename.split('.')[1:] + rest.split('.'): + obj = getattr(obj, name) + return obj + except Exception: + raise LookupError('Error resolving reference %s: error looking up object' % ref) def maybe_ref(ref): """ - Returns the object that the given reference points to, if it is indeed - a reference. If it is not a reference, the object is returned as-is. + Returns the object that the given reference points to, if it is indeed a reference. + If it is not a reference, the object is returned as-is. """ + if not isinstance(ref, str): return ref return ref_to_obj(ref) -def to_unicode(string, encoding='ascii'): - """ - Safely converts a string to a unicode representation on any - Python version. - """ - if hasattr(string, 'decode'): - return string.decode(encoding, 'ignore') - return string +if six.PY2: + def repr_escape(string): + if isinstance(string, six.text_type): + return string.encode('ascii', 'backslashreplace') + return string +else: + repr_escape = lambda string: string -if sys.version_info < (3, 0): # pragma: nocover - iteritems = lambda d: d.iteritems() - itervalues = lambda d: d.itervalues() - xrange = xrange -else: # pragma: nocover - iteritems = lambda d: d.items() - itervalues = lambda d: d.values() - xrange = range +def check_callable_args(func, args, kwargs): + """ + Ensures that the given callable can be called with the given arguments. + + :type args: tuple + :type kwargs: dict + """ + + pos_kwargs_conflicts = [] # parameters that have a match in both args and kwargs + positional_only_kwargs = [] # positional-only parameters that have a match in kwargs + unsatisfied_args = [] # parameters in signature that don't have a match in args or kwargs + unsatisfied_kwargs = [] # keyword-only arguments that don't have a match in kwargs + unmatched_args = list(args) # args that didn't match any of the parameters in the signature + unmatched_kwargs = list(kwargs) # kwargs that didn't match any of the parameters in the signature + has_varargs = has_var_kwargs = False # indicates if the signature defines *args and **kwargs respectively + + if signature: + try: + sig = signature(func) + except ValueError: + return # signature() doesn't work against every kind of callable + + for param in six.itervalues(sig.parameters): + if param.kind == param.POSITIONAL_OR_KEYWORD: + if param.name in unmatched_kwargs and unmatched_args: + pos_kwargs_conflicts.append(param.name) + elif unmatched_args: + del unmatched_args[0] + elif param.name in unmatched_kwargs: + unmatched_kwargs.remove(param.name) + elif param.default is param.empty: + unsatisfied_args.append(param.name) + elif param.kind == param.POSITIONAL_ONLY: + if unmatched_args: + del unmatched_args[0] + elif param.name in unmatched_kwargs: + unmatched_kwargs.remove(param.name) + positional_only_kwargs.append(param.name) + elif param.default is param.empty: + unsatisfied_args.append(param.name) + elif param.kind == param.KEYWORD_ONLY: + if param.name in unmatched_kwargs: + unmatched_kwargs.remove(param.name) + elif param.default is param.empty: + unsatisfied_kwargs.append(param.name) + elif param.kind == param.VAR_POSITIONAL: + has_varargs = True + elif param.kind == param.VAR_KEYWORD: + has_var_kwargs = True + else: + if not isfunction(func) and not ismethod(func) and hasattr(func, '__call__'): + func = func.__call__ + + try: + argspec = getargspec(func) + except TypeError: + return # getargspec() doesn't work certain callables + + argspec_args = argspec.args if not ismethod(func) else argspec.args[1:] + has_varargs = bool(argspec.varargs) + has_var_kwargs = bool(argspec.keywords) + for arg, default in six.moves.zip_longest(argspec_args, argspec.defaults or (), fillvalue=undefined): + if arg in unmatched_kwargs and unmatched_args: + pos_kwargs_conflicts.append(arg) + elif unmatched_args: + del unmatched_args[0] + elif arg in unmatched_kwargs: + unmatched_kwargs.remove(arg) + elif default is undefined: + unsatisfied_args.append(arg) + + # Make sure there are no conflicts between args and kwargs + if pos_kwargs_conflicts: + raise ValueError('The following arguments are supplied in both args and kwargs: %s' % + ', '.join(pos_kwargs_conflicts)) + + # Check if keyword arguments are being fed to positional-only parameters + if positional_only_kwargs: + raise ValueError('The following arguments cannot be given as keyword arguments: %s' % + ', '.join(positional_only_kwargs)) + + # Check that the number of positional arguments minus the number of matched kwargs matches the argspec + if unsatisfied_args: + raise ValueError('The following arguments have not been supplied: %s' % ', '.join(unsatisfied_args)) + + # Check that all keyword-only arguments have been supplied + if unsatisfied_kwargs: + raise ValueError('The following keyword-only arguments have not been supplied in kwargs: %s' % + ', '.join(unsatisfied_kwargs)) + + # Check that the callable can accept the given number of positional arguments + if not has_varargs and unmatched_args: + raise ValueError('The list of positional arguments is longer than the target callable can handle ' + '(allowed: %d, given in args: %d)' % (len(args) - len(unmatched_args), len(args))) + + # Check that the callable can accept the given keyword arguments + if not has_var_kwargs and unmatched_kwargs: + raise ValueError('The target callable does not accept the following keyword arguments: %s' % + ', '.join(unmatched_kwargs)) diff --git a/lib/beets/autotag/hooks.py b/lib/beets/autotag/hooks.py index 74c8cf82..883703f2 100644 --- a/lib/beets/autotag/hooks.py +++ b/lib/beets/autotag/hooks.py @@ -206,8 +206,8 @@ def string_dist(str1, str2): an edit distance, normalized by the string length, with a number of tweaks that reflect intuition about text. """ - if str1 == None and str2 == None: return 0.0 - if str1 == None or str2 == None: return 1.0 + if str1 is None and str2 is None: return 0.0 + if str1 is None or str2 is None: return 1.0 str1 = str1.lower() str2 = str2.lower() diff --git a/lib/bs4/builder/_html5lib.py b/lib/bs4/builder/_html5lib.py index 7de36ae7..d46b695b 100644 --- a/lib/bs4/builder/_html5lib.py +++ b/lib/bs4/builder/_html5lib.py @@ -268,7 +268,7 @@ class Element(html5lib.treebuilders._base.Node): return self.element.contents def getNameTuple(self): - if self.namespace == None: + if self.namespace is None: return namespaces["html"], self.name else: return self.namespace, self.name diff --git a/lib/cherrypy/__init__.py b/lib/cherrypy/__init__.py index 2ea2b324..7870ef0d 100644 --- a/lib/cherrypy/__init__.py +++ b/lib/cherrypy/__init__.py @@ -53,11 +53,10 @@ with customized or extended components. The core API's are: * Server API * WSGI API -These API's are described in the CherryPy specification: -http://www.cherrypy.org/wiki/CherryPySpec +These API's are described in the `CherryPy specification `_. """ -__version__ = "3.2.2" +__version__ = "3.6.0" from cherrypy._cpcompat import urljoin as _urljoin, urlencode as _urlencode from cherrypy._cpcompat import basestring, unicodestr, set @@ -94,6 +93,7 @@ except ImportError: engine.listeners['before_request'] = set() engine.listeners['after_request'] = set() + class _TimeoutMonitor(process.plugins.Monitor): def __init__(self, bus): @@ -125,6 +125,24 @@ engine.thread_manager.subscribe() engine.signal_handler = process.plugins.SignalHandler(engine) +class _HandleSignalsPlugin(object): + + """Handle signals from other processes based on the configured + platform handlers above.""" + + def __init__(self, bus): + self.bus = bus + + def subscribe(self): + """Add the handlers based on the platform""" + if hasattr(self.bus, "signal_handler"): + self.bus.signal_handler.subscribe() + if hasattr(self.bus, "console_control_handler"): + self.bus.console_control_handler.subscribe() + +engine.signals = _HandleSignalsPlugin(engine) + + from cherrypy import _cpserver server = _cpserver.Server() server.subscribe() @@ -152,18 +170,16 @@ def quickstart(root=None, script_name="", config=None): tree.mount(root, script_name, config) - if hasattr(engine, "signal_handler"): - engine.signal_handler.subscribe() - if hasattr(engine, "console_control_handler"): - engine.console_control_handler.subscribe() - + engine.signals.subscribe() engine.start() engine.block() from cherrypy._cpcompat import threadlocal as _local + class _Serving(_local): + """An interface for registering request and response objects. Rather than have a separate "thread local" object for the request and @@ -258,7 +274,10 @@ request = _ThreadLocalProxy('request') response = _ThreadLocalProxy('response') # Create thread_data object as a thread-specific all-purpose storage + + class _ThreadData(_local): + """A container for thread-specific data.""" thread_data = _ThreadData() @@ -283,7 +302,9 @@ except ImportError: from cherrypy import _cplogging + class _GlobalLogManager(_cplogging.LogManager): + """A site-wide LogManager; routes to app.log or global log as appropriate. This :class:`LogManager` implements @@ -294,8 +315,10 @@ class _GlobalLogManager(_cplogging.LogManager): """ def __call__(self, *args, **kwargs): - """Log the given message to the app.log or global log as appropriate.""" - # Do NOT use try/except here. See http://www.cherrypy.org/ticket/945 + """Log the given message to the app.log or global log as appropriate. + """ + # Do NOT use try/except here. See + # https://bitbucket.org/cherrypy/cherrypy/issue/945 if hasattr(request, 'app') and hasattr(request.app, 'log'): log = request.app.log else: @@ -303,7 +326,8 @@ class _GlobalLogManager(_cplogging.LogManager): return log.error(*args, **kwargs) def access(self): - """Log an access message to the app.log or global log as appropriate.""" + """Log an access message to the app.log or global log as appropriate. + """ try: return request.app.log.access() except AttributeError: @@ -317,6 +341,7 @@ log.error_file = '' # Using an access file makes CP about 10% slower. Leave off by default. log.access_file = '' + def _buslog(msg, level): log.error(msg, 'ENGINE', severity=level) engine.subscribe('log', _buslog) @@ -336,7 +361,8 @@ def expose(func=None, alias=None): parents[a.replace(".", "_")] = func return func - import sys, types + import sys + import types if isinstance(func, (types.FunctionType, types.MethodType)): if alias is None: # @expose @@ -363,6 +389,7 @@ def expose(func=None, alias=None): alias = func return expose_ + def popargs(*args, **kwargs): """A decorator for _cp_dispatch (cherrypy.dispatch.Dispatcher.dispatch_method_name). @@ -442,34 +469,34 @@ def popargs(*args, **kwargs): """ - #Since keyword arg comes after *args, we have to process it ourselves - #for lower versions of python. + # Since keyword arg comes after *args, we have to process it ourselves + # for lower versions of python. handler = None handler_call = False - for k,v in kwargs.items(): + for k, v in kwargs.items(): if k == 'handler': handler = v else: raise TypeError( - "cherrypy.popargs() got an unexpected keyword argument '{0}'" \ + "cherrypy.popargs() got an unexpected keyword argument '{0}'" .format(k) - ) + ) import inspect if handler is not None \ - and (hasattr(handler, '__call__') or inspect.isclass(handler)): + and (hasattr(handler, '__call__') or inspect.isclass(handler)): handler_call = True def decorated(cls_or_self=None, vpath=None): if inspect.isclass(cls_or_self): - #cherrypy.popargs is a class decorator + # cherrypy.popargs is a class decorator cls = cls_or_self setattr(cls, dispatch.Dispatcher.dispatch_method_name, decorated) return cls - #We're in the actual function + # We're in the actual function self = cls_or_self parms = {} for arg in args: @@ -486,9 +513,9 @@ def popargs(*args, **kwargs): request.params.update(parms) - #If we are the ultimate handler, then to prevent our _cp_dispatch - #from being called again, we will resolve remaining elements through - #getattr() directly. + # If we are the ultimate handler, then to prevent our _cp_dispatch + # from being called again, we will resolve remaining elements through + # getattr() directly. if vpath: return getattr(self, vpath.pop(0), None) else: @@ -496,6 +523,7 @@ def popargs(*args, **kwargs): return decorated + def url(path="", qs="", script_name=None, base=None, relative=None): """Create an absolute URL for the given path. @@ -613,7 +641,7 @@ config.defaults = { 'tools.log_headers.on': True, 'tools.trailing_slash.on': True, 'tools.encode.on': True - } +} config.namespaces["log"] = lambda k, v: setattr(log, k, v) config.namespaces["checker"] = lambda k, v: setattr(checker, k, v) # Must reset to get our defaults applied. diff --git a/lib/cherrypy/_cpchecker.py b/lib/cherrypy/_cpchecker.py index 3205ed09..4ef82597 100644 --- a/lib/cherrypy/_cpchecker.py +++ b/lib/cherrypy/_cpchecker.py @@ -6,6 +6,7 @@ from cherrypy._cpcompat import iteritems, copykeys, builtins class Checker(object): + """A checker for CherryPy sites and their mounted applications. When this object is called at engine startup, it executes each @@ -22,7 +23,6 @@ class Checker(object): on = True """If True (the default), run all checks; if False, turn off all checks.""" - def __init__(self): self._populate_known_types() @@ -48,7 +48,8 @@ class Checker(object): global_config_contained_paths = False def check_app_config_entries_dont_start_with_script_name(self): - """Check for Application config with sections that repeat script_name.""" + """Check for Application config with sections that repeat script_name. + """ for sn, app in cherrypy.tree.apps.items(): if not isinstance(app, cherrypy.Application): continue @@ -61,8 +62,9 @@ class Checker(object): key_atoms = key.strip("/").split("/") if key_atoms[:len(sn_atoms)] == sn_atoms: warnings.warn( - "The application mounted at %r has config " \ - "entries that start with its script name: %r" % (sn, key)) + "The application mounted at %r has config " + "entries that start with its script name: %r" % (sn, + key)) def check_site_config_entries_in_app_config(self): """Check for mounted Applications that have site-scoped config.""" @@ -76,13 +78,15 @@ class Checker(object): for key, value in iteritems(entries): for n in ("engine.", "server.", "tree.", "checker."): if key.startswith(n): - msg.append("[%s] %s = %s" % (section, key, value)) + msg.append("[%s] %s = %s" % + (section, key, value)) if msg: msg.insert(0, - "The application mounted at %r contains the following " - "config entries, which are only allowed in site-wide " - "config. Move them to a [global] section and pass them " - "to cherrypy.config.update() instead of tree.mount()." % sn) + "The application mounted at %r contains the " + "following config entries, which are only allowed " + "in site-wide config. Move them to a [global] " + "section and pass them to cherrypy.config.update() " + "instead of tree.mount()." % sn) warnings.warn(os.linesep.join(msg)) def check_skipped_app_config(self): @@ -102,7 +106,9 @@ class Checker(object): return def check_app_config_brackets(self): - """Check for Application config with extraneous brackets in section names.""" + """Check for Application config with extraneous brackets in section + names. + """ for sn, app in cherrypy.tree.apps.items(): if not isinstance(app, cherrypy.Application): continue @@ -111,7 +117,7 @@ class Checker(object): for key in app.config.keys(): if key.startswith("[") or key.endswith("]"): warnings.warn( - "The application mounted at %r has config " \ + "The application mounted at %r has config " "section names with extraneous brackets: %r. " "Config *files* need brackets; config *dicts* " "(e.g. passed to tree.mount) do not." % (sn, key)) @@ -144,16 +150,20 @@ class Checker(object): "though a root is provided.") testdir = os.path.join(root, dir[1:]) if os.path.exists(testdir): - msg += ("\nIf you meant to serve the " - "filesystem folder at %r, remove " - "the leading slash from dir." % testdir) + msg += ( + "\nIf you meant to serve the " + "filesystem folder at %r, remove the " + "leading slash from dir." % (testdir,)) else: if not root: - msg = "dir is a relative path and no root provided." + msg = ( + "dir is a relative path and " + "no root provided.") else: fulldir = os.path.join(root, dir) if not os.path.isabs(fulldir): - msg = "%r is not an absolute path." % fulldir + msg = ("%r is not an absolute path." % ( + fulldir,)) if fulldir and not os.path.exists(fulldir): if msg: @@ -165,9 +175,7 @@ class Checker(object): warnings.warn("%s\nsection: [%s]\nroot: %r\ndir: %r" % (msg, section, root, dir)) - # -------------------------- Compatibility -------------------------- # - obsolete = { 'server.default_content_type': 'tools.response_headers.headers', 'log_access_file': 'log.access_file', @@ -180,7 +188,7 @@ class Checker(object): 'throw_errors': 'request.throw_errors', 'profiler.on': ('cherrypy.tree.mount(profiler.make_app(' 'cherrypy.Application(Root())))'), - } + } deprecated = {} @@ -213,9 +221,7 @@ class Checker(object): continue self._compat(app.config) - # ------------------------ Known Namespaces ------------------------ # - extra_config_namespaces = [] def _known_ns(self, app): @@ -235,20 +241,24 @@ class Checker(object): if atoms[0] not in ns: # Spit out a special warning if a known # namespace is preceded by "cherrypy." - if (atoms[0] == "cherrypy" and atoms[1] in ns): - msg = ("The config entry %r is invalid; " - "try %r instead.\nsection: [%s]" - % (k, ".".join(atoms[1:]), section)) + if atoms[0] == "cherrypy" and atoms[1] in ns: + msg = ( + "The config entry %r is invalid; " + "try %r instead.\nsection: [%s]" + % (k, ".".join(atoms[1:]), section)) else: - msg = ("The config entry %r is invalid, because " - "the %r config namespace is unknown.\n" - "section: [%s]" % (k, atoms[0], section)) + msg = ( + "The config entry %r is invalid, " + "because the %r config namespace " + "is unknown.\n" + "section: [%s]" % (k, atoms[0], section)) warnings.warn(msg) elif atoms[0] == "tools": if atoms[1] not in dir(cherrypy.tools): - msg = ("The config entry %r may be invalid, " - "because the %r tool was not found.\n" - "section: [%s]" % (k, atoms[1], section)) + msg = ( + "The config entry %r may be invalid, " + "because the %r tool was not found.\n" + "section: [%s]" % (k, atoms[1], section)) warnings.warn(msg) def check_config_namespaces(self): @@ -258,11 +268,7 @@ class Checker(object): continue self._known_ns(app) - - - # -------------------------- Config Types -------------------------- # - known_config_types = {} def _populate_known_types(self): @@ -314,14 +320,13 @@ class Checker(object): continue self._known_types(app.config) - # -------------------- Specific config warnings -------------------- # - def check_localhost(self): """Warn if any socket_host is 'localhost'. See #711.""" for k, v in cherrypy.config.items(): if k == 'server.socket_host' and v == 'localhost': warnings.warn("The use of 'localhost' as a socket host can " - "cause problems on newer systems, since 'localhost' can " - "map to either an IPv4 or an IPv6 address. You should " - "use '127.0.0.1' or '[::1]' instead.") + "cause problems on newer systems, since " + "'localhost' can map to either an IPv4 or an " + "IPv6 address. You should use '127.0.0.1' " + "or '[::1]' instead.") diff --git a/lib/cherrypy/_cpcompat.py b/lib/cherrypy/_cpcompat.py index ed24c1ab..8a98b38b 100644 --- a/lib/cherrypy/_cpcompat.py +++ b/lib/cherrypy/_cpcompat.py @@ -18,6 +18,7 @@ output. import os import re import sys +import threading if sys.version_info >= (3, 0): py3k = True @@ -25,14 +26,23 @@ if sys.version_info >= (3, 0): unicodestr = str nativestr = unicodestr basestring = (bytes, str) + def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given encoding.""" + """Return the given native string as a byte string in the given + encoding. + """ + assert_native(n) # In Python 3, the native string type is unicode return n.encode(encoding) + def ntou(n, encoding='ISO-8859-1'): - """Return the given native string as a unicode string with the given encoding.""" + """Return the given native string as a unicode string with the given + encoding. + """ + assert_native(n) # In Python 3, the native string type is unicode return n + def tonative(n, encoding='ISO-8859-1'): """Return the given string as a native string in the given encoding.""" # In Python 3, the native string type is unicode @@ -50,27 +60,36 @@ else: unicodestr = unicode nativestr = bytestr basestring = basestring + def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given encoding.""" + """Return the given native string as a byte string in the given + encoding. + """ + assert_native(n) # In Python 2, the native string type is bytes. Assume it's already # in the given encoding, which for ISO-8859-1 is almost always what # was intended. return n + def ntou(n, encoding='ISO-8859-1'): - """Return the given native string as a unicode string with the given encoding.""" + """Return the given native string as a unicode string with the given + encoding. + """ + assert_native(n) # In Python 2, the native string type is bytes. - # First, check for the special encoding 'escape'. The test suite uses this - # to signal that it wants to pass a string with embedded \uXXXX escapes, - # but without having to prefix it with u'' for Python 2, but no prefix - # for Python 3. + # First, check for the special encoding 'escape'. The test suite uses + # this to signal that it wants to pass a string with embedded \uXXXX + # escapes, but without having to prefix it with u'' for Python 2, + # but no prefix for Python 3. if encoding == 'escape': return unicode( re.sub(r'\\u([0-9a-zA-Z]{4})', lambda m: unichr(int(m.group(1), 16)), n.decode('ISO-8859-1'))) - # Assume it's already in the given encoding, which for ISO-8859-1 is almost - # always what was intended. + # Assume it's already in the given encoding, which for ISO-8859-1 + # is almost always what was intended. return n.decode(encoding) + def tonative(n, encoding='ISO-8859-1'): """Return the given string as a native string in the given encoding.""" # In Python 2, the native string type is bytes. @@ -86,6 +105,11 @@ else: # bytes: BytesIO = StringIO + +def assert_native(n): + if not isinstance(n, nativestr): + raise TypeError("n must be a native str (got %s)" % type(n).__name__) + try: set = set except NameError: @@ -100,6 +124,7 @@ except ImportError: # the legacy API of base64 from base64 import decodestring as _base64_decodebytes + def base64_decode(n, encoding='ISO-8859-1'): """Return the native string base64-decoded (as a native string).""" if isinstance(n, unicodestr): @@ -198,28 +223,31 @@ except ImportError: import __builtin__ as builtins try: - # Python 2. We have to do it in this order so Python 2 builds + # Python 2. We try Python 2 first clients on Python 2 # don't try to import the 'http' module from cherrypy.lib from Cookie import SimpleCookie, CookieError - from httplib import BadStatusLine, HTTPConnection, HTTPSConnection, IncompleteRead, NotConnected + from httplib import BadStatusLine, HTTPConnection, IncompleteRead + from httplib import NotConnected from BaseHTTPServer import BaseHTTPRequestHandler except ImportError: # Python 3 from http.cookies import SimpleCookie, CookieError - from http.client import BadStatusLine, HTTPConnection, HTTPSConnection, IncompleteRead, NotConnected + from http.client import BadStatusLine, HTTPConnection, IncompleteRead + from http.client import NotConnected from http.server import BaseHTTPRequestHandler -try: - # Python 2. We have to do it in this order so Python 2 builds - # don't try to import the 'http' module from cherrypy.lib - from httplib import HTTPSConnection -except ImportError: +# Some platforms don't expose HTTPSConnection, so handle it separately +if py3k: try: - # Python 3 from http.client import HTTPSConnection except ImportError: # Some platforms which don't have SSL don't expose HTTPSConnection HTTPSConnection = None +else: + try: + from httplib import HTTPSConnection + except ImportError: + HTTPSConnection = None try: # Python 2 @@ -233,16 +261,19 @@ if hasattr(threading.Thread, "daemon"): # Python 2.6+ def get_daemon(t): return t.daemon + def set_daemon(t, val): t.daemon = val else: def get_daemon(t): return t.isDaemon() + def set_daemon(t, val): t.setDaemon(val) try: from email.utils import formatdate + def HTTPDate(timeval=None): return formatdate(timeval, usegmt=True) except ImportError: @@ -251,40 +282,49 @@ except ImportError: try: # Python 3 from urllib.parse import unquote as parse_unquote + def unquote_qs(atom, encoding, errors='strict'): - return parse_unquote(atom.replace('+', ' '), encoding=encoding, errors=errors) + return parse_unquote( + atom.replace('+', ' '), + encoding=encoding, + errors=errors) except ImportError: # Python 2 from urllib import unquote as parse_unquote + def unquote_qs(atom, encoding, errors='strict'): return parse_unquote(atom.replace('+', ' ')).decode(encoding, errors) try: - # Prefer simplejson, which is usually more advanced than the builtin module. + # Prefer simplejson, which is usually more advanced than the builtin + # module. import simplejson as json json_decode = json.JSONDecoder().decode - json_encode = json.JSONEncoder().iterencode + _json_encode = json.JSONEncoder().iterencode except ImportError: - if py3k: - # Python 3.0: json is part of the standard library, - # but outputs unicode. We need bytes. + if sys.version_info >= (2, 6): + # Python >=2.6 : json is part of the standard library import json json_decode = json.JSONDecoder().decode _json_encode = json.JSONEncoder().iterencode + else: + json = None + + def json_decode(s): + raise ValueError('No JSON library is available') + + def _json_encode(s): + raise ValueError('No JSON library is available') +finally: + if json and py3k: + # The two Python 3 implementations (simplejson/json) + # outputs str. We need bytes. def json_encode(value): for chunk in _json_encode(value): yield chunk.encode('utf8') - elif sys.version_info >= (2, 6): - # Python 2.6: json is part of the standard library - import json - json_decode = json.JSONDecoder().decode - json_encode = json.JSONEncoder().iterencode else: - json = None - def json_decode(s): - raise ValueError('No JSON library is available') - def json_encode(s): - raise ValueError('No JSON library is available') + json_encode = _json_encode + try: import cPickle as pickle @@ -296,11 +336,13 @@ except ImportError: try: os.urandom(20) import binascii + def random20(): return binascii.hexlify(os.urandom(20)).decode('ascii') except (AttributeError, NotImplementedError): import random # os.urandom not available until Python 2.4. Fall back to random.random. + def random20(): return sha('%s' % random.random()).hexdigest() @@ -316,3 +358,26 @@ except NameError: # Python 2 def next(i): return i.next() + +if sys.version_info >= (3, 3): + Timer = threading.Timer + Event = threading.Event +else: + # Python 3.2 and earlier + Timer = threading._Timer + Event = threading._Event + +# Prior to Python 2.6, the Thread class did not have a .daemon property. +# This mix-in adds that property. + + +class SetDaemonProperty: + + def __get_daemon(self): + return self.isDaemon() + + def __set_daemon(self, daemon): + self.setDaemon(daemon) + + if sys.version_info < (2, 6): + daemon = property(__get_daemon, __set_daemon) diff --git a/lib/cherrypy/_cpcompat_subprocess.py b/lib/cherrypy/_cpcompat_subprocess.py new file mode 100644 index 00000000..478f4a74 --- /dev/null +++ b/lib/cherrypy/_cpcompat_subprocess.py @@ -0,0 +1,1544 @@ +# subprocess - Subprocesses with accessible I/O streams +# +# For more information about this module, see PEP 324. +# +# This module should remain compatible with Python 2.2, see PEP 291. +# +# Copyright (c) 2003-2005 by Peter Astrand +# +# Licensed to PSF under a Contributor Agreement. +# See http://www.python.org/2.4/license for licensing details. + +r"""subprocess - Subprocesses with accessible I/O streams + +This module allows you to spawn processes, connect to their +input/output/error pipes, and obtain their return codes. This module +intends to replace several other, older modules and functions, like: + +os.system +os.spawn* +os.popen* +popen2.* +commands.* + +Information about how the subprocess module can be used to replace these +modules and functions can be found below. + + + +Using the subprocess module +=========================== +This module defines one class called Popen: + +class Popen(args, bufsize=0, executable=None, + stdin=None, stdout=None, stderr=None, + preexec_fn=None, close_fds=False, shell=False, + cwd=None, env=None, universal_newlines=False, + startupinfo=None, creationflags=0): + + +Arguments are: + +args should be a string, or a sequence of program arguments. The +program to execute is normally the first item in the args sequence or +string, but can be explicitly set by using the executable argument. + +On UNIX, with shell=False (default): In this case, the Popen class +uses os.execvp() to execute the child program. args should normally +be a sequence. A string will be treated as a sequence with the string +as the only item (the program to execute). + +On UNIX, with shell=True: If args is a string, it specifies the +command string to execute through the shell. If args is a sequence, +the first item specifies the command string, and any additional items +will be treated as additional shell arguments. + +On Windows: the Popen class uses CreateProcess() to execute the child +program, which operates on strings. If args is a sequence, it will be +converted to a string using the list2cmdline method. Please note that +not all MS Windows applications interpret the command line the same +way: The list2cmdline is designed for applications using the same +rules as the MS C runtime. + +bufsize, if given, has the same meaning as the corresponding argument +to the built-in open() function: 0 means unbuffered, 1 means line +buffered, any other positive value means use a buffer of +(approximately) that size. A negative bufsize means to use the system +default, which usually means fully buffered. The default value for +bufsize is 0 (unbuffered). + +stdin, stdout and stderr specify the executed programs' standard +input, standard output and standard error file handles, respectively. +Valid values are PIPE, an existing file descriptor (a positive +integer), an existing file object, and None. PIPE indicates that a +new pipe to the child should be created. With None, no redirection +will occur; the child's file handles will be inherited from the +parent. Additionally, stderr can be STDOUT, which indicates that the +stderr data from the applications should be captured into the same +file handle as for stdout. + +If preexec_fn is set to a callable object, this object will be called +in the child process just before the child is executed. + +If close_fds is true, all file descriptors except 0, 1 and 2 will be +closed before the child process is executed. + +if shell is true, the specified command will be executed through the +shell. + +If cwd is not None, the current directory will be changed to cwd +before the child is executed. + +If env is not None, it defines the environment variables for the new +process. + +If universal_newlines is true, the file objects stdout and stderr are +opened as a text files, but lines may be terminated by any of '\n', +the Unix end-of-line convention, '\r', the Macintosh convention or +'\r\n', the Windows convention. All of these external representations +are seen as '\n' by the Python program. Note: This feature is only +available if Python is built with universal newline support (the +default). Also, the newlines attribute of the file objects stdout, +stdin and stderr are not updated by the communicate() method. + +The startupinfo and creationflags, if given, will be passed to the +underlying CreateProcess() function. They can specify things such as +appearance of the main window and priority for the new process. +(Windows only) + + +This module also defines some shortcut functions: + +call(*popenargs, **kwargs): + Run command with arguments. Wait for command to complete, then + return the returncode attribute. + + The arguments are the same as for the Popen constructor. Example: + + retcode = call(["ls", "-l"]) + +check_call(*popenargs, **kwargs): + Run command with arguments. Wait for command to complete. If the + exit code was zero then return, otherwise raise + CalledProcessError. The CalledProcessError object will have the + return code in the returncode attribute. + + The arguments are the same as for the Popen constructor. Example: + + check_call(["ls", "-l"]) + +check_output(*popenargs, **kwargs): + Run command with arguments and return its output as a byte string. + + If the exit code was non-zero it raises a CalledProcessError. The + CalledProcessError object will have the return code in the returncode + attribute and output in the output attribute. + + The arguments are the same as for the Popen constructor. Example: + + output = check_output(["ls", "-l", "/dev/null"]) + + +Exceptions +---------- +Exceptions raised in the child process, before the new program has +started to execute, will be re-raised in the parent. Additionally, +the exception object will have one extra attribute called +'child_traceback', which is a string containing traceback information +from the childs point of view. + +The most common exception raised is OSError. This occurs, for +example, when trying to execute a non-existent file. Applications +should prepare for OSErrors. + +A ValueError will be raised if Popen is called with invalid arguments. + +check_call() and check_output() will raise CalledProcessError, if the +called process returns a non-zero return code. + + +Security +-------- +Unlike some other popen functions, this implementation will never call +/bin/sh implicitly. This means that all characters, including shell +metacharacters, can safely be passed to child processes. + + +Popen objects +============= +Instances of the Popen class have the following methods: + +poll() + Check if child process has terminated. Returns returncode + attribute. + +wait() + Wait for child process to terminate. Returns returncode attribute. + +communicate(input=None) + Interact with process: Send data to stdin. Read data from stdout + and stderr, until end-of-file is reached. Wait for process to + terminate. The optional input argument should be a string to be + sent to the child process, or None, if no data should be sent to + the child. + + communicate() returns a tuple (stdout, stderr). + + Note: The data read is buffered in memory, so do not use this + method if the data size is large or unlimited. + +The following attributes are also available: + +stdin + If the stdin argument is PIPE, this attribute is a file object + that provides input to the child process. Otherwise, it is None. + +stdout + If the stdout argument is PIPE, this attribute is a file object + that provides output from the child process. Otherwise, it is + None. + +stderr + If the stderr argument is PIPE, this attribute is file object that + provides error output from the child process. Otherwise, it is + None. + +pid + The process ID of the child process. + +returncode + The child return code. A None value indicates that the process + hasn't terminated yet. A negative value -N indicates that the + child was terminated by signal N (UNIX only). + + +Replacing older functions with the subprocess module +==================================================== +In this section, "a ==> b" means that b can be used as a replacement +for a. + +Note: All functions in this section fail (more or less) silently if +the executed program cannot be found; this module raises an OSError +exception. + +In the following examples, we assume that the subprocess module is +imported with "from subprocess import *". + + +Replacing /bin/sh shell backquote +--------------------------------- +output=`mycmd myarg` +==> +output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0] + + +Replacing shell pipe line +------------------------- +output=`dmesg | grep hda` +==> +p1 = Popen(["dmesg"], stdout=PIPE) +p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) +output = p2.communicate()[0] + + +Replacing os.system() +--------------------- +sts = os.system("mycmd" + " myarg") +==> +p = Popen("mycmd" + " myarg", shell=True) +pid, sts = os.waitpid(p.pid, 0) + +Note: + +* Calling the program through the shell is usually not required. + +* It's easier to look at the returncode attribute than the + exitstatus. + +A more real-world example would look like this: + +try: + retcode = call("mycmd" + " myarg", shell=True) + if retcode < 0: + print >>sys.stderr, "Child was terminated by signal", -retcode + else: + print >>sys.stderr, "Child returned", retcode +except OSError, e: + print >>sys.stderr, "Execution failed:", e + + +Replacing os.spawn* +------------------- +P_NOWAIT example: + +pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg") +==> +pid = Popen(["/bin/mycmd", "myarg"]).pid + + +P_WAIT example: + +retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg") +==> +retcode = call(["/bin/mycmd", "myarg"]) + + +Vector example: + +os.spawnvp(os.P_NOWAIT, path, args) +==> +Popen([path] + args[1:]) + + +Environment example: + +os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env) +==> +Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"}) + + +Replacing os.popen* +------------------- +pipe = os.popen("cmd", mode='r', bufsize) +==> +pipe = Popen("cmd", shell=True, bufsize=bufsize, stdout=PIPE).stdout + +pipe = os.popen("cmd", mode='w', bufsize) +==> +pipe = Popen("cmd", shell=True, bufsize=bufsize, stdin=PIPE).stdin + + +(child_stdin, child_stdout) = os.popen2("cmd", mode, bufsize) +==> +p = Popen("cmd", shell=True, bufsize=bufsize, + stdin=PIPE, stdout=PIPE, close_fds=True) +(child_stdin, child_stdout) = (p.stdin, p.stdout) + + +(child_stdin, + child_stdout, + child_stderr) = os.popen3("cmd", mode, bufsize) +==> +p = Popen("cmd", shell=True, bufsize=bufsize, + stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) +(child_stdin, + child_stdout, + child_stderr) = (p.stdin, p.stdout, p.stderr) + + +(child_stdin, child_stdout_and_stderr) = os.popen4("cmd", mode, + bufsize) +==> +p = Popen("cmd", shell=True, bufsize=bufsize, + stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) +(child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout) + +On Unix, os.popen2, os.popen3 and os.popen4 also accept a sequence as +the command to execute, in which case arguments will be passed +directly to the program without shell intervention. This usage can be +replaced as follows: + +(child_stdin, child_stdout) = os.popen2(["/bin/ls", "-l"], mode, + bufsize) +==> +p = Popen(["/bin/ls", "-l"], bufsize=bufsize, stdin=PIPE, stdout=PIPE) +(child_stdin, child_stdout) = (p.stdin, p.stdout) + +Return code handling translates as follows: + +pipe = os.popen("cmd", 'w') +... +rc = pipe.close() +if rc is not None and rc % 256: + print "There were some errors" +==> +process = Popen("cmd", 'w', shell=True, stdin=PIPE) +... +process.stdin.close() +if process.wait() != 0: + print "There were some errors" + + +Replacing popen2.* +------------------ +(child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode) +==> +p = Popen(["somestring"], shell=True, bufsize=bufsize + stdin=PIPE, stdout=PIPE, close_fds=True) +(child_stdout, child_stdin) = (p.stdout, p.stdin) + +On Unix, popen2 also accepts a sequence as the command to execute, in +which case arguments will be passed directly to the program without +shell intervention. This usage can be replaced as follows: + +(child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, + mode) +==> +p = Popen(["mycmd", "myarg"], bufsize=bufsize, + stdin=PIPE, stdout=PIPE, close_fds=True) +(child_stdout, child_stdin) = (p.stdout, p.stdin) + +The popen2.Popen3 and popen2.Popen4 basically works as subprocess.Popen, +except that: + +* subprocess.Popen raises an exception if the execution fails +* the capturestderr argument is replaced with the stderr argument. +* stdin=PIPE and stdout=PIPE must be specified. +* popen2 closes all filedescriptors by default, but you have to specify + close_fds=True with subprocess.Popen. +""" + +import sys +mswindows = (sys.platform == "win32") + +import os +import types +import traceback +import gc +import signal +import errno + +try: + set +except NameError: + from sets import Set as set + +# Exception classes used by this module. + + +class CalledProcessError(Exception): + + """This exception is raised when a process run by check_call() or + check_output() returns a non-zero exit status. + The exit status will be stored in the returncode attribute; + check_output() will also store the output in the output attribute. + """ + + def __init__(self, returncode, cmd, output=None): + self.returncode = returncode + self.cmd = cmd + self.output = output + + def __str__(self): + return "Command '%s' returned non-zero exit status %d" % ( + self.cmd, self.returncode) + + +if mswindows: + import threading + import msvcrt + import _subprocess + + class STARTUPINFO: + dwFlags = 0 + hStdInput = None + hStdOutput = None + hStdError = None + wShowWindow = 0 + + class pywintypes: + error = IOError +else: + import select + _has_poll = hasattr(select, 'poll') + import fcntl + import pickle + + # When select or poll has indicated that the file is writable, + # we can write up to _PIPE_BUF bytes without risk of blocking. + # POSIX defines PIPE_BUF as >= 512. + _PIPE_BUF = getattr(select, 'PIPE_BUF', 512) + + +__all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", + "check_output", "CalledProcessError"] + +if mswindows: + from _subprocess import CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP, \ + STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, \ + STD_ERROR_HANDLE, SW_HIDE, \ + STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW + + __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP", + "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE", + "STD_ERROR_HANDLE", "SW_HIDE", + "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"]) +try: + MAXFD = os.sysconf("SC_OPEN_MAX") +except: + MAXFD = 256 + +_active = [] + + +def _cleanup(): + for inst in _active[:]: + res = inst._internal_poll(_deadstate=sys.maxint) + if res is not None: + try: + _active.remove(inst) + except ValueError: + # This can happen if two threads create a new Popen instance. + # It's harmless that it was already removed, so ignore. + pass + +PIPE = -1 +STDOUT = -2 + + +def _eintr_retry_call(func, *args): + while True: + try: + return func(*args) + except (OSError, IOError), e: + if e.errno == errno.EINTR: + continue + raise + + +def call(*popenargs, **kwargs): + """Run command with arguments. Wait for command to complete, then + return the returncode attribute. + + The arguments are the same as for the Popen constructor. Example: + + retcode = call(["ls", "-l"]) + """ + return Popen(*popenargs, **kwargs).wait() + + +def check_call(*popenargs, **kwargs): + """Run command with arguments. Wait for command to complete. If + the exit code was zero then return, otherwise raise + CalledProcessError. The CalledProcessError object will have the + return code in the returncode attribute. + + The arguments are the same as for the Popen constructor. Example: + + check_call(["ls", "-l"]) + """ + retcode = call(*popenargs, **kwargs) + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise CalledProcessError(retcode, cmd) + return 0 + + +def check_output(*popenargs, **kwargs): + r"""Run command with arguments and return its output as a byte string. + + If the exit code was non-zero it raises a CalledProcessError. The + CalledProcessError object will have the return code in the returncode + attribute and output in the output attribute. + + The arguments are the same as for the Popen constructor. Example: + + >>> check_output(["ls", "-l", "/dev/null"]) + 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' + + The stdout argument is not allowed as it is used internally. + To capture standard error in the result, use stderr=STDOUT. + + >>> check_output(["/bin/sh", "-c", + ... "ls -l non_existent_file ; exit 0"], + ... stderr=STDOUT) + 'ls: non_existent_file: No such file or directory\n' + """ + if 'stdout' in kwargs: + raise ValueError('stdout argument not allowed, it will be overridden.') + process = Popen(stdout=PIPE, *popenargs, **kwargs) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise CalledProcessError(retcode, cmd, output=output) + return output + + +def list2cmdline(seq): + """ + Translate a sequence of arguments into a command line + string, using the same rules as the MS C runtime: + + 1) Arguments are delimited by white space, which is either a + space or a tab. + + 2) A string surrounded by double quotation marks is + interpreted as a single argument, regardless of white space + contained within. A quoted string can be embedded in an + argument. + + 3) A double quotation mark preceded by a backslash is + interpreted as a literal double quotation mark. + + 4) Backslashes are interpreted literally, unless they + immediately precede a double quotation mark. + + 5) If backslashes immediately precede a double quotation mark, + every pair of backslashes is interpreted as a literal + backslash. If the number of backslashes is odd, the last + backslash escapes the next double quotation mark as + described in rule 3. + """ + + # See + # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx + # or search http://msdn.microsoft.com for + # "Parsing C++ Command-Line Arguments" + result = [] + needquote = False + for arg in seq: + bs_buf = [] + + # Add a space to separate this argument from the others + if result: + result.append(' ') + + needquote = (" " in arg) or ("\t" in arg) or not arg + if needquote: + result.append('"') + + for c in arg: + if c == '\\': + # Don't know if we need to double yet. + bs_buf.append(c) + elif c == '"': + # Double backslashes. + result.append('\\' * len(bs_buf) * 2) + bs_buf = [] + result.append('\\"') + else: + # Normal char + if bs_buf: + result.extend(bs_buf) + bs_buf = [] + result.append(c) + + # Add remaining backslashes, if any. + if bs_buf: + result.extend(bs_buf) + + if needquote: + result.extend(bs_buf) + result.append('"') + + return ''.join(result) + + +class Popen(object): + + def __init__(self, args, bufsize=0, executable=None, + stdin=None, stdout=None, stderr=None, + preexec_fn=None, close_fds=False, shell=False, + cwd=None, env=None, universal_newlines=False, + startupinfo=None, creationflags=0): + """Create new Popen instance.""" + _cleanup() + + self._child_created = False + if not isinstance(bufsize, (int, long)): + raise TypeError("bufsize must be an integer") + + if mswindows: + if preexec_fn is not None: + raise ValueError("preexec_fn is not supported on Windows " + "platforms") + if close_fds and (stdin is not None or stdout is not None or + stderr is not None): + raise ValueError("close_fds is not supported on Windows " + "platforms if you redirect " + "stdin/stdout/stderr") + else: + # POSIX + if startupinfo is not None: + raise ValueError("startupinfo is only supported on Windows " + "platforms") + if creationflags != 0: + raise ValueError("creationflags is only supported on Windows " + "platforms") + + self.stdin = None + self.stdout = None + self.stderr = None + self.pid = None + self.returncode = None + self.universal_newlines = universal_newlines + + # Input and output objects. The general principle is like + # this: + # + # Parent Child + # ------ ----- + # p2cwrite ---stdin---> p2cread + # c2pread <--stdout--- c2pwrite + # errread <--stderr--- errwrite + # + # On POSIX, the child objects are file descriptors. On + # Windows, these are Windows file handles. The parent objects + # are file descriptors on both platforms. The parent objects + # are None when not using PIPEs. The child objects are None + # when not redirecting. + + (p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) = self._get_handles(stdin, stdout, stderr) + + self._execute_child(args, executable, preexec_fn, close_fds, + cwd, env, universal_newlines, + startupinfo, creationflags, shell, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) + + if mswindows: + if p2cwrite is not None: + p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) + if c2pread is not None: + c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) + if errread is not None: + errread = msvcrt.open_osfhandle(errread.Detach(), 0) + + if p2cwrite is not None: + self.stdin = os.fdopen(p2cwrite, 'wb', bufsize) + if c2pread is not None: + if universal_newlines: + self.stdout = os.fdopen(c2pread, 'rU', bufsize) + else: + self.stdout = os.fdopen(c2pread, 'rb', bufsize) + if errread is not None: + if universal_newlines: + self.stderr = os.fdopen(errread, 'rU', bufsize) + else: + self.stderr = os.fdopen(errread, 'rb', bufsize) + + def _translate_newlines(self, data): + data = data.replace("\r\n", "\n") + data = data.replace("\r", "\n") + return data + + def __del__(self, _maxint=sys.maxint, _active=_active): + # If __init__ hasn't had a chance to execute (e.g. if it + # was passed an undeclared keyword argument), we don't + # have a _child_created attribute at all. + if not getattr(self, '_child_created', False): + # We didn't get to successfully create a child process. + return + # In case the child hasn't been waited on, check if it's done. + self._internal_poll(_deadstate=_maxint) + if self.returncode is None and _active is not None: + # Child is still running, keep us alive until we can wait on it. + _active.append(self) + + def communicate(self, input=None): + """Interact with process: Send data to stdin. Read data from + stdout and stderr, until end-of-file is reached. Wait for + process to terminate. The optional input argument should be a + string to be sent to the child process, or None, if no data + should be sent to the child. + + communicate() returns a tuple (stdout, stderr).""" + + # Optimization: If we are only using one pipe, or no pipe at + # all, using select() or threads is unnecessary. + if [self.stdin, self.stdout, self.stderr].count(None) >= 2: + stdout = None + stderr = None + if self.stdin: + if input: + try: + self.stdin.write(input) + except IOError, e: + if e.errno != errno.EPIPE and e.errno != errno.EINVAL: + raise + self.stdin.close() + elif self.stdout: + stdout = _eintr_retry_call(self.stdout.read) + self.stdout.close() + elif self.stderr: + stderr = _eintr_retry_call(self.stderr.read) + self.stderr.close() + self.wait() + return (stdout, stderr) + + return self._communicate(input) + + def poll(self): + return self._internal_poll() + + if mswindows: + # + # Windows methods + # + def _get_handles(self, stdin, stdout, stderr): + """Construct and return tuple with IO objects: + p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite + """ + if stdin is None and stdout is None and stderr is None: + return (None, None, None, None, None, None) + + p2cread, p2cwrite = None, None + c2pread, c2pwrite = None, None + errread, errwrite = None, None + + if stdin is None: + p2cread = _subprocess.GetStdHandle( + _subprocess.STD_INPUT_HANDLE) + if p2cread is None: + p2cread, _ = _subprocess.CreatePipe(None, 0) + elif stdin == PIPE: + p2cread, p2cwrite = _subprocess.CreatePipe(None, 0) + elif isinstance(stdin, int): + p2cread = msvcrt.get_osfhandle(stdin) + else: + # Assuming file-like object + p2cread = msvcrt.get_osfhandle(stdin.fileno()) + p2cread = self._make_inheritable(p2cread) + + if stdout is None: + c2pwrite = _subprocess.GetStdHandle( + _subprocess.STD_OUTPUT_HANDLE) + if c2pwrite is None: + _, c2pwrite = _subprocess.CreatePipe(None, 0) + elif stdout == PIPE: + c2pread, c2pwrite = _subprocess.CreatePipe(None, 0) + elif isinstance(stdout, int): + c2pwrite = msvcrt.get_osfhandle(stdout) + else: + # Assuming file-like object + c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) + c2pwrite = self._make_inheritable(c2pwrite) + + if stderr is None: + errwrite = _subprocess.GetStdHandle( + _subprocess.STD_ERROR_HANDLE) + if errwrite is None: + _, errwrite = _subprocess.CreatePipe(None, 0) + elif stderr == PIPE: + errread, errwrite = _subprocess.CreatePipe(None, 0) + elif stderr == STDOUT: + errwrite = c2pwrite + elif isinstance(stderr, int): + errwrite = msvcrt.get_osfhandle(stderr) + else: + # Assuming file-like object + errwrite = msvcrt.get_osfhandle(stderr.fileno()) + errwrite = self._make_inheritable(errwrite) + + return (p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) + + def _make_inheritable(self, handle): + """Return a duplicate of handle, which is inheritable""" + return _subprocess.DuplicateHandle( + _subprocess.GetCurrentProcess(), + handle, + _subprocess.GetCurrentProcess(), + 0, + 1, + _subprocess.DUPLICATE_SAME_ACCESS + ) + + def _find_w9xpopen(self): + """Find and return absolut path to w9xpopen.exe""" + w9xpopen = os.path.join( + os.path.dirname(_subprocess.GetModuleFileName(0)), + "w9xpopen.exe") + if not os.path.exists(w9xpopen): + # Eeek - file-not-found - possibly an embedding + # situation - see if we can locate it in sys.exec_prefix + w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix), + "w9xpopen.exe") + if not os.path.exists(w9xpopen): + raise RuntimeError("Cannot locate w9xpopen.exe, which is " + "needed for Popen to work with your " + "shell or platform.") + return w9xpopen + + def _execute_child(self, args, executable, preexec_fn, close_fds, + cwd, env, universal_newlines, + startupinfo, creationflags, shell, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite): + """Execute program (MS Windows version)""" + + if not isinstance(args, types.StringTypes): + args = list2cmdline(args) + + # Process startup details + if startupinfo is None: + startupinfo = STARTUPINFO() + if None not in (p2cread, c2pwrite, errwrite): + startupinfo.dwFlags |= _subprocess.STARTF_USESTDHANDLES + startupinfo.hStdInput = p2cread + startupinfo.hStdOutput = c2pwrite + startupinfo.hStdError = errwrite + + if shell: + startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW + startupinfo.wShowWindow = _subprocess.SW_HIDE + comspec = os.environ.get("COMSPEC", "cmd.exe") + args = '{} /c "{}"'.format(comspec, args) + if (_subprocess.GetVersion() >= 0x80000000 or + os.path.basename(comspec).lower() == "command.com"): + # Win9x, or using command.com on NT. We need to + # use the w9xpopen intermediate program. For more + # information, see KB Q150956 + # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp) + w9xpopen = self._find_w9xpopen() + args = '"%s" %s' % (w9xpopen, args) + # Not passing CREATE_NEW_CONSOLE has been known to + # cause random failures on win9x. Specifically a + # dialog: "Your program accessed mem currently in + # use at xxx" and a hopeful warning about the + # stability of your system. Cost is Ctrl+C wont + # kill children. + creationflags |= _subprocess.CREATE_NEW_CONSOLE + + # Start the process + try: + try: + hp, ht, pid, tid = _subprocess.CreateProcess( + executable, args, + # no special + # security + None, None, + int(not close_fds), + creationflags, + env, + cwd, + startupinfo) + except pywintypes.error, e: + # Translate pywintypes.error to WindowsError, which is + # a subclass of OSError. FIXME: We should really + # translate errno using _sys_errlist (or similar), but + # how can this be done from Python? + raise WindowsError(*e.args) + finally: + # Child is launched. Close the parent's copy of those pipe + # handles that only the child should have open. You need + # to make sure that no handles to the write end of the + # output pipe are maintained in this process or else the + # pipe will not close when the child process exits and the + # ReadFile will hang. + if p2cread is not None: + p2cread.Close() + if c2pwrite is not None: + c2pwrite.Close() + if errwrite is not None: + errwrite.Close() + + # Retain the process handle, but close the thread handle + self._child_created = True + self._handle = hp + self.pid = pid + ht.Close() + + def _internal_poll( + self, _deadstate=None, + _WaitForSingleObject=_subprocess.WaitForSingleObject, + _WAIT_OBJECT_0=_subprocess.WAIT_OBJECT_0, + _GetExitCodeProcess=_subprocess.GetExitCodeProcess + ): + """Check if child process has terminated. Returns returncode + attribute. + + This method is called by __del__, so it can only refer to objects + in its local scope. + + """ + if self.returncode is None: + if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0: + self.returncode = _GetExitCodeProcess(self._handle) + return self.returncode + + def wait(self): + """Wait for child process to terminate. Returns returncode + attribute.""" + if self.returncode is None: + _subprocess.WaitForSingleObject(self._handle, + _subprocess.INFINITE) + self.returncode = _subprocess.GetExitCodeProcess(self._handle) + return self.returncode + + def _readerthread(self, fh, buffer): + buffer.append(fh.read()) + + def _communicate(self, input): + stdout = None # Return + stderr = None # Return + + if self.stdout: + stdout = [] + stdout_thread = threading.Thread(target=self._readerthread, + args=(self.stdout, stdout)) + stdout_thread.setDaemon(True) + stdout_thread.start() + if self.stderr: + stderr = [] + stderr_thread = threading.Thread(target=self._readerthread, + args=(self.stderr, stderr)) + stderr_thread.setDaemon(True) + stderr_thread.start() + + if self.stdin: + if input is not None: + try: + self.stdin.write(input) + except IOError, e: + if e.errno != errno.EPIPE: + raise + self.stdin.close() + + if self.stdout: + stdout_thread.join() + if self.stderr: + stderr_thread.join() + + # All data exchanged. Translate lists into strings. + if stdout is not None: + stdout = stdout[0] + if stderr is not None: + stderr = stderr[0] + + # Translate newlines, if requested. We cannot let the file + # object do the translation: It is based on stdio, which is + # impossible to combine with select (unless forcing no + # buffering). + if self.universal_newlines and hasattr(file, 'newlines'): + if stdout: + stdout = self._translate_newlines(stdout) + if stderr: + stderr = self._translate_newlines(stderr) + + self.wait() + return (stdout, stderr) + + def send_signal(self, sig): + """Send a signal to the process + """ + if sig == signal.SIGTERM: + self.terminate() + elif sig == signal.CTRL_C_EVENT: + os.kill(self.pid, signal.CTRL_C_EVENT) + elif sig == signal.CTRL_BREAK_EVENT: + os.kill(self.pid, signal.CTRL_BREAK_EVENT) + else: + raise ValueError("Unsupported signal: {}".format(sig)) + + def terminate(self): + """Terminates the process + """ + _subprocess.TerminateProcess(self._handle, 1) + + kill = terminate + + else: + # + # POSIX methods + # + def _get_handles(self, stdin, stdout, stderr): + """Construct and return tuple with IO objects: + p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite + """ + p2cread, p2cwrite = None, None + c2pread, c2pwrite = None, None + errread, errwrite = None, None + + if stdin is None: + pass + elif stdin == PIPE: + p2cread, p2cwrite = self.pipe_cloexec() + elif isinstance(stdin, int): + p2cread = stdin + else: + # Assuming file-like object + p2cread = stdin.fileno() + + if stdout is None: + pass + elif stdout == PIPE: + c2pread, c2pwrite = self.pipe_cloexec() + elif isinstance(stdout, int): + c2pwrite = stdout + else: + # Assuming file-like object + c2pwrite = stdout.fileno() + + if stderr is None: + pass + elif stderr == PIPE: + errread, errwrite = self.pipe_cloexec() + elif stderr == STDOUT: + errwrite = c2pwrite + elif isinstance(stderr, int): + errwrite = stderr + else: + # Assuming file-like object + errwrite = stderr.fileno() + + return (p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite) + + def _set_cloexec_flag(self, fd, cloexec=True): + try: + cloexec_flag = fcntl.FD_CLOEXEC + except AttributeError: + cloexec_flag = 1 + + old = fcntl.fcntl(fd, fcntl.F_GETFD) + if cloexec: + fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) + else: + fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) + + def pipe_cloexec(self): + """Create a pipe with FDs set CLOEXEC.""" + # Pipes' FDs are set CLOEXEC by default because we don't want them + # to be inherited by other subprocesses: the CLOEXEC flag is + # removed from the child's FDs by _dup2(), between fork() and + # exec(). + # This is not atomic: we would need the pipe2() syscall for that. + r, w = os.pipe() + self._set_cloexec_flag(r) + self._set_cloexec_flag(w) + return r, w + + def _close_fds(self, but): + if hasattr(os, 'closerange'): + os.closerange(3, but) + os.closerange(but + 1, MAXFD) + else: + for i in xrange(3, MAXFD): + if i == but: + continue + try: + os.close(i) + except: + pass + + def _execute_child(self, args, executable, preexec_fn, close_fds, + cwd, env, universal_newlines, + startupinfo, creationflags, shell, + p2cread, p2cwrite, + c2pread, c2pwrite, + errread, errwrite): + """Execute program (POSIX version)""" + + if isinstance(args, types.StringTypes): + args = [args] + else: + args = list(args) + + if shell: + args = ["/bin/sh", "-c"] + args + if executable: + args[0] = executable + + if executable is None: + executable = args[0] + + # For transferring possible exec failure from child to parent + # The first char specifies the exception type: 0 means + # OSError, 1 means some other error. + errpipe_read, errpipe_write = self.pipe_cloexec() + try: + try: + gc_was_enabled = gc.isenabled() + # Disable gc to avoid bug where gc -> file_dealloc -> + # write to stderr -> hang. + # http://bugs.python.org/issue1336 + gc.disable() + try: + self.pid = os.fork() + except: + if gc_was_enabled: + gc.enable() + raise + self._child_created = True + if self.pid == 0: + # Child + try: + # Close parent's pipe ends + if p2cwrite is not None: + os.close(p2cwrite) + if c2pread is not None: + os.close(c2pread) + if errread is not None: + os.close(errread) + os.close(errpipe_read) + + # When duping fds, if there arises a situation + # where one of the fds is either 0, 1 or 2, it + # is possible that it is overwritten (#12607). + if c2pwrite == 0: + c2pwrite = os.dup(c2pwrite) + if errwrite == 0 or errwrite == 1: + errwrite = os.dup(errwrite) + + # Dup fds for child + def _dup2(a, b): + # dup2() removes the CLOEXEC flag but + # we must do it ourselves if dup2() + # would be a no-op (issue #10806). + if a == b: + self._set_cloexec_flag(a, False) + elif a is not None: + os.dup2(a, b) + _dup2(p2cread, 0) + _dup2(c2pwrite, 1) + _dup2(errwrite, 2) + + # Close pipe fds. Make sure we don't close the + # same fd more than once, or standard fds. + closed = set([None]) + for fd in [p2cread, c2pwrite, errwrite]: + if fd not in closed and fd > 2: + os.close(fd) + closed.add(fd) + + # Close all other fds, if asked for + if close_fds: + self._close_fds(but=errpipe_write) + + if cwd is not None: + os.chdir(cwd) + + if preexec_fn: + preexec_fn() + + if env is None: + os.execvp(executable, args) + else: + os.execvpe(executable, args, env) + + except: + exc_type, exc_value, tb = sys.exc_info() + # Save the traceback and attach it to the exception + # object + exc_lines = traceback.format_exception(exc_type, + exc_value, + tb) + exc_value.child_traceback = ''.join(exc_lines) + os.write(errpipe_write, pickle.dumps(exc_value)) + + # This exitcode won't be reported to applications, + # so it really doesn't matter what we return. + os._exit(255) + + # Parent + if gc_was_enabled: + gc.enable() + finally: + # be sure the FD is closed no matter what + os.close(errpipe_write) + + if p2cread is not None and p2cwrite is not None: + os.close(p2cread) + if c2pwrite is not None and c2pread is not None: + os.close(c2pwrite) + if errwrite is not None and errread is not None: + os.close(errwrite) + + # Wait for exec to fail or succeed; possibly raising exception + # Exception limited to 1M + data = _eintr_retry_call(os.read, errpipe_read, 1048576) + finally: + # be sure the FD is closed no matter what + os.close(errpipe_read) + + if data != "": + try: + _eintr_retry_call(os.waitpid, self.pid, 0) + except OSError, e: + if e.errno != errno.ECHILD: + raise + child_exception = pickle.loads(data) + for fd in (p2cwrite, c2pread, errread): + if fd is not None: + os.close(fd) + raise child_exception + + def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED, + _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED, + _WEXITSTATUS=os.WEXITSTATUS): + # This method is called (indirectly) by __del__, so it cannot + # refer to anything outside of its local scope.""" + if _WIFSIGNALED(sts): + self.returncode = -_WTERMSIG(sts) + elif _WIFEXITED(sts): + self.returncode = _WEXITSTATUS(sts) + else: + # Should never happen + raise RuntimeError("Unknown child exit status!") + + def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid, + _WNOHANG=os.WNOHANG, _os_error=os.error): + """Check if child process has terminated. Returns returncode + attribute. + + This method is called by __del__, so it cannot reference anything + outside of the local scope (nor can any methods it calls). + + """ + if self.returncode is None: + try: + pid, sts = _waitpid(self.pid, _WNOHANG) + if pid == self.pid: + self._handle_exitstatus(sts) + except _os_error: + if _deadstate is not None: + self.returncode = _deadstate + return self.returncode + + def wait(self): + """Wait for child process to terminate. Returns returncode + attribute.""" + if self.returncode is None: + try: + pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) + except OSError, e: + if e.errno != errno.ECHILD: + raise + # This happens if SIGCLD is set to be ignored or waiting + # for child processes has otherwise been disabled for our + # process. This child is dead, we can't get the status. + sts = 0 + self._handle_exitstatus(sts) + return self.returncode + + def _communicate(self, input): + if self.stdin: + # Flush stdio buffer. This might block, if the user has + # been writing to .stdin in an uncontrolled fashion. + self.stdin.flush() + if not input: + self.stdin.close() + + if _has_poll: + stdout, stderr = self._communicate_with_poll(input) + else: + stdout, stderr = self._communicate_with_select(input) + + # All data exchanged. Translate lists into strings. + if stdout is not None: + stdout = ''.join(stdout) + if stderr is not None: + stderr = ''.join(stderr) + + # Translate newlines, if requested. We cannot let the file + # object do the translation: It is based on stdio, which is + # impossible to combine with select (unless forcing no + # buffering). + if self.universal_newlines and hasattr(file, 'newlines'): + if stdout: + stdout = self._translate_newlines(stdout) + if stderr: + stderr = self._translate_newlines(stderr) + + self.wait() + return (stdout, stderr) + + def _communicate_with_poll(self, input): + stdout = None # Return + stderr = None # Return + fd2file = {} + fd2output = {} + + poller = select.poll() + + def register_and_append(file_obj, eventmask): + poller.register(file_obj.fileno(), eventmask) + fd2file[file_obj.fileno()] = file_obj + + def close_unregister_and_remove(fd): + poller.unregister(fd) + fd2file[fd].close() + fd2file.pop(fd) + + if self.stdin and input: + register_and_append(self.stdin, select.POLLOUT) + + select_POLLIN_POLLPRI = select.POLLIN | select.POLLPRI + if self.stdout: + register_and_append(self.stdout, select_POLLIN_POLLPRI) + fd2output[self.stdout.fileno()] = stdout = [] + if self.stderr: + register_and_append(self.stderr, select_POLLIN_POLLPRI) + fd2output[self.stderr.fileno()] = stderr = [] + + input_offset = 0 + while fd2file: + try: + ready = poller.poll() + except select.error, e: + if e.args[0] == errno.EINTR: + continue + raise + + for fd, mode in ready: + if mode & select.POLLOUT: + chunk = input[input_offset: input_offset + _PIPE_BUF] + try: + input_offset += os.write(fd, chunk) + except OSError, e: + if e.errno == errno.EPIPE: + close_unregister_and_remove(fd) + else: + raise + else: + if input_offset >= len(input): + close_unregister_and_remove(fd) + elif mode & select_POLLIN_POLLPRI: + data = os.read(fd, 4096) + if not data: + close_unregister_and_remove(fd) + fd2output[fd].append(data) + else: + # Ignore hang up or errors. + close_unregister_and_remove(fd) + + return (stdout, stderr) + + def _communicate_with_select(self, input): + read_set = [] + write_set = [] + stdout = None # Return + stderr = None # Return + + if self.stdin and input: + write_set.append(self.stdin) + if self.stdout: + read_set.append(self.stdout) + stdout = [] + if self.stderr: + read_set.append(self.stderr) + stderr = [] + + input_offset = 0 + while read_set or write_set: + try: + rlist, wlist, xlist = select.select( + read_set, write_set, []) + except select.error, e: + if e.args[0] == errno.EINTR: + continue + raise + + if self.stdin in wlist: + chunk = input[input_offset: input_offset + _PIPE_BUF] + try: + bytes_written = os.write(self.stdin.fileno(), chunk) + except OSError, e: + if e.errno == errno.EPIPE: + self.stdin.close() + write_set.remove(self.stdin) + else: + raise + else: + input_offset += bytes_written + if input_offset >= len(input): + self.stdin.close() + write_set.remove(self.stdin) + + if self.stdout in rlist: + data = os.read(self.stdout.fileno(), 1024) + if data == "": + self.stdout.close() + read_set.remove(self.stdout) + stdout.append(data) + + if self.stderr in rlist: + data = os.read(self.stderr.fileno(), 1024) + if data == "": + self.stderr.close() + read_set.remove(self.stderr) + stderr.append(data) + + return (stdout, stderr) + + def send_signal(self, sig): + """Send a signal to the process + """ + os.kill(self.pid, sig) + + def terminate(self): + """Terminate the process with SIGTERM + """ + self.send_signal(signal.SIGTERM) + + def kill(self): + """Kill the process with SIGKILL + """ + self.send_signal(signal.SIGKILL) + + +def _demo_posix(): + # + # Example 1: Simple redirection: Get process list + # + plist = Popen(["ps"], stdout=PIPE).communicate()[0] + print "Process list:" + print plist + + # + # Example 2: Change uid before executing child + # + if os.getuid() == 0: + p = Popen(["id"], preexec_fn=lambda: os.setuid(100)) + p.wait() + + # + # Example 3: Connecting several subprocesses + # + print "Looking for 'hda'..." + p1 = Popen(["dmesg"], stdout=PIPE) + p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) + print repr(p2.communicate()[0]) + + # + # Example 4: Catch execution error + # + print + print "Trying a weird file..." + try: + print Popen(["/this/path/does/not/exist"]).communicate() + except OSError, e: + if e.errno == errno.ENOENT: + print "The file didn't exist. I thought so..." + print "Child traceback:" + print e.child_traceback + else: + print "Error", e.errno + else: + print >>sys.stderr, "Gosh. No error." + + +def _demo_windows(): + # + # Example 1: Connecting several subprocesses + # + print "Looking for 'PROMPT' in set output..." + p1 = Popen("set", stdout=PIPE, shell=True) + p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE) + print repr(p2.communicate()[0]) + + # + # Example 2: Simple execution of program + # + print "Executing calc..." + p = Popen("calc") + p.wait() + + +if __name__ == "__main__": + if mswindows: + _demo_windows() + else: + _demo_posix() diff --git a/lib/cherrypy/_cpconfig.py b/lib/cherrypy/_cpconfig.py index e2b7dee0..c11bc1d1 100644 --- a/lib/cherrypy/_cpconfig.py +++ b/lib/cherrypy/_cpconfig.py @@ -125,6 +125,7 @@ from cherrypy.lib import reprconf # Deprecated in CherryPy 3.2--remove in 3.3 NamespaceSet = reprconf.NamespaceSet + def merge(base, other): """Merge one app config (from a dict, file, or filename) into another. @@ -146,6 +147,7 @@ def merge(base, other): class Config(reprconf.Config): + """The 'global' configuration data for the entire CherryPy process.""" def update(self, config): @@ -157,7 +159,7 @@ class Config(reprconf.Config): def _apply(self, config): """Update self from a dict.""" - if isinstance(config.get("global", None), dict): + if isinstance(config.get("global"), dict): if len(config) > 1: cherrypy.checker.global_config_contained_paths = True config = config["global"] @@ -171,6 +173,7 @@ class Config(reprconf.Config): raise TypeError( "The cherrypy.config decorator does not accept positional " "arguments; you must use keyword arguments.") + def tool_decorator(f): if not hasattr(f, "_cp_config"): f._cp_config = {} @@ -180,25 +183,26 @@ class Config(reprconf.Config): return tool_decorator +# Sphinx begin config.environments Config.environments = environments = { "staging": { - 'engine.autoreload_on': False, + 'engine.autoreload.on': False, 'checker.on': False, 'tools.log_headers.on': False, 'request.show_tracebacks': False, 'request.show_mismatched_params': False, - }, + }, "production": { - 'engine.autoreload_on': False, + 'engine.autoreload.on': False, 'checker.on': False, 'tools.log_headers.on': False, 'request.show_tracebacks': False, 'request.show_mismatched_params': False, 'log.screen': False, - }, + }, "embedded": { # For use with CherryPy embedded in another deployment stack. - 'engine.autoreload_on': False, + 'engine.autoreload.on': False, 'checker.on': False, 'tools.log_headers.on': False, 'request.show_tracebacks': False, @@ -206,16 +210,17 @@ Config.environments = environments = { 'log.screen': False, 'engine.SIGHUP': None, 'engine.SIGTERM': None, - }, + }, "test_suite": { - 'engine.autoreload_on': False, + 'engine.autoreload.on': False, 'checker.on': False, 'tools.log_headers.on': False, 'request.show_tracebacks': True, 'request.show_mismatched_params': True, 'log.screen': False, - }, - } + }, +} +# Sphinx end config.environments def _server_namespace_handler(k, v): @@ -245,9 +250,24 @@ def _server_namespace_handler(k, v): setattr(cherrypy.server, k, v) Config.namespaces["server"] = _server_namespace_handler + def _engine_namespace_handler(k, v): """Backward compatibility handler for the "engine" namespace.""" engine = cherrypy.engine + + deprecated = { + 'autoreload_on': 'autoreload.on', + 'autoreload_frequency': 'autoreload.frequency', + 'autoreload_match': 'autoreload.match', + 'reload_files': 'autoreload.files', + 'deadlock_poll_freq': 'timeout_monitor.frequency' + } + + if k in deprecated: + engine.log( + 'WARNING: Use of engine.%s is deprecated and will be removed in a ' + 'future version. Use engine.%s instead.' % (k, deprecated[k])) + if k == 'autoreload_on': if v: engine.autoreload.subscribe() @@ -272,7 +292,10 @@ def _engine_namespace_handler(k, v): if v and hasattr(getattr(plugin, 'subscribe', None), '__call__'): plugin.subscribe() return - elif (not v) and hasattr(getattr(plugin, 'unsubscribe', None), '__call__'): + elif ( + (not v) and + hasattr(getattr(plugin, 'unsubscribe', None), '__call__') + ): plugin.unsubscribe() return setattr(plugin, attrname, v) @@ -286,10 +309,9 @@ def _tree_namespace_handler(k, v): if isinstance(v, dict): for script_name, app in v.items(): cherrypy.tree.graft(app, script_name) - cherrypy.engine.log("Mounted: %s on %s" % (app, script_name or "/")) + cherrypy.engine.log("Mounted: %s on %s" % + (app, script_name or "/")) else: cherrypy.tree.graft(v, v.script_name) cherrypy.engine.log("Mounted: %s on %s" % (v, v.script_name or "/")) Config.namespaces["tree"] = _tree_namespace_handler - - diff --git a/lib/cherrypy/_cpdispatch.py b/lib/cherrypy/_cpdispatch.py index e92d9306..1c2d7df8 100644 --- a/lib/cherrypy/_cpdispatch.py +++ b/lib/cherrypy/_cpdispatch.py @@ -22,6 +22,7 @@ from cherrypy._cpcompat import set class PageHandler(object): + """Callable which sets response.body.""" def __init__(self, callable, *args, **kwargs): @@ -29,6 +30,32 @@ class PageHandler(object): self.args = args self.kwargs = kwargs + def get_args(self): + return cherrypy.serving.request.args + + def set_args(self, args): + cherrypy.serving.request.args = args + return cherrypy.serving.request.args + + args = property( + get_args, + set_args, + doc="The ordered args should be accessible from post dispatch hooks" + ) + + def get_kwargs(self): + return cherrypy.serving.request.kwargs + + def set_kwargs(self, kwargs): + cherrypy.serving.request.kwargs = kwargs + return cherrypy.serving.request.kwargs + + kwargs = property( + get_kwargs, + set_kwargs, + doc="The named kwargs should be accessible from post dispatch hooks" + ) + def __call__(self): try: return self.callable(*self.args, **self.kwargs) @@ -54,7 +81,8 @@ def test_callable_spec(callable, callable_args, callable_kwargs): 2. Too little parameters are passed to the function. There are 3 sources of parameters to a cherrypy handler. - 1. query string parameters are passed as keyword parameters to the handler. + 1. query string parameters are passed as keyword parameters to the + handler. 2. body parameters are also passed as keyword parameters. 3. when partial matching occurs, the final path atoms are passed as positional args. @@ -65,10 +93,11 @@ def test_callable_spec(callable, callable_args, callable_kwargs): show_mismatched_params = getattr( cherrypy.serving.request, 'show_mismatched_params', False) try: - (args, varargs, varkw, defaults) = inspect.getargspec(callable) + (args, varargs, varkw, defaults) = getargspec(callable) except TypeError: if isinstance(callable, object) and hasattr(callable, '__call__'): - (args, varargs, varkw, defaults) = inspect.getargspec(callable.__call__) + (args, varargs, varkw, + defaults) = getargspec(callable.__call__) else: # If it wasn't one of our own types, re-raise # the original error @@ -125,7 +154,7 @@ def test_callable_spec(callable, callable_args, callable_kwargs): # arguments it's definitely a 404. message = None if show_mismatched_params: - message="Missing parameters: %s" % ",".join(missing_args) + message = "Missing parameters: %s" % ",".join(missing_args) raise cherrypy.HTTPError(404, message=message) # the extra positional arguments come from the path - 404 Not Found @@ -147,8 +176,8 @@ def test_callable_spec(callable, callable_args, callable_kwargs): message = None if show_mismatched_params: - message="Multiple values for parameters: "\ - "%s" % ",".join(multiple_args) + message = "Multiple values for parameters: "\ + "%s" % ",".join(multiple_args) raise cherrypy.HTTPError(error, message=message) if not varkw and varkw_usage > 0: @@ -158,8 +187,8 @@ def test_callable_spec(callable, callable_args, callable_kwargs): if extra_qs_params: message = None if show_mismatched_params: - message="Unexpected query string "\ - "parameters: %s" % ", ".join(extra_qs_params) + message = "Unexpected query string "\ + "parameters: %s" % ", ".join(extra_qs_params) raise cherrypy.HTTPError(404, message=message) # If there were any extra body parameters, it's a 400 Not Found @@ -167,8 +196,8 @@ def test_callable_spec(callable, callable_args, callable_kwargs): if extra_body_params: message = None if show_mismatched_params: - message="Unexpected body parameters: "\ - "%s" % ", ".join(extra_body_params) + message = "Unexpected body parameters: "\ + "%s" % ", ".join(extra_body_params) raise cherrypy.HTTPError(400, message=message) @@ -176,10 +205,16 @@ try: import inspect except ImportError: test_callable_spec = lambda callable, args, kwargs: None - +else: + getargspec = inspect.getargspec + # Python 3 requires using getfullargspec if keyword-only arguments are present + if hasattr(inspect, 'getfullargspec'): + def getargspec(callable): + return inspect.getfullargspec(callable)[:4] class LateParamPageHandler(PageHandler): + """When passing cherrypy.request.params to the page handler, we do not want to capture that dict too early; we want to give tools like the decoding tool a chance to modify the params dict in-between the lookup @@ -195,6 +230,7 @@ class LateParamPageHandler(PageHandler): return kwargs def _set_kwargs(self, kwargs): + cherrypy.serving.request.kwargs = kwargs self._kwargs = kwargs kwargs = property(_get_kwargs, _set_kwargs, @@ -205,17 +241,22 @@ class LateParamPageHandler(PageHandler): if sys.version_info < (3, 0): punctuation_to_underscores = string.maketrans( string.punctuation, '_' * len(string.punctuation)) + def validate_translator(t): if not isinstance(t, str) or len(t) != 256: - raise ValueError("The translate argument must be a str of len 256.") + raise ValueError( + "The translate argument must be a str of len 256.") else: punctuation_to_underscores = str.maketrans( string.punctuation, '_' * len(string.punctuation)) + def validate_translator(t): if not isinstance(t, dict): raise ValueError("The translate argument must be a dict.") + class Dispatcher(object): + """CherryPy Dispatcher which walks a tree of objects to find a handler. The tree is rooted at cherrypy.request.app.root, and each hierarchical @@ -304,31 +345,31 @@ class Dispatcher(object): if dispatch and hasattr(dispatch, '__call__') and not \ getattr(dispatch, 'exposed', False) and \ pre_len > 1: - #Don't expose the hidden 'index' token to _cp_dispatch - #We skip this if pre_len == 1 since it makes no sense - #to call a dispatcher when we have no tokens left. + # Don't expose the hidden 'index' token to _cp_dispatch + # We skip this if pre_len == 1 since it makes no sense + # to call a dispatcher when we have no tokens left. index_name = iternames.pop() subnode = dispatch(vpath=iternames) iternames.append(index_name) else: - #We didn't find a path, but keep processing in case there - #is a default() handler. + # We didn't find a path, but keep processing in case there + # is a default() handler. iternames.pop(0) else: - #We found the path, remove the vpath entry + # We found the path, remove the vpath entry iternames.pop(0) segleft = len(iternames) if segleft > pre_len: - #No path segment was removed. Raise an error. + # No path segment was removed. Raise an error. raise cherrypy.CherryPyException( "A vpath segment was added. Custom dispatchers may only " + "remove elements. While trying to process " + "{0} in {1}".format(name, fullpath) - ) + ) elif segleft == pre_len: - #Assume that the handler used the current path segment, but - #did not pop it. This allows things like - #return getattr(self, vpath[0], None) + # Assume that the handler used the current path segment, but + # did not pop it. This allows things like + # return getattr(self, vpath[0], None) iternames.pop(0) segleft -= 1 node = subnode @@ -353,14 +394,16 @@ class Dispatcher(object): object_trail.append([name, node, nodeconf, segleft]) def set_conf(): - """Collapse all object_trail config into cherrypy.request.config.""" + """Collapse all object_trail config into cherrypy.request.config. + """ base = cherrypy.config.copy() # Note that we merge the config from each node # even if that node was None. for name, obj, conf, segleft in object_trail: base.update(conf) if 'tools.staticdir.dir' in conf: - base['tools.staticdir.section'] = '/' + '/'.join(fullpath[0:fullpath_len - segleft]) + base['tools.staticdir.section'] = '/' + \ + '/'.join(fullpath[0:fullpath_len - segleft]) return base # Try successive objects (reverse order) @@ -377,13 +420,15 @@ class Dispatcher(object): if getattr(defhandler, 'exposed', False): # Insert any extra _cp_config from the default handler. conf = getattr(defhandler, "_cp_config", {}) - object_trail.insert(i+1, ["default", defhandler, conf, segleft]) + object_trail.insert( + i + 1, ["default", defhandler, conf, segleft]) request.config = set_conf() - # See http://www.cherrypy.org/ticket/613 + # See https://bitbucket.org/cherrypy/cherrypy/issue/613 request.is_index = path.endswith("/") return defhandler, fullpath[fullpath_len - segleft:-1] - # Uncomment the next line to restrict positional params to "default". + # Uncomment the next line to restrict positional params to + # "default". # if i < num_candidates - 2: continue # Try the current leaf. @@ -407,6 +452,7 @@ class Dispatcher(object): class MethodDispatcher(Dispatcher): + """Additional dispatch based on cherrypy.request.method.upper(). Methods named GET, POST, etc will be called on an exposed class. @@ -450,9 +496,10 @@ class MethodDispatcher(Dispatcher): class RoutesDispatcher(object): + """A Routes based dispatcher for CherryPy.""" - def __init__(self, full_result=False): + def __init__(self, full_result=False, **mapper_options): """ Routes dispatcher @@ -463,7 +510,7 @@ class RoutesDispatcher(object): import routes self.full_result = full_result self.controllers = {} - self.mapper = routes.Mapper() + self.mapper = routes.Mapper(**mapper_options) self.mapper.controller_scan = self.controllers.keys def connect(self, name, route, controller, **kwargs): @@ -565,13 +612,15 @@ class RoutesDispatcher(object): def XMLRPCDispatcher(next_dispatcher=Dispatcher()): from cherrypy.lib import xmlrpcutil + def xmlrpc_dispatch(path_info): path_info = xmlrpcutil.patched_path(path_info) return next_dispatcher(path_info) return xmlrpc_dispatch -def VirtualHost(next_dispatcher=Dispatcher(), use_x_forwarded_host=True, **domains): +def VirtualHost(next_dispatcher=Dispatcher(), use_x_forwarded_host=True, + **domains): """ Select a different handler based on the Host header. @@ -611,6 +660,7 @@ def VirtualHost(next_dispatcher=Dispatcher(), use_x_forwarded_host=True, **domai headers may contain the port number. """ from cherrypy.lib import httputil + def vhost_dispatch(path_info): request = cherrypy.serving.request header = request.headers.get @@ -625,7 +675,8 @@ def VirtualHost(next_dispatcher=Dispatcher(), use_x_forwarded_host=True, **domai result = next_dispatcher(path_info) - # Touch up staticdir config. See http://www.cherrypy.org/ticket/614. + # Touch up staticdir config. See + # https://bitbucket.org/cherrypy/cherrypy/issue/614. section = request.config.get('tools.staticdir.section') if section: section = section[len(prefix):] @@ -633,4 +684,3 @@ def VirtualHost(next_dispatcher=Dispatcher(), use_x_forwarded_host=True, **domai return result return vhost_dispatch - diff --git a/lib/cherrypy/_cperror.py b/lib/cherrypy/_cperror.py index 3a60d150..6256595b 100644 --- a/lib/cherrypy/_cperror.py +++ b/lib/cherrypy/_cperror.py @@ -2,8 +2,9 @@ CherryPy provides (and uses) exceptions for declaring that the HTTP response should be a status other than the default "200 OK". You can ``raise`` them like -normal Python exceptions. You can also call them and they will raise themselves; -this means you can set an :class:`HTTPError` +normal Python exceptions. You can also call them and they will raise +themselves; this means you can set an +:class:`HTTPError` or :class:`HTTPRedirect` as the :attr:`request.handler`. @@ -21,7 +22,8 @@ POST, however, is neither safe nor idempotent--if you charge a credit card, you don't want to be charged twice by a redirect! For this reason, *none* of the 3xx responses permit a user-agent (browser) to -resubmit a POST on redirection without first confirming the action with the user: +resubmit a POST on redirection without first confirming the action with the +user: ===== ================================= =========== 300 Multiple Choices Confirm with the user @@ -53,14 +55,16 @@ Anticipated HTTP responses -------------------------- The 'error_page' config namespace can be used to provide custom HTML output for -expected responses (like 404 Not Found). Supply a filename from which the output -will be read. The contents will be interpolated with the values %(status)s, -%(message)s, %(traceback)s, and %(version)s using plain old Python -`string formatting `_. +expected responses (like 404 Not Found). Supply a filename from which the +output will be read. The contents will be interpolated with the values +%(status)s, %(message)s, %(traceback)s, and %(version)s using plain old Python +`string formatting `_. :: - _cp_config = {'error_page.404': os.path.join(localDir, "static/index.html")} + _cp_config = { + 'error_page.404': os.path.join(localDir, "static/index.html") + } Beginning in version 3.1, you may also provide a function or other callable as @@ -72,7 +76,8 @@ version arguments that are interpolated into templates:: cherrypy.config.update({'error_page.402': error_page_402}) Also in 3.1, in addition to the numbered error codes, you may also supply -"error_page.default" to handle all codes which do not have their own error_page entry. +"error_page.default" to handle all codes which do not have their own error_page +entry. @@ -81,8 +86,9 @@ Unanticipated errors CherryPy also has a generic error handling mechanism: whenever an unanticipated error occurs in your code, it will call -:func:`Request.error_response` to set -the response status, headers, and body. By default, this is the same output as +:func:`Request.error_response` to +set the response status, headers, and body. By default, this is the same +output as :class:`HTTPError(500) `. If you want to provide some other behavior, you generally replace "request.error_response". @@ -93,40 +99,50 @@ send an e-mail containing the error:: def handle_error(): cherrypy.response.status = 500 - cherrypy.response.body = ["Sorry, an error occured"] - sendMail('error@domain.com', 'Error in your web app', _cperror.format_exc()) + cherrypy.response.body = [ + "Sorry, an error occured" + ] + sendMail('error@domain.com', + 'Error in your web app', + _cperror.format_exc()) class Root: _cp_config = {'request.error_response': handle_error} -Note that you have to explicitly set :attr:`response.body ` +Note that you have to explicitly set +:attr:`response.body ` and not simply return an error message as a result. """ from cgi import escape as _escape from sys import exc_info as _exc_info from traceback import format_exception as _format_exception -from cherrypy._cpcompat import basestring, bytestr, iteritems, ntob, tonative, urljoin as _urljoin +from cherrypy._cpcompat import basestring, bytestr, iteritems, ntob +from cherrypy._cpcompat import tonative, urljoin as _urljoin from cherrypy.lib import httputil as _httputil class CherryPyException(Exception): + """A base class for CherryPy exceptions.""" pass class TimeoutError(CherryPyException): + """Exception raised when Response.timed_out is detected.""" pass class InternalRedirect(CherryPyException): + """Exception raised to switch to the handler for a different URL. This exception will redirect processing to another path within the site (without informing the client). Provide the new path as an argument when - raising the exception. Provide any params in the querystring for the new URL. + raising the exception. Provide any params in the querystring for the new + URL. """ def __init__(self, path, query_string=""): @@ -152,6 +168,7 @@ class InternalRedirect(CherryPyException): class HTTPRedirect(CherryPyException): + """Exception raised when the request should be redirected. This exception will force a HTTP redirect to the URL or URL's you give it. @@ -222,7 +239,8 @@ class HTTPRedirect(CherryPyException): CherryPyException.__init__(self, abs_urls, status) def set_response(self): - """Modify cherrypy.response status, headers, and body to represent self. + """Modify cherrypy.response status, headers, and body to represent + self. CherryPy uses this internally, but you can also use it to create an HTTPRedirect object and set its output without *raising* the exception. @@ -240,13 +258,16 @@ class HTTPRedirect(CherryPyException): # "Unless the request method was HEAD, the entity of the response # SHOULD contain a short hypertext note with a hyperlink to the # new URI(s)." - msg = {300: "This resource can be found at %s.", - 301: "This resource has permanently moved to %s.", - 302: "This resource resides temporarily at %s.", - 303: "This resource can be found at %s.", - 307: "This resource has moved temporarily to %s.", - }[status] - msgs = [msg % (u, u) for u in self.urls] + msg = { + 300: "This resource can be found at ", + 301: "This resource has permanently moved to ", + 302: "This resource resides temporarily at ", + 303: "This resource can be found at ", + 307: "This resource has moved temporarily to ", + }[status] + msg += '%s.' + from xml.sax import saxutils + msgs = [msg % (saxutils.quoteattr(u), u) for u in self.urls] response.body = ntob("
    \n".join(msgs), 'utf-8') # Previous code may have set C-L, so we have to reset it # (allow finalize to set it). @@ -311,24 +332,27 @@ def clean_headers(status): class HTTPError(CherryPyException): + """Exception used to return an HTTP error code (4xx-5xx) to the client. - This exception can be used to automatically send a response using a http status - code, with an appropriate error page. It takes an optional + This exception can be used to automatically send a response using a + http status code, with an appropriate error page. It takes an optional ``status`` argument (which must be between 400 and 599); it defaults to 500 ("Internal Server Error"). It also takes an optional ``message`` argument, which will be returned in the response body. See - `RFC 2616 `_ + `RFC2616 `_ for a complete list of available error codes and when to use them. Examples:: raise cherrypy.HTTPError(403) - raise cherrypy.HTTPError("403 Forbidden", "You are not allowed to access this resource.") + raise cherrypy.HTTPError( + "403 Forbidden", "You are not allowed to access this resource.") """ status = None - """The HTTP status code. May be of type int or str (with a Reason-Phrase).""" + """The HTTP status code. May be of type int or str (with a Reason-Phrase). + """ code = None """The integer HTTP status code.""" @@ -352,7 +376,8 @@ class HTTPError(CherryPyException): CherryPyException.__init__(self, status, message) def set_response(self): - """Modify cherrypy.response status, headers, and body to represent self. + """Modify cherrypy.response status, headers, and body to represent + self. CherryPy uses this internally, but you can also use it to create an HTTPError object and set its output without *raising* the exception. @@ -369,11 +394,11 @@ class HTTPError(CherryPyException): tb = None if cherrypy.serving.request.show_tracebacks: tb = format_exc() - response.headers['Content-Type'] = "text/html;charset=utf-8" + response.headers.pop('Content-Length', None) - content = ntob(self.get_error_page(self.status, traceback=tb, - message=self._message), 'utf-8') + content = self.get_error_page(self.status, traceback=tb, + message=self._message) response.body = content _be_ie_unfriendly(self.code) @@ -387,6 +412,7 @@ class HTTPError(CherryPyException): class NotFound(HTTPError): + """Exception raised when a URL could not be mapped to any handler (404). This is equivalent to raising @@ -402,7 +428,8 @@ class NotFound(HTTPError): HTTPError.__init__(self, 404, "The path '%s' was not found." % path) -_HTTPErrorTemplate = ''' @@ -425,12 +452,15 @@ _HTTPErrorTemplate = '''%(message)s

    %(traceback)s
    - Powered by CherryPy %(version)s + + Powered by CherryPy %(version)s +
    ''' + def get_error_page(status, **kwargs): """Return an HTML page, containing a pretty error response. @@ -464,13 +494,33 @@ def get_error_page(status, **kwargs): # Use a custom template or callable for the error page? pages = cherrypy.serving.request.error_page error_page = pages.get(code) or pages.get('default') + + # Default template, can be overridden below. + template = _HTTPErrorTemplate if error_page: try: if hasattr(error_page, '__call__'): - return error_page(**kwargs) + # The caller function may be setting headers manually, + # so we delegate to it completely. We may be returning + # an iterator as well as a string here. + # + # We *must* make sure any content is not unicode. + result = error_page(**kwargs) + if cherrypy.lib.is_iterator(result): + from cherrypy.lib.encoding import UTF8StreamEncoder + return UTF8StreamEncoder(result) + elif isinstance(result, cherrypy._cpcompat.unicodestr): + return result.encode('utf-8') + else: + if not isinstance(result, cherrypy._cpcompat.bytestr): + raise ValueError('error page function did not ' + 'return a bytestring, unicodestring or an ' + 'iterator - returned object of type %s.' + % (type(result).__name__)) + return result else: - data = open(error_page, 'rb').read() - return tonative(data) % kwargs + # Load the template from this path. + template = tonative(open(error_page, 'rb').read()) except: e = _format_exception(*_exc_info())[-1] m = kwargs['message'] @@ -479,14 +529,18 @@ def get_error_page(status, **kwargs): m += "In addition, the custom error page failed:\n
    %s" % e kwargs['message'] = m - return _HTTPErrorTemplate % kwargs + response = cherrypy.serving.response + response.headers['Content-Type'] = "text/html;charset=utf-8" + result = template % kwargs + return result.encode('utf-8') + _ie_friendly_error_sizes = { 400: 512, 403: 256, 404: 512, 405: 256, 406: 512, 408: 512, 409: 512, 410: 256, 500: 512, 501: 512, 505: 512, - } +} def _be_ie_unfriendly(status): @@ -525,6 +579,7 @@ def format_exc(exc=None): finally: del exc + def bare_error(extrabody=None): """Produce status, headers, body for a critical error. @@ -550,7 +605,5 @@ def bare_error(extrabody=None): return (ntob("500 Internal Server Error"), [(ntob('Content-Type'), ntob('text/plain')), - (ntob('Content-Length'), ntob(str(len(body)),'ISO-8859-1'))], + (ntob('Content-Length'), ntob(str(len(body)), 'ISO-8859-1'))], [body]) - - diff --git a/lib/cherrypy/_cplogging.py b/lib/cherrypy/_cplogging.py index ebe5a931..554fd7ef 100644 --- a/lib/cherrypy/_cplogging.py +++ b/lib/cherrypy/_cplogging.py @@ -34,10 +34,11 @@ and another set of rules specific to each application. The global log manager is found at :func:`cherrypy.log`, and the log manager for each application is found at :attr:`app.log`. If you're inside a request, the latter is reachable from -``cherrypy.request.app.log``; if you're outside a request, you'll have to obtain -a reference to the ``app``: either the return value of +``cherrypy.request.app.log``; if you're outside a request, you'll have to +obtain a reference to the ``app``: either the return value of :func:`tree.mount()` or, if you used -:func:`quickstart()` instead, via ``cherrypy.tree.apps['/']``. +:func:`quickstart()` instead, via +``cherrypy.tree.apps['/']``. By default, the global logs are named "cherrypy.error" and "cherrypy.access", and the application logs are named "cherrypy.error.2378745" and @@ -55,6 +56,13 @@ errors! The format of access messages is highly formalized, but the error log isn't--it receives messages from a variety of sources (including full error tracebacks, if enabled). +If you are logging the access log and error log to the same source, then there +is a possibility that a specially crafted error message may replicate an access +log message as described in CWE-117. In this case it is the application +developer's responsibility to manually escape data before using CherryPy's log() +functionality, or they may create an application that is vulnerable to CWE-117. +This would be achieved by using a custom handler escape any special characters, +and attached as described below. Custom Handlers =============== @@ -113,6 +121,7 @@ from cherrypy._cpcompat import ntob, py3k class NullHandler(logging.Handler): + """A no-op logging handler to silence the logging.lastResort handler.""" def handle(self, record): @@ -126,6 +135,7 @@ class NullHandler(logging.Handler): class LogManager(object): + """An object to assist both simple and advanced logging. ``cherrypy.log`` is an instance of this class. @@ -166,8 +176,10 @@ class LogManager(object): self.error_log = logging.getLogger("%s.error" % logger_root) self.access_log = logging.getLogger("%s.access" % logger_root) else: - self.error_log = logging.getLogger("%s.error.%s" % (logger_root, appid)) - self.access_log = logging.getLogger("%s.access.%s" % (logger_root, appid)) + self.error_log = logging.getLogger( + "%s.error.%s" % (logger_root, appid)) + self.access_log = logging.getLogger( + "%s.access.%s" % (logger_root, appid)) self.error_log.setLevel(logging.INFO) self.access_log.setLevel(logging.INFO) @@ -187,7 +199,8 @@ class LogManager(object): h.stream = open(h.baseFilename, h.mode) h.release() - def error(self, msg='', context='', severity=logging.INFO, traceback=False): + def error(self, msg='', context='', severity=logging.INFO, + traceback=False): """Write the given ``msg`` to the error log. This is not just for errors! Applications may call this at any time @@ -207,8 +220,9 @@ class LogManager(object): def access(self): """Write to the access log (in Apache/NCSA Combined Log format). - See http://httpd.apache.org/docs/2.0/logs.html#combined for format - details. + See the + `apache documentation `_ + for format details. CherryPy calls this automatically for you. Note there are no arguments; it collects the data itself from @@ -242,6 +256,7 @@ class LogManager(object): 'b': dict.get(outheaders, 'Content-Length', '') or "-", 'f': dict.get(inheaders, 'Referer', ''), 'a': dict.get(inheaders, 'User-Agent', ''), + 'o': dict.get(inheaders, 'Host', '-'), } if py3k: for k, v in atoms.items(): @@ -261,7 +276,8 @@ class LogManager(object): atoms[k] = v try: - self.access_log.log(logging.INFO, self.access_log_format.format(**atoms)) + self.access_log.log( + logging.INFO, self.access_log_format.format(**atoms)) except: self(traceback=True) else: @@ -277,7 +293,8 @@ class LogManager(object): atoms[k] = v.replace('"', '\\"') try: - self.access_log.log(logging.INFO, self.access_log_format % atoms) + self.access_log.log( + logging.INFO, self.access_log_format % atoms) except: self(traceback=True) @@ -295,15 +312,13 @@ class LogManager(object): if getattr(h, "_cpbuiltin", None) == key: return h - # ------------------------- Screen handlers ------------------------- # - def _set_screen_handler(self, log, enable, stream=None): h = self._get_builtin_handler(log, "screen") if enable: if not h: if stream is None: - stream=sys.stderr + stream = sys.stderr h = logging.StreamHandler(stream) h.setFormatter(logfmt) h._cpbuiltin = "screen" @@ -320,7 +335,7 @@ class LogManager(object): self._set_screen_handler(self.error_log, newvalue, stream=sys.stderr) self._set_screen_handler(self.access_log, newvalue, stream=sys.stdout) screen = property(_get_screen, _set_screen, - doc="""Turn stderr/stdout logging on or off. + doc="""Turn stderr/stdout logging on or off. If you set this to True, it'll add the appropriate StreamHandler for you. If you set it to False, it will remove the handler. @@ -354,10 +369,11 @@ class LogManager(object): if h: return h.baseFilename return '' + def _set_error_file(self, newvalue): self._set_file_handler(self.error_log, newvalue) error_file = property(_get_error_file, _set_error_file, - doc="""The filename for self.error_log. + doc="""The filename for self.error_log. If you set this to a string, it'll add the appropriate FileHandler for you. If you set it to ``None`` or ``''``, it will remove the handler. @@ -368,10 +384,11 @@ class LogManager(object): if h: return h.baseFilename return '' + def _set_access_file(self, newvalue): self._set_file_handler(self.access_log, newvalue) access_file = property(_get_access_file, _set_access_file, - doc="""The filename for self.access_log. + doc="""The filename for self.access_log. If you set this to a string, it'll add the appropriate FileHandler for you. If you set it to ``None`` or ``''``, it will remove the handler. @@ -396,7 +413,7 @@ class LogManager(object): def _set_wsgi(self, newvalue): self._set_wsgi_handler(self.error_log, newvalue) wsgi = property(_get_wsgi, _set_wsgi, - doc="""Write errors to wsgi.errors. + doc="""Write errors to wsgi.errors. If you set this to True, it'll add the appropriate :class:`WSGIErrorHandler` for you @@ -406,6 +423,7 @@ class LogManager(object): class WSGIErrorHandler(logging.Handler): + "A handler class which writes logging records to environ['wsgi.errors']." def flush(self): @@ -428,7 +446,8 @@ class WSGIErrorHandler(logging.Handler): msg = self.format(record) fs = "%s\n" import types - if not hasattr(types, "UnicodeType"): #if no unicode support... + # if no unicode support... + if not hasattr(types, "UnicodeType"): stream.write(fs % msg) else: try: diff --git a/lib/cherrypy/_cpmodpy.py b/lib/cherrypy/_cpmodpy.py index 66f98309..02154d69 100644 --- a/lib/cherrypy/_cpmodpy.py +++ b/lib/cherrypy/_cpmodpy.py @@ -35,11 +35,11 @@ Listen 8080 LoadModule python_module /usr/lib/apache2/modules/mod_python.so - PythonPath "sys.path+['/path/to/my/application']" - SetHandler python-program - PythonHandler cherrypy._cpmodpy::handler - PythonOption cherrypy.setup myapp::setup_server - PythonDebug On + PythonPath "sys.path+['/path/to/my/application']" + SetHandler python-program + PythonHandler cherrypy._cpmodpy::handler + PythonOption cherrypy.setup myapp::setup_server + PythonDebug On # End @@ -67,11 +67,11 @@ from cherrypy.lib import httputil # ------------------------------ Request-handling - def setup(req): from mod_python import apache - # Run any setup functions defined by a "PythonOption cherrypy.setup" directive. + # Run any setup functions defined by a "PythonOption cherrypy.setup" + # directive. options = req.get_options() if 'cherrypy.setup' in options: for function in options['cherrypy.setup'].split(): @@ -106,7 +106,7 @@ def setup(req): elif logging.WARNING >= level: newlevel = apache.APLOG_WARNING # On Windows, req.server is required or the msg will vanish. See - # http://www.modpython.org/pipermail/mod_python/2003-October/014291.html. + # http://www.modpython.org/pipermail/mod_python/2003-October/014291.html # Also, "When server is not specified...LogLevel does not apply..." apache.log_error(msg, newlevel, req.server) engine.subscribe('log', _log) @@ -124,6 +124,7 @@ def setup(req): class _ReadOnlyRequest: expose = ('read', 'readline', 'readlines') + def __init__(self, req): for method in self.expose: self.__dict__[method] = getattr(req, method) @@ -132,6 +133,8 @@ class _ReadOnlyRequest: recursive = False _isSetUp = False + + def handler(req): from mod_python import apache try: @@ -142,9 +145,11 @@ def handler(req): # Obtain a Request object from CherryPy local = req.connection.local_addr - local = httputil.Host(local[0], local[1], req.connection.local_host or "") + local = httputil.Host( + local[0], local[1], req.connection.local_host or "") remote = req.connection.remote_addr - remote = httputil.Host(remote[0], remote[1], req.connection.remote_host or "") + remote = httputil.Host( + remote[0], remote[1], req.connection.remote_host or "") scheme = req.parsed_uri[0] or 'http' req.get_basic_auth_pw() @@ -210,10 +215,12 @@ def handler(req): if not recursive: if ir.path in redirections: - raise RuntimeError("InternalRedirector visited the " - "same URL twice: %r" % ir.path) + raise RuntimeError( + "InternalRedirector visited the same URL " + "twice: %r" % ir.path) else: - # Add the *previous* path_info + qs to redirections. + # Add the *previous* path_info + qs to + # redirections. if qs: qs = "?" + qs redirections.append(sn + path + qs) @@ -224,8 +231,9 @@ def handler(req): qs = ir.query_string rfile = BytesIO() - send_response(req, response.output_status, response.header_list, - response.body, response.stream) + send_response( + req, response.output_status, response.header_list, + response.body, response.stream) finally: app.release_serving() except: @@ -260,14 +268,12 @@ def send_response(req, status, headers, body, stream=False): req.write(seg) - # --------------- Startup tools for CherryPy + mod_python --------------- # - - import os import re try: import subprocess + def popen(fullcmd): p = subprocess.Popen(fullcmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -284,8 +290,12 @@ def read_process(cmd, args=""): pipeout = popen(fullcmd) try: firstline = pipeout.readline() - if (re.search(ntob("(not recognized|No such file|not found)"), firstline, - re.IGNORECASE)): + cmd_not_found = re.search( + ntob("(not recognized|No such file|not found)"), + firstline, + re.IGNORECASE + ) + if cmd_not_found: raise IOError('%s must be on your system path.' % cmd) output = firstline + pipeout.read() finally: @@ -341,4 +351,3 @@ LoadModule python_module modules/mod_python.so def stop(self): os.popen("apache -k stop") self.ready = False - diff --git a/lib/cherrypy/_cpnative_server.py b/lib/cherrypy/_cpnative_server.py index 401bce0a..e303573d 100644 --- a/lib/cherrypy/_cpnative_server.py +++ b/lib/cherrypy/_cpnative_server.py @@ -46,9 +46,11 @@ class NativeGateway(wsgiserver.Gateway): request.app = app request.prev = prev - # Run the CherryPy Request object and obtain the response + # Run the CherryPy Request object and obtain the + # response try: - request.run(method, path, qs, req.request_protocol, headers, rfile) + request.run(method, path, qs, + req.request_protocol, headers, rfile) break except cherrypy.InternalRedirect: ir = sys.exc_info()[1] @@ -57,10 +59,12 @@ class NativeGateway(wsgiserver.Gateway): if not self.recursive: if ir.path in redirections: - raise RuntimeError("InternalRedirector visited the " - "same URL twice: %r" % ir.path) + raise RuntimeError( + "InternalRedirector visited the same " + "URL twice: %r" % ir.path) else: - # Add the *previous* path_info + qs to redirections. + # Add the *previous* path_info + qs to + # redirections. if qs: qs = "?" + qs redirections.append(sn + path + qs) @@ -78,7 +82,7 @@ class NativeGateway(wsgiserver.Gateway): app.release_serving() except: tb = format_exc() - #print tb + # print tb cherrypy.log(tb, 'NATIVE_ADAPTER', severity=logging.ERROR) s, h, b = bare_error() self.send_response(s, h, b) @@ -102,6 +106,7 @@ class NativeGateway(wsgiserver.Gateway): class CPHTTPServer(wsgiserver.HTTPServer): + """Wrapper for wsgiserver.HTTPServer. wsgiserver has been designed to not reference CherryPy in any way, @@ -123,8 +128,10 @@ class CPHTTPServer(wsgiserver.HTTPServer): maxthreads=server_adapter.thread_pool_max, server_name=server_name) - self.max_request_header_size = self.server_adapter.max_request_header_size or 0 - self.max_request_body_size = self.server_adapter.max_request_body_size or 0 + self.max_request_header_size = ( + self.server_adapter.max_request_header_size or 0) + self.max_request_body_size = ( + self.server_adapter.max_request_body_size or 0) self.request_queue_size = self.server_adapter.socket_queue_size self.timeout = self.server_adapter.socket_timeout self.shutdown_timeout = self.server_adapter.shutdown_timeout @@ -145,5 +152,3 @@ class CPHTTPServer(wsgiserver.HTTPServer): self.server_adapter.ssl_certificate, self.server_adapter.ssl_private_key, self.server_adapter.ssl_certificate_chain) - - diff --git a/lib/cherrypy/_cpreqbody.py b/lib/cherrypy/_cpreqbody.py index 9ee8d846..d2dbbc92 100644 --- a/lib/cherrypy/_cpreqbody.py +++ b/lib/cherrypy/_cpreqbody.py @@ -3,8 +3,10 @@ .. versionadded:: 3.2 Application authors have complete control over the parsing of HTTP request -entities. In short, :attr:`cherrypy.request.body` -is now always set to an instance of :class:`RequestBody`, +entities. In short, +:attr:`cherrypy.request.body` +is now always set to an instance of +:class:`RequestBody`, and *that* class is a subclass of :class:`Entity`. When an HTTP request includes an entity body, it is often desirable to @@ -21,9 +23,9 @@ key to look up a value in the :attr:`request.body.processors` dict. If the full media type is not found, then the major type is tried; for example, if no processor -is found for the 'image/jpeg' type, then we look for a processor for the 'image' -types altogether. If neither the full type nor the major type has a matching -processor, then a default processor is used +is found for the 'image/jpeg' type, then we look for a processor for the +'image' types altogether. If neither the full type nor the major type has a +matching processor, then a default processor is used (:func:`default_proc`). For most types, this means no processing is done, and the body is left unread as a raw byte stream. Processors are configurable in an 'on_start_resource' hook. @@ -74,31 +76,36 @@ Here's the built-in JSON tool for an example:: 415, 'Expected an application/json content type') request.body.processors['application/json'] = json_processor -We begin by defining a new ``json_processor`` function to stick in the ``processors`` -dictionary. All processor functions take a single argument, the ``Entity`` instance -they are to process. It will be called whenever a request is received (for those -URI's where the tool is turned on) which has a ``Content-Type`` of -"application/json". +We begin by defining a new ``json_processor`` function to stick in the +``processors`` dictionary. All processor functions take a single argument, +the ``Entity`` instance they are to process. It will be called whenever a +request is received (for those URI's where the tool is turned on) which +has a ``Content-Type`` of "application/json". -First, it checks for a valid ``Content-Length`` (raising 411 if not valid), then -reads the remaining bytes on the socket. The ``fp`` object knows its own length, so -it won't hang waiting for data that never arrives. It will return when all data -has been read. Then, we decode those bytes using Python's built-in ``json`` module, -and stick the decoded result onto ``request.json`` . If it cannot be decoded, we -raise 400. +First, it checks for a valid ``Content-Length`` (raising 411 if not valid), +then reads the remaining bytes on the socket. The ``fp`` object knows its +own length, so it won't hang waiting for data that never arrives. It will +return when all data has been read. Then, we decode those bytes using +Python's built-in ``json`` module, and stick the decoded result onto +``request.json`` . If it cannot be decoded, we raise 400. -If the "force" argument is True (the default), the ``Tool`` clears the ``processors`` -dict so that request entities of other ``Content-Types`` aren't parsed at all. Since -there's no entry for those invalid MIME types, the ``default_proc`` method of ``cherrypy.request.body`` -is called. But this does nothing by default (usually to provide the page handler an opportunity to handle it.) -But in our case, we want to raise 415, so we replace ``request.body.default_proc`` +If the "force" argument is True (the default), the ``Tool`` clears the +``processors`` dict so that request entities of other ``Content-Types`` +aren't parsed at all. Since there's no entry for those invalid MIME +types, the ``default_proc`` method of ``cherrypy.request.body`` is +called. But this does nothing by default (usually to provide the page +handler an opportunity to handle it.) +But in our case, we want to raise 415, so we replace +``request.body.default_proc`` with the error (``HTTPError`` instances, when called, raise themselves). -If we were defining a custom processor, we can do so without making a ``Tool``. Just add the config entry:: +If we were defining a custom processor, we can do so without making a ``Tool``. +Just add the config entry:: request.body.processors = {'application/json': json_processor} -Note that you can only replace the ``processors`` dict wholesale this way, not update the existing one. +Note that you can only replace the ``processors`` dict wholesale this way, +not update the existing one. """ try: @@ -129,7 +136,7 @@ from cherrypy._cpcompat import basestring, ntob, ntou from cherrypy.lib import httputil -# -------------------------------- Processors -------------------------------- # +# ------------------------------- Processors -------------------------------- # def process_urlencoded(entity): """Read application/x-www-form-urlencoded data into entity.params.""" @@ -209,8 +216,10 @@ def process_multipart(entity): if part.fp.done: break + def process_multipart_form_data(entity): - """Read all multipart/form-data parts into entity.parts or entity.params.""" + """Read all multipart/form-data parts into entity.parts or entity.params. + """ process_multipart(entity) kept_parts = [] @@ -235,6 +244,7 @@ def process_multipart_form_data(entity): entity.parts = kept_parts + def _old_process_multipart(entity): """The behavior of 3.2 and lower. Deprecated and will be changed in 3.3.""" process_multipart(entity) @@ -263,11 +273,9 @@ def _old_process_multipart(entity): params[key] = value - -# --------------------------------- Entities --------------------------------- # - - +# -------------------------------- Entities --------------------------------- # class Entity(object): + """An HTTP request body, or MIME multipart body. This class collects information about the HTTP request entity. When a @@ -277,16 +285,19 @@ class Entity(object): Between the ``before_request_body`` and ``before_handler`` tools, CherryPy tries to process the request body (if any) by calling - :func:`request.body.process`, a dict. + :func:`request.body.process`. + This uses the ``content_type`` of the Entity to look up a suitable + processor in + :attr:`Entity.processors`, + a dict. If a matching processor cannot be found for the complete Content-Type, it tries again using the major type. For example, if a request with an entity of type "image/jpeg" arrives, but no processor can be found for that complete type, then one is sought for the major type "image". If a processor is still not found, then the - :func:`default_proc` method of the - Entity is called (which does nothing by default; you can override this too). + :func:`default_proc` method + of the Entity is called (which does nothing by default; you can + override this too). CherryPy includes processors for the "application/x-www-form-urlencoded" type, the "multipart/form-data" type, and the "multipart" major type. @@ -381,7 +392,8 @@ class Entity(object): """A dict of Content-Type names to processor methods.""" parts = None - """A list of Part instances if ``Content-Type`` is of major type "multipart".""" + """A list of Part instances if ``Content-Type`` is of major type + "multipart".""" part_class = None """The class used for multipart parts. @@ -414,7 +426,8 @@ class Entity(object): self.content_type = httputil.HeaderElement.from_str( self.default_content_type) - # Copy the class 'attempt_charsets', prepending any Content-Type charset + # Copy the class 'attempt_charsets', prepending any Content-Type + # charset dec = self.content_type.params.get("charset", None) if dec: self.attempt_charsets = [dec] + [c for c in self.attempt_charsets @@ -426,7 +439,10 @@ class Entity(object): self.length = None clen = headers.get('Content-Length', None) # If Transfer-Encoding is 'chunked', ignore any Content-Length. - if clen is not None and 'chunked' not in headers.get('Transfer-Encoding', ''): + if ( + clen is not None and + 'chunked' not in headers.get('Transfer-Encoding', '') + ): try: self.length = int(clen) except ValueError: @@ -444,12 +460,18 @@ class Entity(object): self.name = self.name[1:-1] if 'filename' in disp.params: self.filename = disp.params['filename'] - if self.filename.startswith('"') and self.filename.endswith('"'): + if ( + self.filename.startswith('"') and + self.filename.endswith('"') + ): self.filename = self.filename[1:-1] # The 'type' attribute is deprecated in 3.2; remove it in 3.3. - type = property(lambda self: self.content_type, - doc="""A deprecated alias for :attr:`content_type`.""") + type = property( + lambda self: self.content_type, + doc="A deprecated alias for " + ":attr:`content_type`." + ) def read(self, size=None, fp_out=None): return self.fp.read(size, fp_out) @@ -473,7 +495,10 @@ class Entity(object): return self.__next__() def read_into_file(self, fp_out=None): - """Read the request body into fp_out (or make_file() if None). Return fp_out.""" + """Read the request body into fp_out (or make_file() if None). + + Return fp_out. + """ if fp_out is None: fp_out = self.make_file() self.read(fp_out=fp_out) @@ -515,7 +540,9 @@ class Entity(object): proc(self) def default_proc(self): - """Called if a more-specific processor is not found for the ``Content-Type``.""" + """Called if a more-specific processor is not found for the + ``Content-Type``. + """ # Leave the fp alone for someone else to read. This works fine # for request.body, but the Part subclasses need to override this # so they can move on to the next part. @@ -523,6 +550,7 @@ class Entity(object): class Part(Entity): + """A MIME part entity, part of a multipart entity.""" # "The default character set, which must be assumed in the absence of a @@ -554,10 +582,11 @@ class Part(Entity): # This is the default in stdlib cgi. We may want to increase it. maxrambytes = 1000 - """The threshold of bytes after which point the ``Part`` will store its data - in a file (generated by :func:`make_file`) - instead of a string. Defaults to 1000, just like the :mod:`cgi` module in - Python's standard library. + """The threshold of bytes after which point the ``Part`` will store + its data in a file (generated by + :func:`make_file`) + instead of a string. Defaults to 1000, just like the :mod:`cgi` + module in Python's standard library. """ def __init__(self, fp, headers, boundary): @@ -607,9 +636,9 @@ class Part(Entity): If the 'fp_out' argument is None (the default), all bytes read are returned in a single byte string. - If the 'fp_out' argument is not None, it must be a file-like object that - supports the 'write' method; all bytes read will be written to the fp, - and that fp is returned. + If the 'fp_out' argument is not None, it must be a file-like + object that supports the 'write' method; all bytes read will be + written to the fp, and that fp is returned. """ endmarker = self.boundary + ntob("--") delim = ntob("") @@ -617,7 +646,7 @@ class Part(Entity): lines = [] seen = 0 while True: - line = self.fp.readline(1<<16) + line = self.fp.readline(1 << 16) if not line: raise EOFError("Illegal end of multipart body.") if line.startswith(ntob("--")) and prev_lf: @@ -664,14 +693,18 @@ class Part(Entity): return result else: raise cherrypy.HTTPError( - 400, "The request entity could not be decoded. The following " - "charsets were attempted: %s" % repr(self.attempt_charsets)) + 400, + "The request entity could not be decoded. The following " + "charsets were attempted: %s" % repr(self.attempt_charsets) + ) else: fp_out.seek(0) return fp_out def default_proc(self): - """Called if a more-specific processor is not found for the ``Content-Type``.""" + """Called if a more-specific processor is not found for the + ``Content-Type``. + """ if self.filename: # Always read into a file if a .filename was given. self.file = self.read_into_file() @@ -683,7 +716,10 @@ class Part(Entity): self.file = result def read_into_file(self, fp_out=None): - """Read the request body into fp_out (or make_file() if None). Return fp_out.""" + """Read the request body into fp_out (or make_file() if None). + + Return fp_out. + """ if fp_out is None: fp_out = self.make_file() self.read_lines_to_boundary(fp_out=fp_out) @@ -696,23 +732,30 @@ try: except ValueError: # Python 2.4 and lower class Infinity(object): + def __cmp__(self, other): return 1 + def __sub__(self, other): return self inf = Infinity() -comma_separated_headers = ['Accept', 'Accept-Charset', 'Accept-Encoding', - 'Accept-Language', 'Accept-Ranges', 'Allow', 'Cache-Control', 'Connection', - 'Content-Encoding', 'Content-Language', 'Expect', 'If-Match', - 'If-None-Match', 'Pragma', 'Proxy-Authenticate', 'Te', 'Trailer', - 'Transfer-Encoding', 'Upgrade', 'Vary', 'Via', 'Warning', 'Www-Authenticate'] +comma_separated_headers = [ + 'Accept', 'Accept-Charset', 'Accept-Encoding', + 'Accept-Language', 'Accept-Ranges', 'Allow', + 'Cache-Control', 'Connection', 'Content-Encoding', + 'Content-Language', 'Expect', 'If-Match', + 'If-None-Match', 'Pragma', 'Proxy-Authenticate', + 'Te', 'Trailer', 'Transfer-Encoding', 'Upgrade', + 'Vary', 'Via', 'Warning', 'Www-Authenticate' +] class SizedReader: - def __init__(self, fp, length, maxbytes, bufsize=DEFAULT_BUFFER_SIZE, has_trailers=False): + def __init__(self, fp, length, maxbytes, bufsize=DEFAULT_BUFFER_SIZE, + has_trailers=False): # Wrap our fp in a buffer so peek() works self.fp = fp self.length = length @@ -736,9 +779,9 @@ class SizedReader: If the 'fp_out' argument is None (the default), all bytes read are returned in a single byte string. - If the 'fp_out' argument is not None, it must be a file-like object that - supports the 'write' method; all bytes read will be written to the fp, - and None is returned. + If the 'fp_out' argument is not None, it must be a file-like + object that supports the 'write' method; all bytes read will be + written to the fp, and None is returned. """ if self.length is None: @@ -889,13 +932,15 @@ class SizedReader: class RequestBody(Entity): + """The entity of the HTTP request.""" bufsize = 8 * 1024 """The buffer size used when reading the socket.""" # Don't parse the request body at all if the client didn't provide - # a Content-Type header. See http://www.cherrypy.org/ticket/790 + # a Content-Type header. See + # https://bitbucket.org/cherrypy/cherrypy/issue/790 default_content_type = '' """This defines a default ``Content-Type`` to use if no Content-Type header is given. The empty string is used for RequestBody, which results in the @@ -907,7 +952,9 @@ class RequestBody(Entity): """ maxbytes = None - """Raise ``MaxSizeExceeded`` if more bytes than this are read from the socket.""" + """Raise ``MaxSizeExceeded`` if more bytes than this are read from + the socket. + """ def __init__(self, fp, headers, params=None, request_params=None): Entity.__init__(self, fp, headers, params) @@ -952,7 +999,8 @@ class RequestBody(Entity): # add them in here. request_params = self.request_params for key, value in self.params.items(): - # Python 2 only: keyword arguments must be byte strings (type 'str'). + # Python 2 only: keyword arguments must be byte strings (type + # 'str'). if sys.version_info < (3, 0): if isinstance(key, unicode): key = key.encode('ISO-8859-1') diff --git a/lib/cherrypy/_cprequest.py b/lib/cherrypy/_cprequest.py index 46c27d29..290bd2eb 100644 --- a/lib/cherrypy/_cprequest.py +++ b/lib/cherrypy/_cprequest.py @@ -13,6 +13,7 @@ from cherrypy.lib import httputil, file_generator class Hook(object): + """A callback and its metadata: failsafe, priority, and kwargs.""" callback = None @@ -71,6 +72,7 @@ class Hook(object): class HookMap(dict): + """A map of call points to lists of callbacks (Hook objects).""" def __new__(cls, points=None): @@ -122,7 +124,11 @@ class HookMap(dict): def __repr__(self): cls = self.__class__ - return "%s.%s(points=%r)" % (cls.__module__, cls.__name__, copykeys(self)) + return "%s.%s(points=%r)" % ( + cls.__module__, + cls.__name__, + copykeys(self) + ) # Config namespace handlers @@ -139,14 +145,17 @@ def hooks_namespace(k, v): v = Hook(v) cherrypy.serving.request.hooks[hookpoint].append(v) + def request_namespace(k, v): """Attach request attributes declared in config.""" - # Provides config entries to set request.body attrs (like attempt_charsets). + # Provides config entries to set request.body attrs (like + # attempt_charsets). if k[:5] == 'body.': setattr(cherrypy.serving.request.body, k[5:], v) else: setattr(cherrypy.serving.request, k, v) + def response_namespace(k, v): """Attach response attributes declared in config.""" # Provides config entries to set default response headers @@ -156,6 +165,7 @@ def response_namespace(k, v): else: setattr(cherrypy.serving.response, k, v) + def error_page_namespace(k, v): """Attach error pages declared in config.""" if k != 'default': @@ -170,6 +180,7 @@ hookpoints = ['on_start_resource', 'before_request_body', class Request(object): + """An HTTP request. This object represents the metadata of an HTTP request message; @@ -304,7 +315,10 @@ class Request(object): methods_with_bodies = ("POST", "PUT") """ A sequence of HTTP methods for which CherryPy will automatically - attempt to read a body from the rfile.""" + attempt to read a body from the rfile. If you are going to change + this property, modify it on the configuration (recommended) + or on the "hook point" `on_start_resource`. + """ body = None """ @@ -419,9 +433,9 @@ class Request(object): If a callable is provided, it will be called by default with keyword arguments 'status', 'message', 'traceback', and 'version', as for a - string-formatting template. The callable must return a string or iterable of - strings which will be set to response.body. It may also override headers or - perform any other processing. + string-formatting template. The callable must return a string or + iterable of strings which will be set to response.body. It may also + override headers or perform any other processing. If no entry is given for an error code, and no 'default' entry exists, a default template will be used. @@ -704,9 +718,10 @@ class Request(object): name = name.title() value = value.strip() - # Warning: if there is more than one header entry for cookies (AFAIK, - # only Konqueror does that), only the last one will remain in headers - # (but they will be correctly stored in request.cookie). + # Warning: if there is more than one header entry for cookies + # (AFAIK, only Konqueror does that), only the last one will + # remain in headers (but they will be correctly stored in + # request.cookie). if "=?" in value: dict.__setitem__(headers, name, httputil.decode_TEXT(value)) else: @@ -738,7 +753,8 @@ class Request(object): # First, see if there is a custom dispatch at this URI. Custom # dispatchers can only be specified in app.config, not in _cp_config # (since custom dispatchers may not even have an app.root). - dispatch = self.app.find_config(path, "request.dispatch", self.dispatch) + dispatch = self.app.find_config( + path, "request.dispatch", self.dispatch) # dispatch() should set self.handler and self.config dispatch(path) @@ -760,13 +776,13 @@ class Request(object): def _get_body_params(self): warnings.warn( - "body_params is deprecated in CherryPy 3.2, will be removed in " - "CherryPy 3.3.", - DeprecationWarning - ) + "body_params is deprecated in CherryPy 3.2, will be removed in " + "CherryPy 3.3.", + DeprecationWarning + ) return self.body.params body_params = property(_get_body_params, - doc= """ + doc=""" If the request Content-Type is 'application/x-www-form-urlencoded' or multipart, this will be a dict of the params pulled from the entity body; that is, it will be the portion of request.params that come @@ -780,6 +796,7 @@ class Request(object): class ResponseBody(object): + """The body of the HTTP response (the response entity).""" if py3k: @@ -822,6 +839,7 @@ class ResponseBody(object): class Response(object): + """An HTTP Response, including status, headers, and body.""" status = "" @@ -889,7 +907,8 @@ class Response(object): newbody = [] for chunk in self.body: if py3k and not isinstance(chunk, bytes): - raise TypeError("Chunk %s is not of type 'bytes'." % repr(chunk)) + raise TypeError("Chunk %s is not of type 'bytes'." % + repr(chunk)) newbody.append(chunk) newbody = ntob('').join(newbody) @@ -906,7 +925,8 @@ class Response(object): headers = self.headers self.status = "%s %s" % (code, reason) - self.output_status = ntob(str(code), 'ascii') + ntob(" ") + headers.encode(reason) + self.output_status = ntob(str(code), 'ascii') + \ + ntob(" ") + headers.encode(reason) if self.stream: # The upshot: wsgiserver will chunk the response if @@ -951,6 +971,3 @@ class Response(object): """ if time.time() > self.time + self.timeout: self.timed_out = True - - - diff --git a/lib/cherrypy/_cpserver.py b/lib/cherrypy/_cpserver.py index efbe5244..a31e7428 100644 --- a/lib/cherrypy/_cpserver.py +++ b/lib/cherrypy/_cpserver.py @@ -12,6 +12,7 @@ from cherrypy.process.servers import * class Server(ServerAdapter): + """An adapter for an HTTP server. You can set attributes (like socket_host and socket_port) @@ -26,15 +27,19 @@ class Server(ServerAdapter): """The TCP port on which to listen for connections.""" _socket_host = '127.0.0.1' + def _get_socket_host(self): return self._socket_host + def _set_socket_host(self, value): if value == '': raise ValueError("The empty string ('') is not an allowed value. " "Use '0.0.0.0' instead to listen on all active " "interfaces (INADDR_ANY).") self._socket_host = value - socket_host = property(_get_socket_host, _set_socket_host, + socket_host = property( + _get_socket_host, + _set_socket_host, doc="""The hostname or IP address on which to listen for connections. Host values may be any IPv4 or IPv6 address, or any valid hostname. @@ -56,6 +61,14 @@ class Server(ServerAdapter): socket_timeout = 10 """The timeout in seconds for accepted connections (default 10).""" + + accepted_queue_size = -1 + """The maximum number of requests which will be queued up before + the server refuses to accept it (default -1, meaning no limit).""" + + accepted_queue_timeout = 10 + """The timeout in seconds for attempting to add a request to the + queue when the queue is full (default 10).""" shutdown_timeout = 5 """The time to wait for HTTP worker threads to clean up.""" @@ -69,11 +82,13 @@ class Server(ServerAdapter): """The number of worker threads to start up in the pool.""" thread_pool_max = -1 - """The maximum size of the worker-thread pool. Use -1 to indicate no limit.""" + """The maximum size of the worker-thread pool. Use -1 to indicate no limit. + """ max_request_header_size = 500 * 1024 - """The maximum number of bytes allowable in the request headers. If exceeded, - the HTTP server should return "413 Request Entity Too Large".""" + """The maximum number of bytes allowable in the request headers. + If exceeded, the HTTP server should return "413 Request Entity Too Large". + """ max_request_body_size = 100 * 1024 * 1024 """The maximum number of bytes allowable in the request body. If exceeded, @@ -100,17 +115,19 @@ class Server(ServerAdapter): if py3k: ssl_module = 'builtin' - """The name of a registered SSL adaptation module to use with the builtin - WSGI server. Builtin options are: 'builtin' (to use the SSL library built - into recent versions of Python). You may also register your - own classes in the wsgiserver.ssl_adapters dict.""" + """The name of a registered SSL adaptation module to use with + the builtin WSGI server. Builtin options are: 'builtin' (to + use the SSL library built into recent versions of Python). + You may also register your own classes in the + wsgiserver.ssl_adapters dict.""" else: ssl_module = 'pyopenssl' - """The name of a registered SSL adaptation module to use with the builtin - WSGI server. Builtin options are 'builtin' (to use the SSL library built - into recent versions of Python) and 'pyopenssl' (to use the PyOpenSSL - project, which you must install separately). You may also register your - own classes in the wsgiserver.ssl_adapters dict.""" + """The name of a registered SSL adaptation module to use with the + builtin WSGI server. Builtin options are 'builtin' (to use the SSL + library built into recent versions of Python) and 'pyopenssl' (to + use the PyOpenSSL project, which you must install separately). You + may also register your own classes in the wsgiserver.ssl_adapters + dict.""" statistics = False """Turns statistics-gathering on or off for aware HTTP servers.""" @@ -157,6 +174,7 @@ class Server(ServerAdapter): if self.socket_host is None and self.socket_port is None: return None return (self.socket_host, self.socket_port) + def _set_bind_addr(self, value): if value is None: self.socket_file = None @@ -174,11 +192,15 @@ class Server(ServerAdapter): raise ValueError("bind_addr must be a (host, port) tuple " "(for TCP sockets) or a string (for Unix " "domain sockets), not %r" % value) - bind_addr = property(_get_bind_addr, _set_bind_addr, - doc='A (host, port) tuple for TCP sockets or a str for Unix domain sockets.') + bind_addr = property( + _get_bind_addr, + _set_bind_addr, + doc='A (host, port) tuple for TCP sockets or ' + 'a str for Unix domain sockets.') def base(self): - """Return the base (scheme://host[:port] or sock file) for this server.""" + """Return the base (scheme://host[:port] or sock file) for this server. + """ if self.socket_file: return self.socket_file @@ -202,4 +224,3 @@ class Server(ServerAdapter): host += ":%s" % port return "%s://%s" % (scheme, host) - diff --git a/lib/cherrypy/_cpthreadinglocal.py b/lib/cherrypy/_cpthreadinglocal.py index 34c17ac4..238c3224 100644 --- a/lib/cherrypy/_cpthreadinglocal.py +++ b/lib/cherrypy/_cpthreadinglocal.py @@ -137,6 +137,7 @@ affects what we see: # Threading import is at end + class _localbase(object): __slots__ = '_local__key', '_local__args', '_local__lock' @@ -158,6 +159,7 @@ class _localbase(object): return self + def _patch(self): key = object.__getattribute__(self, '_local__key') d = currentThread().__dict__.get(key) @@ -175,6 +177,7 @@ def _patch(self): else: object.__setattr__(self, '__dict__', d) + class local(_localbase): def __getattribute__(self, name): @@ -204,7 +207,6 @@ class local(_localbase): finally: lock.release() - def __del__(): threading_enumerate = enumerate __getattribute__ = object.__getattribute__ @@ -231,7 +233,7 @@ class local(_localbase): try: del __dict__[key] except KeyError: - pass # didn't have anything in this thread + pass # didn't have anything in this thread return __del__ __del__ = __del__() diff --git a/lib/cherrypy/_cptools.py b/lib/cherrypy/_cptools.py index 2f24e65f..06a56e87 100644 --- a/lib/cherrypy/_cptools.py +++ b/lib/cherrypy/_cptools.py @@ -43,10 +43,14 @@ def _getargs(func): return co.co_varnames[:co.co_argcount] -_attr_error = ("CherryPy Tools cannot be turned on directly. Instead, turn them " - "on via config, or use them as decorators on your page handlers.") +_attr_error = ( + "CherryPy Tools cannot be turned on directly. Instead, turn them " + "on via config, or use them as decorators on your page handlers." +) + class Tool(object): + """A registered function for use with CherryPy request-processing hooks. help(tool.callable) should give you more information about this Tool. @@ -64,6 +68,7 @@ class Tool(object): def _get_on(self): raise AttributeError(_attr_error) + def _set_on(self, value): raise AttributeError(_attr_error) on = property(_get_on, _set_on) @@ -117,6 +122,7 @@ class Tool(object): raise TypeError("The %r Tool does not accept positional " "arguments; you must use keyword arguments." % self._name) + def tool_decorator(f): if not hasattr(f, "_cp_config"): f._cp_config = {} @@ -142,6 +148,7 @@ class Tool(object): class HandlerTool(Tool): + """Tool which is called 'before main', that may skip normal handlers. If the tool successfully handles the request (by setting response.body), @@ -191,6 +198,7 @@ class HandlerTool(Tool): class HandlerWrapperTool(Tool): + """Tool which wraps request.handler in a provided wrapper function. The 'newhandler' arg must be a handler wrapper function that takes a @@ -209,20 +217,23 @@ class HandlerWrapperTool(Tool): cherrypy.tools.jinja = HandlerWrapperTool(interpolator) """ - def __init__(self, newhandler, point='before_handler', name=None, priority=50): + def __init__(self, newhandler, point='before_handler', name=None, + priority=50): self.newhandler = newhandler self._point = point self._name = name self._priority = priority - def callable(self, debug=False): + def callable(self, *args, **kwargs): innerfunc = cherrypy.serving.request.handler + def wrap(*args, **kwargs): return self.newhandler(innerfunc, *args, **kwargs) cherrypy.serving.request.handler = wrap class ErrorTool(Tool): + """Tool which is used to replace the default request.error_response.""" def __init__(self, callable, name=None): @@ -249,6 +260,7 @@ from cherrypy.lib import auth_basic, auth_digest class SessionTool(Tool): + """Session Tool for CherryPy. sessions.locking @@ -258,7 +270,8 @@ class SessionTool(Tool): When 'early', the session will be locked before reading the request body. This is off by default for safety reasons; for example, a large upload would block the session, denying an AJAX - progress meter (see http://www.cherrypy.org/ticket/630). + progress meter + (`issue `_). When 'explicit' (or any other value), you need to call cherrypy.session.acquire_lock() yourself before using @@ -314,9 +327,8 @@ class SessionTool(Tool): _sessions.set_response_cookie(**conf) - - class XMLRPCController(object): + """A Controller (page handler collection) for XML-RPC. To use it, have your controllers subclass this base class (it will @@ -364,7 +376,7 @@ class XMLRPCController(object): body = subhandler(*(vpath + rpcparams), **params) else: - # http://www.cherrypy.org/ticket/533 + # https://bitbucket.org/cherrypy/cherrypy/issue/533 # if a method is not found, an xmlrpclib.Fault should be returned # raising an exception here will do that; see # cherrypy.lib.xmlrpcutil.on_error @@ -387,6 +399,7 @@ class SessionAuthTool(HandlerTool): class CachingTool(Tool): + """Caching Tool for CherryPy.""" def _wrapper(self, **kwargs): @@ -397,7 +410,7 @@ class CachingTool(Tool): if request.cacheable: # Note the devious technique here of adding hooks on the fly request.hooks.attach('before_finalize', _caching.tee_output, - priority = 90) + priority=90) _wrapper.priority = 20 def _setup(self): @@ -409,8 +422,8 @@ class CachingTool(Tool): priority=p, **conf) - class Toolbox(object): + """A collection of Tools. This object also functions as a config namespace handler for itself. @@ -431,6 +444,7 @@ class Toolbox(object): def __enter__(self): """Populate request.toolmaps from tools specified in config.""" cherrypy.serving.request.toolmaps[self.namespace] = map = {} + def populate(k, v): toolname, arg = k.split(".", 1) bucket = map.setdefault(toolname, {}) @@ -459,6 +473,7 @@ class DeprecatedTool(Tool): def __call__(self, *args, **kwargs): warnings.warn(self.warnmsg) + def tool_decorator(f): return f return tool_decorator @@ -487,12 +502,16 @@ _d.sessions = SessionTool() _d.xmlrpc = ErrorTool(_xmlrpc.on_error) _d.caching = CachingTool('before_handler', _caching.get, 'caching') _d.expires = Tool('before_finalize', _caching.expires) -_d.tidy = DeprecatedTool('before_finalize', - "The tidy tool has been removed from the standard distribution of CherryPy. " - "The most recent version can be found at http://tools.cherrypy.org/browser.") -_d.nsgmls = DeprecatedTool('before_finalize', - "The nsgmls tool has been removed from the standard distribution of CherryPy. " - "The most recent version can be found at http://tools.cherrypy.org/browser.") +_d.tidy = DeprecatedTool( + 'before_finalize', + "The tidy tool has been removed from the standard distribution of " + "CherryPy. The most recent version can be found at " + "http://tools.cherrypy.org/browser.") +_d.nsgmls = DeprecatedTool( + 'before_finalize', + "The nsgmls tool has been removed from the standard distribution of " + "CherryPy. The most recent version can be found at " + "http://tools.cherrypy.org/browser.") _d.ignore_headers = Tool('before_request_body', cptools.ignore_headers) _d.referer = Tool('before_request_body', cptools.referer) _d.basic_auth = Tool('on_start_resource', auth.basic_auth) diff --git a/lib/cherrypy/_cptree.py b/lib/cherrypy/_cptree.py index b150b3dd..a31b2793 100644 --- a/lib/cherrypy/_cptree.py +++ b/lib/cherrypy/_cptree.py @@ -1,7 +1,6 @@ """CherryPy Application and Tree objects.""" import os -import sys import cherrypy from cherrypy._cpcompat import ntou, py3k @@ -10,6 +9,7 @@ from cherrypy.lib import httputil class Application(object): + """A CherryPy Application. Servers and gateways should not instantiate Request objects directly. @@ -62,10 +62,10 @@ class Application(object): return "%s.%s(%r, %r)" % (self.__module__, self.__class__.__name__, self.root, self.script_name) - script_name_doc = """The URI "mount point" for this app. A mount point is that portion of - the URI which is constant for all URIs that are serviced by this - application; it does not include scheme, host, or proxy ("virtual host") - portions of the URI. + script_name_doc = """The URI "mount point" for this app. A mount point + is that portion of the URI which is constant for all URIs that are + serviced by this application; it does not include scheme, host, or proxy + ("virtual host") portions of the URI. For example, if script_name is "/my/cool/app", then the URL "http://www.example.com/my/cool/app/page1" might be handled by a @@ -77,11 +77,15 @@ class Application(object): If script_name is explicitly set to None, then the script_name will be provided for each call from request.wsgi_environ['SCRIPT_NAME']. """ + def _get_script_name(self): - if self._script_name is None: - # None signals that the script name should be pulled from WSGI environ. - return cherrypy.serving.request.wsgi_environ['SCRIPT_NAME'].rstrip("/") - return self._script_name + if self._script_name is not None: + return self._script_name + + # A `_script_name` with a value of None signals that the script name + # should be pulled from WSGI environ. + return cherrypy.serving.request.wsgi_environ['SCRIPT_NAME'].rstrip("/") + def _set_script_name(self, value): if value: value = value.rstrip("/") @@ -148,6 +152,7 @@ class Application(object): class Tree(object): + """A registry of CherryPy applications, mounted at diverse points. An instance of this class may also be used as a WSGI callable @@ -201,8 +206,9 @@ class Tree(object): if isinstance(root, Application): app = root if script_name != "" and script_name != app.script_name: - raise ValueError("Cannot specify a different script name and " - "pass an Application instance to cherrypy.mount") + raise ValueError( + "Cannot specify a different script name and pass an " + "Application instance to cherrypy.mount") script_name = app.script_name else: app = Application(root, script_name) @@ -273,7 +279,8 @@ class Tree(object): # Python 2/WSGI u.0: all strings MUST be of type unicode enc = environ[ntou('wsgi.url_encoding')] environ[ntou('SCRIPT_NAME')] = sn.decode(enc) - environ[ntou('PATH_INFO')] = path[len(sn.rstrip("/")):].decode(enc) + environ[ntou('PATH_INFO')] = path[ + len(sn.rstrip("/")):].decode(enc) else: # Python 2/WSGI 1.x: all strings MUST be of type str environ['SCRIPT_NAME'] = sn @@ -285,6 +292,8 @@ class Tree(object): environ['PATH_INFO'] = path[len(sn.rstrip("/")):] else: # Python 3/WSGI 1.x: all strings MUST be ISO-8859-1 str - environ['SCRIPT_NAME'] = sn.encode('utf-8').decode('ISO-8859-1') - environ['PATH_INFO'] = path[len(sn.rstrip("/")):].encode('utf-8').decode('ISO-8859-1') + environ['SCRIPT_NAME'] = sn.encode( + 'utf-8').decode('ISO-8859-1') + environ['PATH_INFO'] = path[ + len(sn.rstrip("/")):].encode('utf-8').decode('ISO-8859-1') return app(environ, start_response) diff --git a/lib/cherrypy/_cpwsgi.py b/lib/cherrypy/_cpwsgi.py index fdc19249..f6db68b0 100644 --- a/lib/cherrypy/_cpwsgi.py +++ b/lib/cherrypy/_cpwsgi.py @@ -13,10 +13,11 @@ import cherrypy as _cherrypy from cherrypy._cpcompat import BytesIO, bytestr, ntob, ntou, py3k, unicodestr from cherrypy import _cperror from cherrypy.lib import httputil - +from cherrypy.lib import is_closable_iterator def downgrade_wsgi_ux_to_1x(environ): - """Return a new environ dict for WSGI 1.x from the given WSGI u.x environ.""" + """Return a new environ dict for WSGI 1.x from the given WSGI u.x environ. + """ env1x = {} url_encoding = environ[ntou('wsgi.url_encoding')] @@ -31,6 +32,7 @@ def downgrade_wsgi_ux_to_1x(environ): class VirtualHost(object): + """Select a different WSGI application based on the Host header. This can be useful when running multiple sites within one CP server. @@ -82,6 +84,7 @@ class VirtualHost(object): class InternalRedirector(object): + """WSGI middleware that handles raised cherrypy.InternalRedirect.""" def __init__(self, nextapp, recursive=False): @@ -107,7 +110,8 @@ class InternalRedirector(object): redirections.append(old_uri) if not self.recursive: - # Check to see if the new URI has been redirected to already + # Check to see if the new URI has been redirected to + # already new_uri = sn + ir.path if ir.query_string: new_uri += "?" + ir.query_string @@ -126,6 +130,7 @@ class InternalRedirector(object): class ExceptionTrapper(object): + """WSGI middleware that traps exceptions.""" def __init__(self, nextapp, throws=(KeyboardInterrupt, SystemExit)): @@ -133,7 +138,12 @@ class ExceptionTrapper(object): self.throws = throws def __call__(self, environ, start_response): - return _TrappedResponse(self.nextapp, environ, start_response, self.throws) + return _TrappedResponse( + self.nextapp, + environ, + start_response, + self.throws + ) class _TrappedResponse(object): @@ -146,7 +156,8 @@ class _TrappedResponse(object): self.start_response = start_response self.throws = throws self.started_response = False - self.response = self.trap(self.nextapp, self.environ, self.start_response) + self.response = self.trap( + self.nextapp, self.environ, self.start_response) self.iter_response = iter(self.response) def __iter__(self): @@ -210,6 +221,7 @@ class _TrappedResponse(object): class AppResponse(object): + """WSGI response iterable for CherryPy applications.""" def __init__(self, environ, start_response, cpapp): @@ -230,16 +242,20 @@ class AppResponse(object): outheaders = [] for k, v in r.header_list: if not isinstance(k, bytestr): - raise TypeError("response.header_list key %r is not a byte string." % k) + raise TypeError( + "response.header_list key %r is not a byte string." % + k) if not isinstance(v, bytestr): - raise TypeError("response.header_list value %r is not a byte string." % v) + raise TypeError( + "response.header_list value %r is not a byte string." % + v) outheaders.append((k, v)) if py3k: - # According to PEP 3333, when using Python 3, the response status - # and headers must be bytes masquerading as unicode; that is, they - # must be of type "str" but are restricted to code points in the - # "latin-1" set. + # According to PEP 3333, when using Python 3, the response + # status and headers must be bytes masquerading as unicode; + # that is, they must be of type "str" but are restricted to + # code points in the "latin-1" set. outstatus = outstatus.decode('ISO-8859-1') outheaders = [(k.decode('ISO-8859-1'), v.decode('ISO-8859-1')) for k, v in outheaders] @@ -262,14 +278,26 @@ class AppResponse(object): def close(self): """Close and de-reference the current request and response. (Core)""" + streaming = _cherrypy.serving.response.stream self.cpapp.release_serving() + # We avoid the expense of examining the iterator to see if it's + # closable unless we are streaming the response, as that's the + # only situation where we are going to have an iterator which + # may not have been exhausted yet. + if streaming and is_closable_iterator(self.iter_response): + iter_close = self.iter_response.close + try: + iter_close() + except Exception: + _cherrypy.log(traceback=True, severity=40) + def run(self): """Create a Request object using environ.""" env = self.environ.get local = httputil.Host('', int(env('SERVER_PORT', 80)), - env('SERVER_NAME', '')) + env('SERVER_NAME', '')) remote = httputil.Host(env('REMOTE_ADDR', ''), int(env('REMOTE_PORT', -1) or -1), env('REMOTE_HOST', '')) @@ -293,16 +321,17 @@ class AppResponse(object): qs = self.environ.get('QUERY_STRING', '') if py3k: - # This isn't perfect; if the given PATH_INFO is in the wrong encoding, - # it may fail to match the appropriate config section URI. But meh. + # This isn't perfect; if the given PATH_INFO is in the + # wrong encoding, it may fail to match the appropriate config + # section URI. But meh. old_enc = self.environ.get('wsgi.url_encoding', 'ISO-8859-1') new_enc = self.cpapp.find_config(self.environ.get('PATH_INFO', ''), "request.uri_encoding", 'utf-8') if new_enc.lower() != old_enc.lower(): - # Even though the path and qs are unicode, the WSGI server is - # required by PEP 3333 to coerce them to ISO-8859-1 masquerading - # as unicode. So we have to encode back to bytes and then decode - # again using the "correct" encoding. + # Even though the path and qs are unicode, the WSGI server + # is required by PEP 3333 to coerce them to ISO-8859-1 + # masquerading as unicode. So we have to encode back to + # bytes and then decode again using the "correct" encoding. try: u_path = path.encode(old_enc).decode(new_enc) u_qs = qs.encode(old_enc).decode(new_enc) @@ -339,6 +368,7 @@ class AppResponse(object): class CPWSGIApp(object): + """A WSGI application object for a CherryPy Application.""" pipeline = [('ExceptionTrapper', ExceptionTrapper), @@ -361,7 +391,8 @@ class CPWSGIApp(object): named WSGI callable (from the pipeline) as keyword arguments.""" response_class = AppResponse - """The class to instantiate and return as the next app in the WSGI chain.""" + """The class to instantiate and return as the next app in the WSGI chain. + """ def __init__(self, cpapp, pipeline=None): self.cpapp = cpapp @@ -405,4 +436,3 @@ class CPWSGIApp(object): name, arg = k.split(".", 1) bucket = self.config.setdefault(name, {}) bucket[arg] = v - diff --git a/lib/cherrypy/_cpwsgi_server.py b/lib/cherrypy/_cpwsgi_server.py index f8db23f2..874e2e9f 100644 --- a/lib/cherrypy/_cpwsgi_server.py +++ b/lib/cherrypy/_cpwsgi_server.py @@ -8,6 +8,7 @@ from cherrypy import wsgiserver class CPWSGIServer(wsgiserver.CherryPyWSGIServer): + """Wrapper for wsgiserver.CherryPyWSGIServer. wsgiserver has been designed to not reference CherryPy in any way, @@ -18,8 +19,12 @@ class CPWSGIServer(wsgiserver.CherryPyWSGIServer): def __init__(self, server_adapter=cherrypy.server): self.server_adapter = server_adapter - self.max_request_header_size = self.server_adapter.max_request_header_size or 0 - self.max_request_body_size = self.server_adapter.max_request_body_size or 0 + self.max_request_header_size = ( + self.server_adapter.max_request_header_size or 0 + ) + self.max_request_body_size = ( + self.server_adapter.max_request_body_size or 0 + ) server_name = (self.server_adapter.socket_host or self.server_adapter.socket_file or @@ -30,10 +35,12 @@ class CPWSGIServer(wsgiserver.CherryPyWSGIServer): s.__init__(self, server_adapter.bind_addr, cherrypy.tree, self.server_adapter.thread_pool, server_name, - max = self.server_adapter.thread_pool_max, - request_queue_size = self.server_adapter.socket_queue_size, - timeout = self.server_adapter.socket_timeout, - shutdown_timeout = self.server_adapter.shutdown_timeout, + max=self.server_adapter.thread_pool_max, + request_queue_size=self.server_adapter.socket_queue_size, + timeout=self.server_adapter.socket_timeout, + shutdown_timeout=self.server_adapter.shutdown_timeout, + accepted_queue_size=self.server_adapter.accepted_queue_size, + accepted_queue_timeout=self.server_adapter.accepted_queue_timeout, ) self.protocol = self.server_adapter.protocol_version self.nodelay = self.server_adapter.nodelay @@ -56,8 +63,8 @@ class CPWSGIServer(wsgiserver.CherryPyWSGIServer): self.server_adapter.ssl_private_key, self.server_adapter.ssl_certificate_chain) - self.stats['Enabled'] = getattr(self.server_adapter, 'statistics', False) + self.stats['Enabled'] = getattr( + self.server_adapter, 'statistics', False) def error_log(self, msg="", level=20, traceback=False): cherrypy.engine.log(msg, level, traceback) - diff --git a/lib/cherrypy/cherryd b/lib/cherrypy/cherryd old mode 100755 new mode 100644 index adb2a02e..5afb27ad --- a/lib/cherrypy/cherryd +++ b/lib/cherrypy/cherryd @@ -7,6 +7,7 @@ import cherrypy from cherrypy.process import plugins, servers from cherrypy import Application + def start(configfiles=None, daemonize=False, environment=None, fastcgi=False, scgi=False, pidfile=None, imports=None, cgi=False): @@ -14,7 +15,7 @@ def start(configfiles=None, daemonize=False, environment=None, sys.path = [''] + sys.path for i in imports or []: exec("import %s" % i) - + for c in configfiles or []: cherrypy.config.update(c) # If there's only one app mounted, merge config into it. @@ -22,26 +23,26 @@ def start(configfiles=None, daemonize=False, environment=None, for app in cherrypy.tree.apps.values(): if isinstance(app, Application): app.merge(c) - + engine = cherrypy.engine - + if environment is not None: cherrypy.config.update({'environment': environment}) - + # Only daemonize if asked to. if daemonize: # Don't print anything to stdout/sterr. cherrypy.config.update({'log.screen': False}) plugins.Daemonizer(engine).subscribe() - + if pidfile: plugins.PIDFile(engine, pidfile).subscribe() - + if hasattr(engine, "signal_handler"): engine.signal_handler.subscribe() if hasattr(engine, "console_control_handler"): engine.console_control_handler.subscribe() - + if (fastcgi and (scgi or cgi)) or (scgi and cgi): cherrypy.log.error("You may only specify one of the cgi, fastcgi, and " "scgi options.", 'ENGINE') @@ -51,7 +52,7 @@ def start(configfiles=None, daemonize=False, environment=None, cherrypy.config.update({'engine.autoreload_on': False}) # Turn off the default HTTP server (which is subscribed by default). cherrypy.server.unsubscribe() - + addr = cherrypy.server.bind_addr if fastcgi: f = servers.FlupFCGIServer(application=cherrypy.tree, @@ -64,7 +65,7 @@ def start(configfiles=None, daemonize=False, environment=None, bindAddress=addr) s = servers.ServerAdapter(engine, httpserver=f, bind_addr=addr) s.subscribe() - + # Always start the engine; this will start all other services try: engine.start() @@ -77,7 +78,7 @@ def start(configfiles=None, daemonize=False, environment=None, if __name__ == '__main__': from optparse import OptionParser - + p = OptionParser() p.add_option('-c', '--config', action="append", dest='config', help="specify config file(s)") @@ -86,7 +87,8 @@ if __name__ == '__main__': p.add_option('-e', '--environment', dest='environment', default=None, help="apply the given config environment") p.add_option('-f', action="store_true", dest='fastcgi', - help="start a fastcgi server instead of the default HTTP server") + help="start a fastcgi server instead of the default HTTP " + "server") p.add_option('-s', action="store_true", dest='scgi', help="start a scgi server instead of the default HTTP server") p.add_option('-x', action="store_true", dest='cgi', @@ -98,12 +100,11 @@ if __name__ == '__main__': p.add_option('-P', '--Path', action="append", dest='Path', help="add the given paths to sys.path") options, args = p.parse_args() - + if options.Path: for p in options.Path: sys.path.insert(0, p) - + start(options.config, options.daemonize, options.environment, options.fastcgi, options.scgi, options.pidfile, options.imports, options.cgi) - diff --git a/lib/cherrypy/lib/__init__.py b/lib/cherrypy/lib/__init__.py index bb72204b..a75a53da 100644 --- a/lib/cherrypy/lib/__init__.py +++ b/lib/cherrypy/lib/__init__.py @@ -3,7 +3,45 @@ # Deprecated in CherryPy 3.2 -- remove in CherryPy 3.3 from cherrypy.lib.reprconf import unrepr, modules, attributes +def is_iterator(obj): + '''Returns a boolean indicating if the object provided implements + the iterator protocol (i.e. like a generator). This will return + false for objects which iterable, but not iterators themselves.''' + from types import GeneratorType + if isinstance(obj, GeneratorType): + return True + elif not hasattr(obj, '__iter__'): + return False + else: + # Types which implement the protocol must return themselves when + # invoking 'iter' upon them. + return iter(obj) is obj + +def is_closable_iterator(obj): + + # Not an iterator. + if not is_iterator(obj): + return False + + # A generator - the easiest thing to deal with. + import inspect + if inspect.isgenerator(obj): + return True + + # A custom iterator. Look for a close method... + if not (hasattr(obj, 'close') and callable(obj.close)): + return False + + # ... which doesn't require any arguments. + try: + inspect.getcallargs(obj.close) + except TypeError: + return False + else: + return True + class file_generator(object): + """Yield the given input (a file object) in chunks (default 64k). (Core)""" def __init__(self, input, chunkSize=65536): @@ -23,6 +61,7 @@ class file_generator(object): raise StopIteration() next = __next__ + def file_generator_limited(fileobj, count, chunk_size=65536): """Yield the given file object in chunks, stopping after `count` bytes has been emitted. Default chunk size is 64kB. (Core) @@ -36,6 +75,7 @@ def file_generator_limited(fileobj, count, chunk_size=65536): remaining -= chunklen yield chunk + def set_vary_header(response, header_name): "Add a Vary header to a response" varies = response.headers.get("Vary", "") diff --git a/lib/cherrypy/lib/auth.py b/lib/cherrypy/lib/auth.py index 0f22b9be..71591aaa 100644 --- a/lib/cherrypy/lib/auth.py +++ b/lib/cherrypy/lib/auth.py @@ -3,7 +3,8 @@ from cherrypy.lib import httpauth def check_auth(users, encrypt=None, realm=None): - """If an authorization header contains credentials, return True, else False.""" + """If an authorization header contains credentials, return True or False. + """ request = cherrypy.serving.request if 'authorization' in request.headers: # make sure the provided credentials are correctly set @@ -17,10 +18,11 @@ def check_auth(users, encrypt=None, realm=None): if hasattr(users, '__call__'): try: # backward compatibility - users = users() # expect it to return a dictionary + users = users() # expect it to return a dictionary if not isinstance(users, dict): - raise ValueError("Authentication users must be a dictionary") + raise ValueError( + "Authentication users must be a dictionary") # fetch the user password password = users.get(ah["username"], None) @@ -44,6 +46,7 @@ def check_auth(users, encrypt=None, realm=None): request.login = False return False + def basic_auth(realm, users, encrypt=None, debug=False): """If auth fails, raise 401 with a basic authentication header. @@ -51,7 +54,8 @@ def basic_auth(realm, users, encrypt=None, debug=False): A string containing the authentication realm. users - A dict of the form: {username: password} or a callable returning a dict. + A dict of the form: {username: password} or a callable returning + a dict. encrypt callable used to encrypt the password returned from the user-agent. @@ -64,9 +68,12 @@ def basic_auth(realm, users, encrypt=None, debug=False): return # inform the user-agent this path is protected - cherrypy.serving.response.headers['www-authenticate'] = httpauth.basicAuth(realm) + cherrypy.serving.response.headers[ + 'www-authenticate'] = httpauth.basicAuth(realm) + + raise cherrypy.HTTPError( + 401, "You are not authorized to access that resource") - raise cherrypy.HTTPError(401, "You are not authorized to access that resource") def digest_auth(realm, users, debug=False): """If auth fails, raise 401 with a digest authentication header. @@ -74,7 +81,8 @@ def digest_auth(realm, users, debug=False): realm A string containing the authentication realm. users - A dict of the form: {username: password} or a callable returning a dict. + A dict of the form: {username: password} or a callable returning + a dict. """ if check_auth(users, realm=realm): if debug: @@ -82,6 +90,8 @@ def digest_auth(realm, users, debug=False): return # inform the user-agent this path is protected - cherrypy.serving.response.headers['www-authenticate'] = httpauth.digestAuth(realm) + cherrypy.serving.response.headers[ + 'www-authenticate'] = httpauth.digestAuth(realm) - raise cherrypy.HTTPError(401, "You are not authorized to access that resource") + raise cherrypy.HTTPError( + 401, "You are not authorized to access that resource") diff --git a/lib/cherrypy/lib/auth_basic.py b/lib/cherrypy/lib/auth_basic.py index cc9c53f2..5ba16f7f 100644 --- a/lib/cherrypy/lib/auth_basic.py +++ b/lib/cherrypy/lib/auth_basic.py @@ -3,7 +3,8 @@ # vim:ts=4:sw=4:expandtab:fileencoding=utf-8 __doc__ = """This module provides a CherryPy 3.x tool which implements -the server-side of HTTP Basic Access Authentication, as described in :rfc:`2617`. +the server-side of HTTP Basic Access Authentication, as described in +:rfc:`2617`. Example usage, using the built-in checkpassword_dict function which uses a dict as the credentials store:: @@ -77,11 +78,13 @@ def basic_auth(realm, checkpassword, debug=False): if debug: cherrypy.log('Auth succeeded', 'TOOLS.AUTH_BASIC') request.login = username - return # successful authentication - except (ValueError, binascii.Error): # split() error, base64.decodestring() error + return # successful authentication + # split() error, base64.decodestring() error + except (ValueError, binascii.Error): raise cherrypy.HTTPError(400, 'Bad Request') # Respond with 401 status and a WWW-Authenticate header - cherrypy.serving.response.headers['www-authenticate'] = 'Basic realm="%s"' % realm - raise cherrypy.HTTPError(401, "You are not authorized to access that resource") - + cherrypy.serving.response.headers[ + 'www-authenticate'] = 'Basic realm="%s"' % realm + raise cherrypy.HTTPError( + 401, "You are not authorized to access that resource") diff --git a/lib/cherrypy/lib/auth_digest.py b/lib/cherrypy/lib/auth_digest.py index 2814516c..e06535dc 100644 --- a/lib/cherrypy/lib/auth_digest.py +++ b/lib/cherrypy/lib/auth_digest.py @@ -41,6 +41,8 @@ def TRACE(msg): # Three helper functions for users of the tool, providing three variants # of get_ha1() functions for three different kinds of credential stores. + + def get_ha1_dict_plain(user_password_dict): """Returns a get_ha1 function which obtains a plaintext password from a dictionary of the form: {username : password}. @@ -57,6 +59,7 @@ def get_ha1_dict_plain(user_password_dict): return get_ha1 + def get_ha1_dict(user_ha1_dict): """Returns a get_ha1 function which obtains a HA1 password hash from a dictionary of the form: {username : HA1}. @@ -67,10 +70,11 @@ def get_ha1_dict(user_ha1_dict): argument to digest_auth(). """ def get_ha1(realm, username): - return user_ha1_dict.get(user) + return user_ha1_dict.get(username) return get_ha1 + def get_ha1_file_htdigest(filename): """Returns a get_ha1 function which obtains a HA1 password hash from a flat file with lines of the same format as that produced by the Apache @@ -99,8 +103,9 @@ def get_ha1_file_htdigest(filename): def synthesize_nonce(s, key, timestamp=None): - """Synthesize a nonce value which resists spoofing and can be checked for staleness. - Returns a string suitable as the value for 'nonce' in the www-authenticate header. + """Synthesize a nonce value which resists spoofing and can be checked + for staleness. Returns a string suitable as the value for 'nonce' in + the www-authenticate header. s A string related to the resource, such as the hostname of the server. @@ -125,6 +130,7 @@ def H(s): class HttpDigestAuthorization (object): + """Class to parse a Digest Authorization header and perform re-calculation of the digest. """ @@ -135,7 +141,7 @@ class HttpDigestAuthorization (object): def __init__(self, auth_header, http_method, debug=False): self.http_method = http_method self.debug = debug - scheme, params = auth_header.split(" ", 1) + scheme, params = auth_header.split(" ", 1) self.scheme = scheme.lower() if self.scheme != 'digest': raise ValueError('Authorization scheme is not "Digest"') @@ -151,84 +157,95 @@ class HttpDigestAuthorization (object): self.nonce = paramsd.get('nonce') self.uri = paramsd.get('uri') self.method = paramsd.get('method') - self.response = paramsd.get('response') # the response digest - self.algorithm = paramsd.get('algorithm', 'MD5') + self.response = paramsd.get('response') # the response digest + self.algorithm = paramsd.get('algorithm', 'MD5').upper() self.cnonce = paramsd.get('cnonce') self.opaque = paramsd.get('opaque') - self.qop = paramsd.get('qop') # qop - self.nc = paramsd.get('nc') # nonce count + self.qop = paramsd.get('qop') # qop + self.nc = paramsd.get('nc') # nonce count # perform some correctness checks if self.algorithm not in valid_algorithms: - raise ValueError(self.errmsg("Unsupported value for algorithm: '%s'" % self.algorithm)) + raise ValueError( + self.errmsg("Unsupported value for algorithm: '%s'" % + self.algorithm)) - has_reqd = self.username and \ - self.realm and \ - self.nonce and \ - self.uri and \ - self.response + has_reqd = ( + self.username and + self.realm and + self.nonce and + self.uri and + self.response + ) if not has_reqd: - raise ValueError(self.errmsg("Not all required parameters are present.")) + raise ValueError( + self.errmsg("Not all required parameters are present.")) if self.qop: if self.qop not in valid_qops: - raise ValueError(self.errmsg("Unsupported value for qop: '%s'" % self.qop)) + raise ValueError( + self.errmsg("Unsupported value for qop: '%s'" % self.qop)) if not (self.cnonce and self.nc): - raise ValueError(self.errmsg("If qop is sent then cnonce and nc MUST be present")) + raise ValueError( + self.errmsg("If qop is sent then " + "cnonce and nc MUST be present")) else: if self.cnonce or self.nc: - raise ValueError(self.errmsg("If qop is not sent, neither cnonce nor nc can be present")) - + raise ValueError( + self.errmsg("If qop is not sent, " + "neither cnonce nor nc can be present")) def __str__(self): return 'authorization : %s' % self.auth_header def validate_nonce(self, s, key): """Validate the nonce. - Returns True if nonce was generated by synthesize_nonce() and the timestamp - is not spoofed, else returns False. + Returns True if nonce was generated by synthesize_nonce() and the + timestamp is not spoofed, else returns False. s - A string related to the resource, such as the hostname of the server. + A string related to the resource, such as the hostname of + the server. key A secret string known only to the server. - Both s and key must be the same values which were used to synthesize the nonce - we are trying to validate. + Both s and key must be the same values which were used to synthesize + the nonce we are trying to validate. """ try: timestamp, hashpart = self.nonce.split(':', 1) - s_timestamp, s_hashpart = synthesize_nonce(s, key, timestamp).split(':', 1) + s_timestamp, s_hashpart = synthesize_nonce( + s, key, timestamp).split(':', 1) is_valid = s_hashpart == hashpart if self.debug: TRACE('validate_nonce: %s' % is_valid) return is_valid - except ValueError: # split() error + except ValueError: # split() error pass return False - def is_nonce_stale(self, max_age_seconds=600): """Returns True if a validated nonce is stale. The nonce contains a - timestamp in plaintext and also a secure hash of the timestamp. You should - first validate the nonce to ensure the plaintext timestamp is not spoofed. + timestamp in plaintext and also a secure hash of the timestamp. + You should first validate the nonce to ensure the plaintext + timestamp is not spoofed. """ try: timestamp, hashpart = self.nonce.split(':', 1) if int(timestamp) + max_age_seconds > int(time.time()): return False - except ValueError: # int() error + except ValueError: # int() error pass if self.debug: TRACE("nonce is stale") return True - def HA2(self, entity_body=''): """Returns the H(A2) string. See :rfc:`2617` section 3.2.2.3.""" # RFC 2617 3.2.2.3 - # If the "qop" directive's value is "auth" or is unspecified, then A2 is: + # If the "qop" directive's value is "auth" or is unspecified, + # then A2 is: # A2 = method ":" digest-uri-value # # If the "qop" value is "auth-int", then A2 is: @@ -238,11 +255,11 @@ class HttpDigestAuthorization (object): elif self.qop == "auth-int": a2 = "%s:%s:%s" % (self.http_method, self.uri, H(entity_body)) else: - # in theory, this should never happen, since I validate qop in __init__() + # in theory, this should never happen, since I validate qop in + # __init__() raise ValueError(self.errmsg("Unrecognized value for qop!")) return H(a2) - def request_digest(self, ha1, entity_body=''): """Calculates the Request-Digest. See :rfc:`2617` section 3.2.2.1. @@ -253,22 +270,24 @@ class HttpDigestAuthorization (object): If 'qop' is set to 'auth-int', then A2 includes a hash of the "entity body". The entity body is the part of the message which follows the HTTP headers. See :rfc:`2617` section - 4.3. This refers to the entity the user agent sent in the request which - has the Authorization header. Typically GET requests don't have an entity, - and POST requests do. + 4.3. This refers to the entity the user agent sent in the + request which has the Authorization header. Typically GET + requests don't have an entity, and POST requests do. """ ha2 = self.HA2(entity_body) # Request-Digest -- RFC 2617 3.2.2.1 if self.qop: - req = "%s:%s:%s:%s:%s" % (self.nonce, self.nc, self.cnonce, self.qop, ha2) + req = "%s:%s:%s:%s:%s" % ( + self.nonce, self.nc, self.cnonce, self.qop, ha2) else: req = "%s:%s" % (self.nonce, ha2) # RFC 2617 3.2.2.2 # - # If the "algorithm" directive's value is "MD5" or is unspecified, then A1 is: - # A1 = unq(username-value) ":" unq(realm-value) ":" passwd + # If the "algorithm" directive's value is "MD5" or is unspecified, + # then A1 is: + # A1 = unq(username-value) ":" unq(realm-value) ":" passwd # # If the "algorithm" directive's value is "MD5-sess", then A1 is # calculated only once - on the first request by the client following @@ -282,8 +301,8 @@ class HttpDigestAuthorization (object): return digest - -def www_authenticate(realm, key, algorithm='MD5', nonce=None, qop=qop_auth, stale=False): +def www_authenticate(realm, key, algorithm='MD5', nonce=None, qop=qop_auth, + stale=False): """Constructs a WWW-Authenticate header for Digest authentication.""" if qop not in valid_qops: raise ValueError("Unsupported value for qop: '%s'" % qop) @@ -293,7 +312,7 @@ def www_authenticate(realm, key, algorithm='MD5', nonce=None, qop=qop_auth, stal if nonce is None: nonce = synthesize_nonce(realm, key) s = 'Digest realm="%s", nonce="%s", algorithm="%s", qop="%s"' % ( - realm, nonce, algorithm, qop) + realm, nonce, algorithm, qop) if stale: s += ', stale="true"' return s @@ -303,11 +322,11 @@ def digest_auth(realm, get_ha1, key, debug=False): """A CherryPy tool which hooks at before_handler to perform HTTP Digest Access Authentication, as specified in :rfc:`2617`. - If the request has an 'authorization' header with a 'Digest' scheme, this - tool authenticates the credentials supplied in that header. If - the request has no 'authorization' header, or if it does but the scheme is - not "Digest", or if authentication fails, the tool sends a 401 response with - a 'WWW-Authenticate' Digest header. + If the request has an 'authorization' header with a 'Digest' scheme, + this tool authenticates the credentials supplied in that header. + If the request has no 'authorization' header, or if it does but the + scheme is not "Digest", or if authentication fails, the tool sends + a 401 response with a 'WWW-Authenticate' Digest header. realm A string containing the authentication realm. @@ -322,7 +341,8 @@ def digest_auth(realm, get_ha1, key, debug=False): None. key - A secret string known only to the server, used in the synthesis of nonces. + A secret string known only to the server, used in the synthesis + of nonces. """ request = cherrypy.serving.request @@ -331,9 +351,11 @@ def digest_auth(realm, get_ha1, key, debug=False): nonce_is_stale = False if auth_header is not None: try: - auth = HttpDigestAuthorization(auth_header, request.method, debug=debug) + auth = HttpDigestAuthorization( + auth_header, request.method, debug=debug) except ValueError: - raise cherrypy.HTTPError(400, "The Authorization header could not be parsed.") + raise cherrypy.HTTPError( + 400, "The Authorization header could not be parsed.") if debug: TRACE(str(auth)) @@ -341,19 +363,22 @@ def digest_auth(realm, get_ha1, key, debug=False): if auth.validate_nonce(realm, key): ha1 = get_ha1(realm, auth.username) if ha1 is not None: - # note that for request.body to be available we need to hook in at - # before_handler, not on_start_resource like 3.1.x digest_auth does. + # note that for request.body to be available we need to + # hook in at before_handler, not on_start_resource like + # 3.1.x digest_auth does. digest = auth.request_digest(ha1, entity_body=request.body) - if digest == auth.response: # authenticated + if digest == auth.response: # authenticated if debug: TRACE("digest matches auth.response") # Now check if nonce is stale. - # The choice of ten minutes' lifetime for nonce is somewhat arbitrary + # The choice of ten minutes' lifetime for nonce is somewhat + # arbitrary nonce_is_stale = auth.is_nonce_stale(max_age_seconds=600) if not nonce_is_stale: request.login = auth.username if debug: - TRACE("authentication of %s successful" % auth.username) + TRACE("authentication of %s successful" % + auth.username) return # Respond with 401 status and a WWW-Authenticate header @@ -361,5 +386,5 @@ def digest_auth(realm, get_ha1, key, debug=False): if debug: TRACE(header) cherrypy.serving.response.headers['WWW-Authenticate'] = header - raise cherrypy.HTTPError(401, "You are not authorized to access that resource") - + raise cherrypy.HTTPError( + 401, "You are not authorized to access that resource") diff --git a/lib/cherrypy/lib/caching.py b/lib/cherrypy/lib/caching.py index d67d14e6..fab6b569 100644 --- a/lib/cherrypy/lib/caching.py +++ b/lib/cherrypy/lib/caching.py @@ -1,7 +1,7 @@ """ -CherryPy implements a simple caching system as a pluggable Tool. This tool tries -to be an (in-process) HTTP/1.1-compliant cache. It's not quite there yet, but -it's probably good enough for most sites. +CherryPy implements a simple caching system as a pluggable Tool. This tool +tries to be an (in-process) HTTP/1.1-compliant cache. It's not quite there +yet, but it's probably good enough for most sites. In general, GET responses are cached (along with selecting headers) and, if another request arrives for the same resource, the caching Tool will return 304 @@ -9,8 +9,8 @@ Not Modified if possible, or serve the cached response otherwise. It also sets request.cached to True if serving a cached representation, and sets request.cacheable to False (so it doesn't get cached again). -If POST, PUT, or DELETE requests are made for a cached resource, they invalidate -(delete) any cached response. +If POST, PUT, or DELETE requests are made for a cached resource, they +invalidate (delete) any cached response. Usage ===== @@ -39,10 +39,11 @@ import time import cherrypy from cherrypy.lib import cptools, httputil -from cherrypy._cpcompat import copyitems, ntob, set_daemon, sorted +from cherrypy._cpcompat import copyitems, ntob, set_daemon, sorted, Event class Cache(object): + """Base class for Cache implementations.""" def get(self): @@ -62,11 +63,9 @@ class Cache(object): raise NotImplemented - -# ------------------------------- Memory Cache ------------------------------- # - - +# ------------------------------ Memory Cache ------------------------------- # class AntiStampedeCache(dict): + """A storage system for cached items which reduces stampede collisions.""" def wait(self, key, timeout=5, debug=False): @@ -81,7 +80,7 @@ class AntiStampedeCache(dict): If timeout is None, no waiting is performed nor sentinels used. """ value = self.get(key) - if isinstance(value, threading._Event): + if isinstance(value, Event): if timeout is None: # Ignore the other thread and recalc it ourselves. if debug: @@ -90,7 +89,8 @@ class AntiStampedeCache(dict): # Wait until it's done or times out. if debug: - cherrypy.log('Waiting up to %s seconds' % timeout, 'TOOLS.CACHING') + cherrypy.log('Waiting up to %s seconds' % + timeout, 'TOOLS.CACHING') value.wait(timeout) if value.result is not None: # The other thread finished its calculation. Use it. @@ -120,7 +120,7 @@ class AntiStampedeCache(dict): """Set the cached value for the given key.""" existing = self.get(key) dict.__setitem__(self, key, value) - if isinstance(existing, threading._Event): + if isinstance(existing, Event): # Set Event.result so other threads waiting on it have # immediate access without needing to poll the cache again. existing.result = value @@ -128,6 +128,7 @@ class AntiStampedeCache(dict): class MemoryCache(Cache): + """An in-memory cache for varying response content. Each key in self.store is a URI, and each value is an AntiStampedeCache. @@ -152,7 +153,8 @@ class MemoryCache(Cache): """The maximum size of the entire cache in bytes; defaults to 10 MB.""" delay = 600 - """Seconds until the cached content expires; defaults to 600 (10 minutes).""" + """Seconds until the cached content expires; defaults to 600 (10 minutes). + """ antistampede_timeout = 5 """Seconds to wait for other threads to release a cache lock.""" @@ -325,13 +327,15 @@ def get(invalid_methods=("POST", "PUT", "DELETE"), debug=False, **kwargs): directive = atoms.pop(0) if directive == 'max-age': if len(atoms) != 1 or not atoms[0].isdigit(): - raise cherrypy.HTTPError(400, "Invalid Cache-Control header") + raise cherrypy.HTTPError( + 400, "Invalid Cache-Control header") max_age = int(atoms[0]) break elif directive == 'no-cache': if debug: - cherrypy.log('Ignoring cache due to Cache-Control: no-cache', - 'TOOLS.CACHING') + cherrypy.log( + 'Ignoring cache due to Cache-Control: no-cache', + 'TOOLS.CACHING') request.cached = False request.cacheable = True return False @@ -348,7 +352,8 @@ def get(invalid_methods=("POST", "PUT", "DELETE"), debug=False, **kwargs): request.cacheable = True return False - # Copy the response headers. See http://www.cherrypy.org/ticket/721. + # Copy the response headers. See + # https://bitbucket.org/cherrypy/cherrypy/issue/721. response.headers = rh = httputil.HeaderMap() for k in h: dict.__setitem__(rh, k, dict.__getitem__(h, k)) @@ -387,7 +392,7 @@ def tee_output(): def tee(body): """Tee response.body into a list.""" if ('no-cache' in response.headers.values('Pragma') or - 'no-store' in response.headers.values('Cache-Control')): + 'no-store' in response.headers.values('Cache-Control')): for chunk in body: yield chunk return diff --git a/lib/cherrypy/lib/covercp.py b/lib/cherrypy/lib/covercp.py index 656d99da..a74ec342 100644 --- a/lib/cherrypy/lib/covercp.py +++ b/lib/cherrypy/lib/covercp.py @@ -24,13 +24,15 @@ import re import sys import cgi from cherrypy._cpcompat import quote_plus -import os, os.path +import os +import os.path localFile = os.path.join(os.path.dirname(__file__), "coverage.cache") the_coverage = None try: from coverage import coverage the_coverage = coverage(data_file=localFile) + def start(): the_coverage.start() except ImportError: @@ -39,7 +41,9 @@ except ImportError: the_coverage = None import warnings - warnings.warn("No code coverage will be performed; coverage.py could not be imported.") + warnings.warn( + "No code coverage will be performed; " + "coverage.py could not be imported.") def start(): pass @@ -118,10 +122,13 @@ TEMPLATE_FORM = """
    - Show percentages
    - Hide files over %%
    + Show percentages +
    + Hide files over + %%
    Exclude files matching
    - +
    @@ -173,7 +180,10 @@ TEMPLATE_LOC_EXCLUDED = """ %s \n""" -TEMPLATE_ITEM = "%s%s%s\n" +TEMPLATE_ITEM = ( + "%s%s%s\n" +) + def _percent(statements, missing): s = len(statements) @@ -182,6 +192,7 @@ def _percent(statements, missing): return int(round(100.0 * e / s)) return 0 + def _show_branch(root, base, path, pct=0, showpct=False, exclude="", coverage=the_coverage): @@ -194,10 +205,16 @@ def _show_branch(root, base, path, pct=0, showpct=False, exclude="", if newpath.lower().startswith(base): relpath = newpath[len(base):] yield "| " * relpath.count(os.sep) - yield "%s\n" % \ - (newpath, quote_plus(exclude), name) + yield ( + "%s\n" % + (newpath, quote_plus(exclude), name) + ) - for chunk in _show_branch(root[name], base, newpath, pct, showpct, exclude, coverage=coverage): + for chunk in _show_branch( + root[name], base, newpath, pct, showpct, + exclude, coverage=coverage + ): yield chunk # Now list the files @@ -217,7 +234,7 @@ def _show_branch(root, base, path, pct=0, showpct=False, exclude="", pass else: pc = _percent(statements, missing) - pc_str = ("%3d%% " % pc).replace(' ',' ') + pc_str = ("%3d%% " % pc).replace(' ', ' ') if pc < float(pct) or pc == -1: pc_str = "%s" % pc_str else: @@ -226,10 +243,12 @@ def _show_branch(root, base, path, pct=0, showpct=False, exclude="", yield TEMPLATE_ITEM % ("| " * (relpath.count(os.sep) + 1), pc_str, newpath, name) + def _skip_file(path, exclude): if exclude: return bool(re.search(exclude, path)) + def _graft(path, tree): d = tree @@ -249,6 +268,7 @@ def _graft(path, tree): if node: d = d.setdefault(node, {}) + def get_tree(base, exclude, coverage=the_coverage): """Return covered module names as a nested dict.""" tree = {} @@ -258,6 +278,7 @@ def get_tree(base, exclude, coverage=the_coverage): _graft(path, tree) return tree + class CoverStats(object): def __init__(self, coverage, root=None): @@ -301,7 +322,8 @@ class CoverStats(object): yield "

    No modules covered.

    " else: for chunk in _show_branch(tree, base, "/", pct, - showpct=='checked', exclude, coverage=self.coverage): + showpct == 'checked', exclude, + coverage=self.coverage): yield chunk yield "
    " @@ -331,7 +353,8 @@ class CoverStats(object): yield template % (lineno, cgi.escape(line)) def report(self, name): - filename, statements, excluded, missing, _ = self.coverage.analysis2(name) + filename, statements, excluded, missing, _ = self.coverage.analysis2( + name) pc = _percent(statements, missing) yield TEMPLATE_COVERAGE % dict(name=os.path.basename(name), fullpath=name, @@ -350,7 +373,7 @@ def serve(path=localFile, port=8080, root=None): if coverage is None: raise ImportError("The coverage module could not be imported.") from coverage import coverage - cov = coverage(data_file = path) + cov = coverage(data_file=path) cov.load() import cherrypy @@ -362,4 +385,3 @@ def serve(path=localFile, port=8080, root=None): if __name__ == "__main__": serve(*tuple(sys.argv[1:])) - diff --git a/lib/cherrypy/lib/cpstats.py b/lib/cherrypy/lib/cpstats.py index 0d77f57b..a8661a14 100644 --- a/lib/cherrypy/lib/cpstats.py +++ b/lib/cherrypy/lib/cpstats.py @@ -21,33 +21,37 @@ to collect stats to import a third-party module. Therefore, we choose to re-use the `logging` module by adding a `statistics` object to it. That `logging.statistics` object is a nested dict. It is not a custom class, -because that would 1) require libraries and applications to import a third- -party module in order to participate, 2) inhibit innovation in extrapolation -approaches and in reporting tools, and 3) be slow. There are, however, some -specifications regarding the structure of the dict. +because that would: - { - +----"SQLAlchemy": { - | "Inserts": 4389745, - | "Inserts per Second": - | lambda s: s["Inserts"] / (time() - s["Start"]), - | C +---"Table Statistics": { - | o | "widgets": {-----------+ - N | l | "Rows": 1.3M, | Record - a | l | "Inserts": 400, | - m | e | },---------------------+ - e | c | "froobles": { - s | t | "Rows": 7845, - p | i | "Inserts": 0, - a | o | }, - c | n +---}, - e | "Slow Queries": - | [{"Query": "SELECT * FROM widgets;", - | "Processing Time": 47.840923343, - | }, - | ], - +----}, - } + 1. require libraries and applications to import a third-party module in + order to participate + 2. inhibit innovation in extrapolation approaches and in reporting tools, and + 3. be slow. + +There are, however, some specifications regarding the structure of the dict.:: + + { + +----"SQLAlchemy": { + | "Inserts": 4389745, + | "Inserts per Second": + | lambda s: s["Inserts"] / (time() - s["Start"]), + | C +---"Table Statistics": { + | o | "widgets": {-----------+ + N | l | "Rows": 1.3M, | Record + a | l | "Inserts": 400, | + m | e | },---------------------+ + e | c | "froobles": { + s | t | "Rows": 7845, + p | i | "Inserts": 0, + a | o | }, + c | n +---}, + e | "Slow Queries": + | [{"Query": "SELECT * FROM widgets;", + | "Processing Time": 47.840923343, + | }, + | ], + +----}, + } The `logging.statistics` dict has four levels. The topmost level is nothing more than a set of names to introduce modularity, usually along the lines of @@ -65,13 +69,13 @@ Each namespace, then, is a dict of named statistical values, such as good on a report: spaces and capitalization are just fine. In addition to scalars, values in a namespace MAY be a (third-layer) -dict, or a list, called a "collection". For example, the CherryPy StatsTool -keeps track of what each request is doing (or has most recently done) -in a 'Requests' collection, where each key is a thread ID; each +dict, or a list, called a "collection". For example, the CherryPy +:class:`StatsTool` keeps track of what each request is doing (or has most +recently done) in a 'Requests' collection, where each key is a thread ID; each value in the subdict MUST be a fourth dict (whew!) of statistical data about each thread. We call each subdict in the collection a "record". Similarly, -the StatsTool also keeps a list of slow queries, where each record contains -data about each slow query, in order. +the :class:`StatsTool` also keeps a list of slow queries, where each record +contains data about each slow query, in order. Values in a namespace or record may also be functions, which brings us to: @@ -86,17 +90,17 @@ scalar values you already have on hand. When it comes time to report on the gathered data, however, we usually have much more freedom in what we can calculate. Therefore, whenever reporting -tools (like the provided StatsPage CherryPy class) fetch the contents of -`logging.statistics` for reporting, they first call `extrapolate_statistics` -(passing the whole `statistics` dict as the only argument). This makes a -deep copy of the statistics dict so that the reporting tool can both iterate -over it and even change it without harming the original. But it also expands -any functions in the dict by calling them. For example, you might have a -'Current Time' entry in the namespace with the value "lambda scope: time.time()". -The "scope" parameter is the current namespace dict (or record, if we're -currently expanding one of those instead), allowing you access to existing -static entries. If you're truly evil, you can even modify more than one entry -at a time. +tools (like the provided :class:`StatsPage` CherryPy class) fetch the contents +of `logging.statistics` for reporting, they first call +`extrapolate_statistics` (passing the whole `statistics` dict as the only +argument). This makes a deep copy of the statistics dict so that the +reporting tool can both iterate over it and even change it without harming +the original. But it also expands any functions in the dict by calling them. +For example, you might have a 'Current Time' entry in the namespace with the +value "lambda scope: time.time()". The "scope" parameter is the current +namespace dict (or record, if we're currently expanding one of those +instead), allowing you access to existing static entries. If you're truly +evil, you can even modify more than one entry at a time. However, don't try to calculate an entry and then use its value in further extrapolations; the order in which the functions are called is not guaranteed. @@ -108,19 +112,20 @@ After the whole thing has been extrapolated, it's time for: Reporting --------- -The StatsPage class grabs the `logging.statistics` dict, extrapolates it all, -and then transforms it to HTML for easy viewing. Each namespace gets its own -header and attribute table, plus an extra table for each collection. This is -NOT part of the statistics specification; other tools can format how they like. +The :class:`StatsPage` class grabs the `logging.statistics` dict, extrapolates +it all, and then transforms it to HTML for easy viewing. Each namespace gets +its own header and attribute table, plus an extra table for each collection. +This is NOT part of the statistics specification; other tools can format how +they like. You can control which columns are output and how they are formatted by updating StatsPage.formatting, which is a dict that mirrors the keys and nesting of `logging.statistics`. The difference is that, instead of data values, it has formatting values. Use None for a given key to indicate to the StatsPage that a -given column should not be output. Use a string with formatting (such as '%.3f') -to interpolate the value(s), or use a callable (such as lambda v: v.isoformat()) -for more advanced formatting. Any entry which is not mentioned in the formatting -dict is output unchanged. +given column should not be output. Use a string with formatting +(such as '%.3f') to interpolate the value(s), or use a callable (such as +lambda v: v.isoformat()) for more advanced formatting. Any entry which is not +mentioned in the formatting dict is output unchanged. Monitoring ---------- @@ -145,12 +150,12 @@ entries to False or True, if present. Usage ===== -To collect statistics on CherryPy applications: +To collect statistics on CherryPy applications:: from cherrypy.lib import cpstats appconfig['/']['tools.cpstats.on'] = True -To collect statistics on your own code: +To collect statistics on your own code:: import logging # Initialize the repository @@ -172,20 +177,22 @@ To collect statistics on your own code: if mystats.get('Enabled', False): mystats['Important Events'] += 1 -To report statistics: +To report statistics:: root.cpstats = cpstats.StatsPage() -To format statistics reports: +To format statistics reports:: See 'Reporting', above. """ -# -------------------------------- Statistics -------------------------------- # +# ------------------------------- Statistics -------------------------------- # import logging -if not hasattr(logging, 'statistics'): logging.statistics = {} +if not hasattr(logging, 'statistics'): + logging.statistics = {} + def extrapolate_statistics(scope): """Return an extrapolated copy of the given scope.""" @@ -201,7 +208,7 @@ def extrapolate_statistics(scope): return c -# --------------------- CherryPy Applications Statistics --------------------- # +# -------------------- CherryPy Applications Statistics --------------------- # import threading import time @@ -211,12 +218,20 @@ import cherrypy appstats = logging.statistics.setdefault('CherryPy Applications', {}) appstats.update({ 'Enabled': True, - 'Bytes Read/Request': lambda s: (s['Total Requests'] and - (s['Total Bytes Read'] / float(s['Total Requests'])) or 0.0), + 'Bytes Read/Request': lambda s: ( + s['Total Requests'] and + (s['Total Bytes Read'] / float(s['Total Requests'])) or + 0.0 + ), 'Bytes Read/Second': lambda s: s['Total Bytes Read'] / s['Uptime'](s), - 'Bytes Written/Request': lambda s: (s['Total Requests'] and - (s['Total Bytes Written'] / float(s['Total Requests'])) or 0.0), - 'Bytes Written/Second': lambda s: s['Total Bytes Written'] / s['Uptime'](s), + 'Bytes Written/Request': lambda s: ( + s['Total Requests'] and + (s['Total Bytes Written'] / float(s['Total Requests'])) or + 0.0 + ), + 'Bytes Written/Second': lambda s: ( + s['Total Bytes Written'] / s['Uptime'](s) + ), 'Current Time': lambda s: time.time(), 'Current Requests': 0, 'Requests/Second': lambda s: float(s['Total Requests']) / s['Uptime'](s), @@ -228,12 +243,13 @@ appstats.update({ 'Total Time': 0, 'Uptime': lambda s: time.time() - s['Start Time'], 'Requests': {}, - }) +}) proc_time = lambda s: time.time() - s['Start Time'] class ByteCountWrapper(object): + """Wraps a file-like object, counting the number of bytes read.""" def __init__(self, rfile): @@ -279,6 +295,7 @@ average_uriset_time = lambda s: s['Count'] and (s['Sum'] / s['Count']) or 0 class StatsTool(cherrypy.Tool): + """Record various information about the current request.""" def __init__(self): @@ -315,10 +332,11 @@ class StatsTool(cherrypy.Tool): 'Request-Line': request.request_line, 'Response Status': None, 'Start Time': time.time(), - } + } - def record_stop(self, uriset=None, slow_queries=1.0, slow_queries_count=100, - debug=False, **kwargs): + def record_stop( + self, uriset=None, slow_queries=1.0, slow_queries_count=100, + debug=False, **kwargs): """Record the end of a request.""" resp = cherrypy.serving.response w = appstats['Requests'][threading._get_ident()] @@ -334,7 +352,8 @@ class StatsTool(cherrypy.Tool): w['Bytes Written'] = cl appstats['Total Bytes Written'] += cl - w['Response Status'] = getattr(resp, 'output_status', None) or resp.status + w['Response Status'] = getattr( + resp, 'output_status', None) or resp.status w['End Time'] = time.time() p = w['End Time'] - w['Start Time'] @@ -388,6 +407,7 @@ missing = object() locale_date = lambda v: time.strftime('%c', time.gmtime(v)) iso_format = lambda v: time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(v)) + def pause_resume(ns): def _pause_resume(enabled): pause_disabled = '' @@ -427,20 +447,20 @@ class StatsPage(object): 'End Time': None, 'Processing Time': '%.3f', 'Start Time': iso_format, - }, + }, 'URI Set Tracking': { 'Avg': '%.3f', 'Max': '%.3f', 'Min': '%.3f', 'Sum': '%.3f', - }, + }, 'Requests': { 'Bytes Read': '%s', 'Bytes Written': '%s', 'End Time': None, 'Processing Time': '%.3f', 'Start Time': None, - }, + }, }, 'CherryPy WSGIServer': { 'Enabled': pause_resume('CherryPy WSGIServer'), @@ -449,7 +469,6 @@ class StatsPage(object): }, } - def index(self): # Transform the raw data into pretty output for HTML yield """ @@ -500,18 +519,25 @@ table.stats2 th { """ % title for i, (key, value) in enumerate(scalars): colnum = i % 3 - if colnum == 0: yield """ + if colnum == 0: + yield """ """ - yield """ - %(key)s%(value)s""" % vars() - if colnum == 2: yield """ + yield ( + """ + %(key)s%(value)s""" % + vars() + ) + if colnum == 2: + yield """ """ - if colnum == 0: yield """ + if colnum == 0: + yield """ """ - elif colnum == 1: yield """ + elif colnum == 1: + yield """ """ yield """ @@ -659,4 +685,3 @@ table.stats2 th { resume.exposed = True resume.cp_config = {'tools.allow.on': True, 'tools.allow.methods': ['POST']} - diff --git a/lib/cherrypy/lib/cptools.py b/lib/cherrypy/lib/cptools.py index 6f268f93..f376282c 100644 --- a/lib/cherrypy/lib/cptools.py +++ b/lib/cherrypy/lib/cptools.py @@ -4,8 +4,9 @@ import logging import re import cherrypy -from cherrypy._cpcompat import basestring, ntob, md5, set +from cherrypy._cpcompat import basestring, md5, set, unicodestr from cherrypy.lib import httputil as _httputil +from cherrypy.lib import is_iterator # Conditional HTTP request support # @@ -79,13 +80,15 @@ def validate_etags(autotags=False, debug=False): 'TOOLS.ETAGS') if conditions == ["*"] or etag in conditions: if debug: - cherrypy.log('request.method: %s' % request.method, 'TOOLS.ETAGS') + cherrypy.log('request.method: %s' % + request.method, 'TOOLS.ETAGS') if request.method in ("GET", "HEAD"): raise cherrypy.HTTPRedirect([], 304) else: raise cherrypy.HTTPError(412, "If-None-Match failed: ETag %r " "matched %r" % (etag, conditions)) + def validate_since(): """Validate the current Last-Modified against If-Modified-Since headers. @@ -207,8 +210,8 @@ def proxy(base=None, local='X-Forwarded-Host', remote='X-Forwarded-For', cherrypy.log('Testing remote %r:%r' % (remote, xff), 'TOOLS.PROXY') if xff: if remote == 'X-Forwarded-For': - # See http://bob.pythonmac.org/archives/2005/09/23/apache-x-forwarded-for-caveat/ - xff = xff.split(',')[-1].strip() + #Bug #1268 + xff = xff.split(',')[0].strip() request.remote.ip = xff @@ -277,6 +280,7 @@ def referer(pattern, accept=True, accept_missing=False, error=403, class SessionAuth(object): + """Assert that the user is logged in.""" session_key = "username" @@ -298,17 +302,20 @@ class SessionAuth(object): def on_check(self, username): pass - def login_screen(self, from_page='..', username='', error_msg='', **kwargs): - return ntob(""" + def login_screen(self, from_page='..', username='', error_msg='', + **kwargs): + return (unicodestr(""" Message: %(error_msg)s - Login:
    - Password:
    -
    + Login: +
    + Password: +
    + +
    -""" % {'from_page': from_page, 'username': username, - 'error_msg': error_msg}, "utf-8") +""") % vars()).encode("utf-8") def do_login(self, username, password, from_page='..', **kwargs): """Login. May raise redirect, or return True if request handled.""" @@ -338,7 +345,8 @@ Message: %(error_msg)s raise cherrypy.HTTPRedirect(from_page) def do_check(self): - """Assert username. May raise redirect, or return True if request handled.""" + """Assert username. Raise redirect, or return True if request handled. + """ sess = cherrypy.session request = cherrypy.serving.request response = cherrypy.serving.response @@ -346,51 +354,51 @@ Message: %(error_msg)s username = sess.get(self.session_key) if not username: sess[self.session_key] = username = self.anonymous() - if self.debug: - cherrypy.log('No session[username], trying anonymous', 'TOOLS.SESSAUTH') + self._debug_message('No session[username], trying anonymous') if not username: url = cherrypy.url(qs=request.query_string) - if self.debug: - cherrypy.log('No username, routing to login_screen with ' - 'from_page %r' % url, 'TOOLS.SESSAUTH') + self._debug_message( + 'No username, routing to login_screen with from_page %(url)r', + locals(), + ) response.body = self.login_screen(url) if "Content-Length" in response.headers: # Delete Content-Length header so finalize() recalcs it. del response.headers["Content-Length"] return True - if self.debug: - cherrypy.log('Setting request.login to %r' % username, 'TOOLS.SESSAUTH') + self._debug_message('Setting request.login to %(username)r', locals()) request.login = username self.on_check(username) + def _debug_message(self, template, context={}): + if not self.debug: + return + cherrypy.log(template % context, 'TOOLS.SESSAUTH') + def run(self): request = cherrypy.serving.request response = cherrypy.serving.response path = request.path_info if path.endswith('login_screen'): - if self.debug: - cherrypy.log('routing %r to login_screen' % path, 'TOOLS.SESSAUTH') - return self.login_screen(**request.params) + self._debug_message('routing %(path)r to login_screen', locals()) + response.body = self.login_screen() + return True elif path.endswith('do_login'): if request.method != 'POST': response.headers['Allow'] = "POST" - if self.debug: - cherrypy.log('do_login requires POST', 'TOOLS.SESSAUTH') + self._debug_message('do_login requires POST') raise cherrypy.HTTPError(405) - if self.debug: - cherrypy.log('routing %r to do_login' % path, 'TOOLS.SESSAUTH') + self._debug_message('routing %(path)r to do_login', locals()) return self.do_login(**request.params) elif path.endswith('do_logout'): if request.method != 'POST': response.headers['Allow'] = "POST" raise cherrypy.HTTPError(405) - if self.debug: - cherrypy.log('routing %r to do_logout' % path, 'TOOLS.SESSAUTH') + self._debug_message('routing %(path)r to do_logout', locals()) return self.do_logout(**request.params) else: - if self.debug: - cherrypy.log('No special path, running do_check', 'TOOLS.SESSAUTH') + self._debug_message('No special path, running do_check') return self.do_check() @@ -412,11 +420,13 @@ def log_traceback(severity=logging.ERROR, debug=False): """Write the last error's traceback to the cherrypy error log.""" cherrypy.log("", "HTTP", severity=severity, traceback=True) + def log_request_headers(debug=False): """Write request headers to the cherrypy error log.""" h = [" %s: %s" % (k, v) for k, v in cherrypy.serving.request.header_list] cherrypy.log('\nRequest Headers:\n' + '\n'.join(h), "HTTP") + def log_hooks(debug=False): """Write request.hooks to the cherrypy error log.""" request = cherrypy.serving.request @@ -438,6 +448,7 @@ def log_hooks(debug=False): cherrypy.log('\nRequest Hooks for ' + cherrypy.url() + ':\n' + '\n'.join(msg), "HTTP") + def redirect(url='', internal=True, debug=False): """Raise InternalRedirect or HTTPRedirect to the given url.""" if debug: @@ -449,6 +460,7 @@ def redirect(url='', internal=True, debug=False): else: raise cherrypy.HTTPRedirect(url) + def trailing_slash(missing=True, extra=False, status=None, debug=False): """Redirect if path_info has (missing|extra) trailing slash.""" request = cherrypy.serving.request @@ -470,17 +482,17 @@ def trailing_slash(missing=True, extra=False, status=None, debug=False): new_url = cherrypy.url(pi[:-1], request.query_string) raise cherrypy.HTTPRedirect(new_url, status=status or 301) + def flatten(debug=False): """Wrap response.body in a generator that recursively iterates over body. This allows cherrypy.response.body to consist of 'nested generators'; that is, a set of generators that yield generators. """ - import types def flattener(input): numchunks = 0 for x in input: - if not isinstance(x, types.GeneratorType): + if not is_iterator(x): numchunks += 1 yield x else: @@ -593,7 +605,8 @@ class MonitoredHeaderMap(_httputil.HeaderMap): def autovary(ignore=None, debug=False): - """Auto-populate the Vary response header based on request.header access.""" + """Auto-populate the Vary response header based on request.header access. + """ request = cherrypy.serving.request req_h = request.headers @@ -606,12 +619,12 @@ def autovary(ignore=None, debug=False): resp_h = cherrypy.serving.response.headers v = set([e.value for e in resp_h.elements('Vary')]) if debug: - cherrypy.log('Accessed headers: %s' % request.headers.accessed_headers, - 'TOOLS.AUTOVARY') + cherrypy.log( + 'Accessed headers: %s' % request.headers.accessed_headers, + 'TOOLS.AUTOVARY') v = v.union(request.headers.accessed_headers) v = v.difference(ignore) v = list(v) v.sort() resp_h['Vary'] = ', '.join(v) request.hooks.attach('before_finalize', set_response_header, 95) - diff --git a/lib/cherrypy/lib/encoding.py b/lib/cherrypy/lib/encoding.py index 1f68143b..a4c2cbd6 100644 --- a/lib/cherrypy/lib/encoding.py +++ b/lib/cherrypy/lib/encoding.py @@ -4,6 +4,7 @@ import time import cherrypy from cherrypy._cpcompat import basestring, BytesIO, ntob, set, unicodestr from cherrypy.lib import file_generator +from cherrypy.lib import is_closable_iterator from cherrypy.lib import set_vary_header @@ -14,13 +15,13 @@ def decode(encoding=None, default_encoding='utf-8'): encoding If not None, restricts the set of charsets attempted while decoding - a request entity to the given set (even if a different charset is given in - the Content-Type request header). + a request entity to the given set (even if a different charset is + given in the Content-Type request header). default_encoding Only in effect if the 'encoding' argument is not given. - If given, the set of charsets attempted while decoding a request entity is - *extended* with the given value(s). + If given, the set of charsets attempted while decoding a request + entity is *extended* with the given value(s). """ body = cherrypy.request.body @@ -33,6 +34,31 @@ def decode(encoding=None, default_encoding='utf-8'): default_encoding = [default_encoding] body.attempt_charsets = body.attempt_charsets + default_encoding +class UTF8StreamEncoder: + def __init__(self, iterator): + self._iterator = iterator + + def __iter__(self): + return self + + def next(self): + return self.__next__() + + def __next__(self): + res = next(self._iterator) + if isinstance(res, unicodestr): + res = res.encode('utf-8') + return res + + def close(self): + if is_closable_iterator(self._iterator): + self._iterator.close() + + def __getattr__(self, attr): + if attr.startswith('__'): + raise AttributeError(self, attr) + return getattr(self._iterator, attr) + class ResponseEncoder: @@ -80,25 +106,24 @@ class ResponseEncoder: if encoding in self.attempted_charsets: return False self.attempted_charsets.add(encoding) - - try: - body = [] - for chunk in self.body: - if isinstance(chunk, unicodestr): + body = [] + for chunk in self.body: + if isinstance(chunk, unicodestr): + try: chunk = chunk.encode(encoding, self.errors) - body.append(chunk) - self.body = body - except (LookupError, UnicodeError): - return False - else: - return True + except (LookupError, UnicodeError): + return False + body.append(chunk) + self.body = body + return True def find_acceptable_charset(self): request = cherrypy.serving.request response = cherrypy.serving.response if self.debug: - cherrypy.log('response.stream %r' % response.stream, 'TOOLS.ENCODE') + cherrypy.log('response.stream %r' % + response.stream, 'TOOLS.ENCODE') if response.stream: encoder = self.encode_stream else: @@ -127,10 +152,12 @@ class ResponseEncoder: # If specified, force this encoding to be used, or fail. encoding = self.encoding.lower() if self.debug: - cherrypy.log('Specified encoding %r' % encoding, 'TOOLS.ENCODE') + cherrypy.log('Specified encoding %r' % + encoding, 'TOOLS.ENCODE') if (not charsets) or "*" in charsets or encoding in charsets: if self.debug: - cherrypy.log('Attempting encoding %r' % encoding, 'TOOLS.ENCODE') + cherrypy.log('Attempting encoding %r' % + encoding, 'TOOLS.ENCODE') if encoder(encoding): return encoding else: @@ -142,7 +169,8 @@ class ResponseEncoder: if encoder(self.default_encoding): return self.default_encoding else: - raise cherrypy.HTTPError(500, self.failmsg % self.default_encoding) + raise cherrypy.HTTPError(500, self.failmsg % + self.default_encoding) else: for element in encs: if element.qvalue > 0: @@ -180,7 +208,8 @@ class ResponseEncoder: msg = "Your client did not send an Accept-Charset header." else: msg = "Your client sent this Accept-Charset header: %s." % ac - msg += " We tried these charsets: %s." % ", ".join(self.attempted_charsets) + _charsets = ", ".join(sorted(self.attempted_charsets)) + msg += " We tried these charsets: %s." % (_charsets,) raise cherrypy.HTTPError(406, msg) def __call__(self, *args, **kwargs): @@ -203,39 +232,42 @@ class ResponseEncoder: ct = response.headers.elements("Content-Type") if self.debug: - cherrypy.log('Content-Type: %r' % [str(h) for h in ct], 'TOOLS.ENCODE') - if ct: + cherrypy.log('Content-Type: %r' % [str(h) + for h in ct], 'TOOLS.ENCODE') + if ct and self.add_charset: ct = ct[0] if self.text_only: if ct.value.lower().startswith("text/"): if self.debug: - cherrypy.log('Content-Type %s starts with "text/"' % ct, - 'TOOLS.ENCODE') + cherrypy.log( + 'Content-Type %s starts with "text/"' % ct, + 'TOOLS.ENCODE') do_find = True else: if self.debug: - cherrypy.log('Not finding because Content-Type %s does ' - 'not start with "text/"' % ct, + cherrypy.log('Not finding because Content-Type %s ' + 'does not start with "text/"' % ct, 'TOOLS.ENCODE') do_find = False else: if self.debug: - cherrypy.log('Finding because not text_only', 'TOOLS.ENCODE') + cherrypy.log('Finding because not text_only', + 'TOOLS.ENCODE') do_find = True if do_find: # Set "charset=..." param on response Content-Type header ct.params['charset'] = self.find_acceptable_charset() - if self.add_charset: - if self.debug: - cherrypy.log('Setting Content-Type %s' % ct, - 'TOOLS.ENCODE') - response.headers["Content-Type"] = str(ct) + if self.debug: + cherrypy.log('Setting Content-Type %s' % ct, + 'TOOLS.ENCODE') + response.headers["Content-Type"] = str(ct) return self.body # GZIP + def compress(body, compress_level): """Compress 'body' at the given compress_level.""" import zlib @@ -265,6 +297,7 @@ def compress(body, compress_level): # ISIZE: 4 bytes yield struct.pack(" All rights reserved. -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Sylvain Hellegouarch nor the names of his contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. + * Neither the name of Sylvain Hellegouarch nor the names of his + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED @@ -57,7 +60,7 @@ __all__ = ("digestAuth", "basicAuth", "doAuth", "checkResponse", "parseAuthorization", "SUPPORTED_ALGORITHM", "md5SessionKey", "calculateNonce", "SUPPORTED_QOP") -################################################################################ +########################################################################## import time from cherrypy._cpcompat import base64_decode, ntob, md5 from cherrypy._cpcompat import parse_http_list, parse_keqv_list @@ -70,16 +73,17 @@ AUTH_INT = "auth-int" SUPPORTED_ALGORITHM = (MD5, MD5_SESS) SUPPORTED_QOP = (AUTH, AUTH_INT) -################################################################################ +########################################################################## # doAuth # DIGEST_AUTH_ENCODERS = { MD5: lambda val: md5(ntob(val)).hexdigest(), MD5_SESS: lambda val: md5(ntob(val)).hexdigest(), -# SHA: lambda val: sha.new(ntob(val)).hexdigest (), + # SHA: lambda val: sha.new(ntob(val)).hexdigest (), } -def calculateNonce (realm, algorithm = MD5): + +def calculateNonce(realm, algorithm=MD5): """This is an auxaliary function that calculates 'nonce' value. It is used to handle sessions.""" @@ -89,44 +93,47 @@ def calculateNonce (realm, algorithm = MD5): try: encoder = DIGEST_AUTH_ENCODERS[algorithm] except KeyError: - raise NotImplementedError ("The chosen algorithm (%s) does not have "\ - "an implementation yet" % algorithm) + raise NotImplementedError("The chosen algorithm (%s) does not have " + "an implementation yet" % algorithm) - return encoder ("%d:%s" % (time.time(), realm)) + return encoder("%d:%s" % (time.time(), realm)) -def digestAuth (realm, algorithm = MD5, nonce = None, qop = AUTH): + +def digestAuth(realm, algorithm=MD5, nonce=None, qop=AUTH): """Challenges the client for a Digest authentication.""" global SUPPORTED_ALGORITHM, DIGEST_AUTH_ENCODERS, SUPPORTED_QOP assert algorithm in SUPPORTED_ALGORITHM assert qop in SUPPORTED_QOP if nonce is None: - nonce = calculateNonce (realm, algorithm) + nonce = calculateNonce(realm, algorithm) return 'Digest realm="%s", nonce="%s", algorithm="%s", qop="%s"' % ( realm, nonce, algorithm, qop ) -def basicAuth (realm): + +def basicAuth(realm): """Challengenes the client for a Basic authentication.""" assert '"' not in realm, "Realms cannot contain the \" (quote) character." return 'Basic realm="%s"' % realm -def doAuth (realm): + +def doAuth(realm): """'doAuth' function returns the challenge string b giving priority over Digest and fallback to Basic authentication when the browser doesn't support the first one. This should be set in the HTTP header under the key 'WWW-Authenticate'.""" - return digestAuth (realm) + " " + basicAuth (realm) + return digestAuth(realm) + " " + basicAuth(realm) -################################################################################ +########################################################################## # Parse authorization parameters # -def _parseDigestAuthorization (auth_params): +def _parseDigestAuthorization(auth_params): # Convert the auth params to a dict items = parse_http_list(auth_params) params = parse_keqv_list(items) @@ -140,8 +147,8 @@ def _parseDigestAuthorization (auth_params): return None # If qop is sent then cnonce and nc MUST be present - if "qop" in params and not ("cnonce" in params \ - and "nc" in params): + if "qop" in params and not ("cnonce" in params + and "nc" in params): return None # If qop is not sent, neither cnonce nor nc can be present @@ -152,7 +159,7 @@ def _parseDigestAuthorization (auth_params): return params -def _parseBasicAuthorization (auth_params): +def _parseBasicAuthorization(auth_params): username, password = base64_decode(auth_params).split(":", 1) return {"username": username, "password": password} @@ -161,18 +168,19 @@ AUTH_SCHEMES = { "digest": _parseDigestAuthorization, } -def parseAuthorization (credentials): + +def parseAuthorization(credentials): """parseAuthorization will convert the value of the 'Authorization' key in the HTTP header to a map itself. If the parsing fails 'None' is returned. """ global AUTH_SCHEMES - auth_scheme, auth_params = credentials.split(" ", 1) - auth_scheme = auth_scheme.lower () + auth_scheme, auth_params = credentials.split(" ", 1) + auth_scheme = auth_scheme.lower() parser = AUTH_SCHEMES[auth_scheme] - params = parser (auth_params) + params = parser(auth_params) if params is None: return @@ -182,10 +190,10 @@ def parseAuthorization (credentials): return params -################################################################################ +########################################################################## # Check provided response for a valid password # -def md5SessionKey (params, password): +def md5SessionKey(params, password): """ If the "algorithm" directive's value is "MD5-sess", then A1 [the session key] is calculated only once - on the first request by the @@ -210,10 +218,11 @@ def md5SessionKey (params, password): params_copy[key] = params[key] params_copy["algorithm"] = MD5_SESS - return _A1 (params_copy, password) + return _A1(params_copy, password) + def _A1(params, password): - algorithm = params.get ("algorithm", MD5) + algorithm = params.get("algorithm", MD5) H = DIGEST_AUTH_ENCODERS[algorithm] if algorithm == MD5: @@ -227,7 +236,7 @@ def _A1(params, password): # This is A1 if qop is set # A1 = H( unq(username-value) ":" unq(realm-value) ":" passwd ) # ":" unq(nonce-value) ":" unq(cnonce-value) - h_a1 = H ("%s:%s:%s" % (params["username"], params["realm"], password)) + h_a1 = H("%s:%s:%s" % (params["username"], params["realm"], password)) return "%s:%s:%s" % (h_a1, params["nonce"], params["cnonce"]) @@ -235,13 +244,13 @@ def _A2(params, method, kwargs): # If the "qop" directive's value is "auth" or is unspecified, then A2 is: # A2 = Method ":" digest-uri-value - qop = params.get ("qop", "auth") + qop = params.get("qop", "auth") if qop == "auth": return method + ":" + params["uri"] elif qop == "auth-int": # If the "qop" value is "auth-int", then A2 is: # A2 = Method ":" digest-uri-value ":" H(entity-body) - entity_body = kwargs.get ("entity_body", "") + entity_body = kwargs.get("entity_body", "") H = kwargs["H"] return "%s:%s:%s" % ( @@ -251,20 +260,22 @@ def _A2(params, method, kwargs): ) else: - raise NotImplementedError ("The 'qop' method is unknown: %s" % qop) + raise NotImplementedError("The 'qop' method is unknown: %s" % qop) -def _computeDigestResponse(auth_map, password, method = "GET", A1 = None,**kwargs): + +def _computeDigestResponse(auth_map, password, method="GET", A1=None, + **kwargs): """ Generates a response respecting the algorithm defined in RFC 2617 """ params = auth_map - algorithm = params.get ("algorithm", MD5) + algorithm = params.get("algorithm", MD5) H = DIGEST_AUTH_ENCODERS[algorithm] KD = lambda secret, data: H(secret + ":" + data) - qop = params.get ("qop", None) + qop = params.get("qop", None) H_A2 = H(_A2(params, method, kwargs)) @@ -297,7 +308,8 @@ def _computeDigestResponse(auth_map, password, method = "GET", A1 = None,**kwarg return KD(H_A1, request) -def _checkDigestResponse(auth_map, password, method = "GET", A1 = None, **kwargs): + +def _checkDigestResponse(auth_map, password, method="GET", A1=None, **kwargs): """This function is used to verify the response given by the client when he tries to authenticate. Optional arguments: @@ -312,43 +324,48 @@ def _checkDigestResponse(auth_map, password, method = "GET", A1 = None, **kwargs if auth_map['realm'] != kwargs.get('realm', None): return False - response = _computeDigestResponse(auth_map, password, method, A1,**kwargs) + response = _computeDigestResponse( + auth_map, password, method, A1, **kwargs) return response == auth_map["response"] -def _checkBasicResponse (auth_map, password, method='GET', encrypt=None, **kwargs): + +def _checkBasicResponse(auth_map, password, method='GET', encrypt=None, + **kwargs): # Note that the Basic response doesn't provide the realm value so we cannot # test it + pass_through = lambda password, username=None: password + encrypt = encrypt or pass_through try: - return encrypt(auth_map["password"], auth_map["username"]) == password + candidate = encrypt(auth_map["password"], auth_map["username"]) except TypeError: - return encrypt(auth_map["password"]) == password + # if encrypt only takes one parameter, it's the password + candidate = encrypt(auth_map["password"]) + return candidate == password AUTH_RESPONSES = { "basic": _checkBasicResponse, "digest": _checkDigestResponse, } -def checkResponse (auth_map, password, method = "GET", encrypt=None, **kwargs): + +def checkResponse(auth_map, password, method="GET", encrypt=None, **kwargs): """'checkResponse' compares the auth_map with the password and optionally other arguments that each implementation might need. If the response is of type 'Basic' then the function has the following signature:: - checkBasicResponse (auth_map, password) -> bool + checkBasicResponse(auth_map, password) -> bool If the response is of type 'Digest' then the function has the following signature:: - checkDigestResponse (auth_map, password, method = 'GET', A1 = None) -> bool + checkDigestResponse(auth_map, password, method='GET', A1=None) -> bool The 'A1' argument is only used in MD5_SESS algorithm based responses. Check md5SessionKey() for more info. """ checker = AUTH_RESPONSES[auth_map["auth_scheme"]] - return checker (auth_map, password, method=method, encrypt=encrypt, **kwargs) - - - - + return checker(auth_map, password, method=method, encrypt=encrypt, + **kwargs) diff --git a/lib/cherrypy/lib/httputil.py b/lib/cherrypy/lib/httputil.py index e30942cb..69a18d45 100644 --- a/lib/cherrypy/lib/httputil.py +++ b/lib/cherrypy/lib/httputil.py @@ -8,24 +8,24 @@ to a public caning. """ from binascii import b2a_base64 -from cherrypy._cpcompat import BaseHTTPRequestHandler, HTTPDate, ntob, ntou, reversed, sorted -from cherrypy._cpcompat import basestring, bytestr, iteritems, nativestr, unicodestr, unquote_qs +from cherrypy._cpcompat import BaseHTTPRequestHandler, HTTPDate, ntob, ntou +from cherrypy._cpcompat import basestring, bytestr, iteritems, nativestr +from cherrypy._cpcompat import reversed, sorted, unicodestr, unquote_qs response_codes = BaseHTTPRequestHandler.responses.copy() -# From http://www.cherrypy.org/ticket/361 +# From https://bitbucket.org/cherrypy/cherrypy/issue/361 response_codes[500] = ('Internal Server Error', - 'The server encountered an unexpected condition ' - 'which prevented it from fulfilling the request.') + 'The server encountered an unexpected condition ' + 'which prevented it from fulfilling the request.') response_codes[503] = ('Service Unavailable', - 'The server is currently unable to handle the ' - 'request due to a temporary overloading or ' - 'maintenance of the server.') + 'The server is currently unable to handle the ' + 'request due to a temporary overloading or ' + 'maintenance of the server.') import re import urllib - def urljoin(*atoms): """Return the given path \*atoms, joined into a single URL. @@ -38,6 +38,7 @@ def urljoin(*atoms): # Special-case the final url of "", and return "/" instead. return url or "/" + def urljoin_bytes(*atoms): """Return the given path *atoms, joined into a single URL. @@ -50,10 +51,12 @@ def urljoin_bytes(*atoms): # Special-case the final url of "", and return "/" instead. return url or ntob("/") + def protocol_from_http(protocol_str): """Return a protocol tuple from the given 'HTTP/x.y' string.""" return int(protocol_str[5]), int(protocol_str[7]) + def get_ranges(headervalue, content_length): """Return a list of (start, stop) indices from a Range header, or None. @@ -100,12 +103,20 @@ def get_ranges(headervalue, content_length): # See rfc quote above. return None # Negative subscript (last N bytes) - result.append((content_length - int(stop), content_length)) + # + # RFC 2616 Section 14.35.1: + # If the entity is shorter than the specified suffix-length, + # the entire entity-body is used. + if int(stop) > content_length: + result.append((0, content_length)) + else: + result.append((content_length - int(stop), content_length)) return result class HeaderElement(object): + """An element (with parameters) from an HTTP header's element list.""" def __init__(self, value, params=None): @@ -122,7 +133,7 @@ class HeaderElement(object): def __str__(self): p = [";%s=%s" % (k, v) for k, v in iteritems(self.params)] - return "%s%s" % (self.value, "".join(p)) + return str("%s%s" % (self.value, "".join(p))) def __bytes__(self): return ntob(self.__str__()) @@ -160,7 +171,9 @@ class HeaderElement(object): q_separator = re.compile(r'; *q *=') + class AcceptElement(HeaderElement): + """An element (with parameters) from an Accept* header's element list. AcceptElement objects are comparable; the more-preferred object will be @@ -206,14 +219,15 @@ class AcceptElement(HeaderElement): else: return self.qvalue < other.qvalue - +RE_HEADER_SPLIT = re.compile(',(?=(?:[^"]*"[^"]*")*[^"]*$)') def header_elements(fieldname, fieldvalue): - """Return a sorted HeaderElement list from a comma-separated header string.""" + """Return a sorted HeaderElement list from a comma-separated header string. + """ if not fieldvalue: return [] result = [] - for element in fieldvalue.split(","): + for element in RE_HEADER_SPLIT.split(fieldvalue): if fieldname.startswith("Accept") or fieldname == 'TE': hv = AcceptElement.from_str(element) else: @@ -222,6 +236,7 @@ def header_elements(fieldname, fieldvalue): return list(reversed(sorted(result))) + def decode_TEXT(value): r"""Decode :rfc:`2047` TEXT (e.g. "=?utf-8?q?f=C3=BCr?=" -> "f\xfcr").""" try: @@ -237,6 +252,7 @@ def decode_TEXT(value): decodedvalue += atom return decodedvalue + def valid_status(status): """Return legal HTTP status Code, Reason-phrase and Message. @@ -332,6 +348,7 @@ def _parse_qs(qs, keep_blank_values=0, strict_parsing=0, encoding='utf-8'): image_map_pattern = re.compile(r"[0-9]+,[0-9]+") + def parse_query_string(query_string, keep_blank_values=True, encoding='utf-8'): """Build a params dictionary from a query_string. @@ -350,6 +367,7 @@ def parse_query_string(query_string, keep_blank_values=True, encoding='utf-8'): class CaseInsensitiveDict(dict): + """A case-insensitive dict subclass. Each key is changed on entry to str(key).title(). @@ -372,7 +390,7 @@ class CaseInsensitiveDict(dict): if hasattr({}, 'has_key'): def has_key(self, key): - return dict.has_key(self, str(key).title()) + return str(key).title() in self def update(self, E): for k in E.keys(): @@ -404,13 +422,15 @@ class CaseInsensitiveDict(dict): # replaced with a single SP before interpretation of the TEXT value." if nativestr == bytestr: header_translate_table = ''.join([chr(i) for i in xrange(256)]) - header_translate_deletechars = ''.join([chr(i) for i in xrange(32)]) + chr(127) + header_translate_deletechars = ''.join( + [chr(i) for i in xrange(32)]) + chr(127) else: header_translate_table = None header_translate_deletechars = bytes(range(32)) + bytes([127]) class HeaderMap(CaseInsensitiveDict): + """A dict subclass for HTTP request and response headers. Each key is changed on entry to str(key).title(). This allows headers @@ -419,7 +439,7 @@ class HeaderMap(CaseInsensitiveDict): Values are header values (decoded according to :rfc:`2047` if necessary). """ - protocol=(1, 1) + protocol = (1, 1) encodings = ["ISO-8859-1"] # Someday, when http-bis is done, this will probably get dropped @@ -441,34 +461,42 @@ class HeaderMap(CaseInsensitiveDict): def output(self): """Transform self into a list of (name, value) tuples.""" - header_list = [] - for k, v in self.items(): + return list(self.encode_header_items(self.items())) + + def encode_header_items(cls, header_items): + """ + Prepare the sequence of name, value tuples into a form suitable for + transmitting on the wire for HTTP. + """ + for k, v in header_items: if isinstance(k, unicodestr): - k = self.encode(k) + k = cls.encode(k) if not isinstance(v, basestring): v = str(v) if isinstance(v, unicodestr): - v = self.encode(v) + v = cls.encode(v) # See header_translate_* constants above. # Replace only if you really know what you're doing. - k = k.translate(header_translate_table, header_translate_deletechars) - v = v.translate(header_translate_table, header_translate_deletechars) + k = k.translate(header_translate_table, + header_translate_deletechars) + v = v.translate(header_translate_table, + header_translate_deletechars) - header_list.append((k, v)) - return header_list + yield (k, v) + encode_header_items = classmethod(encode_header_items) - def encode(self, v): + def encode(cls, v): """Return the given header name or value, encoded for HTTP output.""" - for enc in self.encodings: + for enc in cls.encodings: try: return v.encode(enc) except UnicodeEncodeError: continue - if self.protocol == (1, 1) and self.use_rfc_2047: + if cls.protocol == (1, 1) and cls.use_rfc_2047: # Encode RFC-2047 TEXT # (e.g. u"\u8200" -> "=?utf-8?b?6IiA?="). # We do our own here instead of using the email module @@ -479,10 +507,12 @@ class HeaderMap(CaseInsensitiveDict): raise ValueError("Could not encode header part %r using " "any of the encodings %r." % - (v, self.encodings)) + (v, cls.encodings)) + encode = classmethod(encode) class Host(object): + """An internet address. name diff --git a/lib/cherrypy/lib/jsontools.py b/lib/cherrypy/lib/jsontools.py index 776bddf6..90b3ff8a 100644 --- a/lib/cherrypy/lib/jsontools.py +++ b/lib/cherrypy/lib/jsontools.py @@ -1,6 +1,6 @@ -import sys import cherrypy -from cherrypy._cpcompat import basestring, ntou, json, json_encode, json_decode +from cherrypy._cpcompat import basestring, ntou, json_encode, json_decode + def json_processor(entity): """Read application/json data into request.json.""" @@ -13,8 +13,9 @@ def json_processor(entity): except ValueError: raise cherrypy.HTTPError(400, 'Invalid JSON document') + def json_in(content_type=[ntou('application/json'), ntou('text/javascript')], - force=True, debug=False, processor = json_processor): + force=True, debug=False, processor=json_processor): """Add a processor to parse JSON request entities: The default processor places the parsed data into request.json. @@ -57,11 +58,14 @@ def json_in(content_type=[ntou('application/json'), ntou('text/javascript')], cherrypy.log('Adding body processor for %s' % ct, 'TOOLS.JSON_IN') request.body.processors[ct] = processor + def json_handler(*args, **kwargs): value = cherrypy.serving.request._json_inner_handler(*args, **kwargs) return json_encode(value) -def json_out(content_type='application/json', debug=False, handler=json_handler): + +def json_out(content_type='application/json', debug=False, + handler=json_handler): """Wrap request.handler to serialize its output to JSON. Sets Content-Type. If the given content_type is None, the Content-Type response header @@ -75,6 +79,11 @@ def json_out(content_type='application/json', debug=False, handler=json_handler) package importable; otherwise, ValueError is raised during processing. """ request = cherrypy.serving.request + # request.handler may be set to None by e.g. the caching tool + # to signal to all components that a response body has already + # been attached, in which case we don't need to wrap anything. + if request.handler is None: + return if debug: cherrypy.log('Replacing %s with JSON handler' % request.handler, 'TOOLS.JSON_OUT') @@ -82,6 +91,6 @@ def json_out(content_type='application/json', debug=False, handler=json_handler) request.handler = handler if content_type is not None: if debug: - cherrypy.log('Setting Content-Type to %s' % content_type, 'TOOLS.JSON_OUT') + cherrypy.log('Setting Content-Type to %s' % + content_type, 'TOOLS.JSON_OUT') cherrypy.serving.response.headers['Content-Type'] = content_type - diff --git a/lib/cherrypy/lib/lockfile.py b/lib/cherrypy/lib/lockfile.py new file mode 100644 index 00000000..4cf7b1b6 --- /dev/null +++ b/lib/cherrypy/lib/lockfile.py @@ -0,0 +1,147 @@ +""" +Platform-independent file locking. Inspired by and modeled after zc.lockfile. +""" + +import os + +try: + import msvcrt +except ImportError: + pass + +try: + import fcntl +except ImportError: + pass + + +class LockError(Exception): + + "Could not obtain a lock" + + msg = "Unable to lock %r" + + def __init__(self, path): + super(LockError, self).__init__(self.msg % path) + + +class UnlockError(LockError): + + "Could not release a lock" + + msg = "Unable to unlock %r" + + +# first, a default, naive locking implementation +class LockFile(object): + + """ + A default, naive locking implementation. Always fails if the file + already exists. + """ + + def __init__(self, path): + self.path = path + try: + fd = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_EXCL) + except OSError: + raise LockError(self.path) + os.close(fd) + + def release(self): + os.remove(self.path) + + def remove(self): + pass + + +class SystemLockFile(object): + + """ + An abstract base class for platform-specific locking. + """ + + def __init__(self, path): + self.path = path + + try: + # Open lockfile for writing without truncation: + self.fp = open(path, 'r+') + except IOError: + # If the file doesn't exist, IOError is raised; Use a+ instead. + # Note that there may be a race here. Multiple processes + # could fail on the r+ open and open the file a+, but only + # one will get the the lock and write a pid. + self.fp = open(path, 'a+') + + try: + self._lock_file() + except: + self.fp.seek(1) + self.fp.close() + del self.fp + raise + + self.fp.write(" %s\n" % os.getpid()) + self.fp.truncate() + self.fp.flush() + + def release(self): + if not hasattr(self, 'fp'): + return + self._unlock_file() + self.fp.close() + del self.fp + + def remove(self): + """ + Attempt to remove the file + """ + try: + os.remove(self.path) + except: + pass + + #@abc.abstract_method + # def _lock_file(self): + # """Attempt to obtain the lock on self.fp. Raise LockError if not + # acquired.""" + + def _unlock_file(self): + """Attempt to obtain the lock on self.fp. Raise UnlockError if not + released.""" + + +class WindowsLockFile(SystemLockFile): + + def _lock_file(self): + # Lock just the first byte + try: + msvcrt.locking(self.fp.fileno(), msvcrt.LK_NBLCK, 1) + except IOError: + raise LockError(self.fp.name) + + def _unlock_file(self): + try: + self.fp.seek(0) + msvcrt.locking(self.fp.fileno(), msvcrt.LK_UNLCK, 1) + except IOError: + raise UnlockError(self.fp.name) + +if 'msvcrt' in globals(): + LockFile = WindowsLockFile + + +class UnixLockFile(SystemLockFile): + + def _lock_file(self): + flags = fcntl.LOCK_EX | fcntl.LOCK_NB + try: + fcntl.flock(self.fp.fileno(), flags) + except IOError: + raise LockError(self.fp.name) + + # no need to implement _unlock_file, it will be unlocked on close() + +if 'fcntl' in globals(): + LockFile = UnixLockFile diff --git a/lib/cherrypy/lib/locking.py b/lib/cherrypy/lib/locking.py new file mode 100644 index 00000000..72dda9b3 --- /dev/null +++ b/lib/cherrypy/lib/locking.py @@ -0,0 +1,47 @@ +import datetime + + +class NeverExpires(object): + def expired(self): + return False + + +class Timer(object): + """ + A simple timer that will indicate when an expiration time has passed. + """ + def __init__(self, expiration): + "Create a timer that expires at `expiration` (UTC datetime)" + self.expiration = expiration + + @classmethod + def after(cls, elapsed): + """ + Return a timer that will expire after `elapsed` passes. + """ + return cls(datetime.datetime.utcnow() + elapsed) + + def expired(self): + return datetime.datetime.utcnow() >= self.expiration + + +class LockTimeout(Exception): + "An exception when a lock could not be acquired before a timeout period" + + +class LockChecker(object): + """ + Keep track of the time and detect if a timeout has expired + """ + def __init__(self, session_id, timeout): + self.session_id = session_id + if timeout: + self.timer = Timer.after(timeout) + else: + self.timer = NeverExpires() + + def expired(self): + if self.timer.expired(): + raise LockTimeout( + "Timeout acquiring lock for %(session_id)s" % vars(self)) + return False diff --git a/lib/cherrypy/lib/profiler.py b/lib/cherrypy/lib/profiler.py index 6ac676b8..5dac386e 100644 --- a/lib/cherrypy/lib/profiler.py +++ b/lib/cherrypy/lib/profiler.py @@ -35,7 +35,8 @@ module from the command line, it will call ``serve()`` for you. def new_func_strip_path(func_name): - """Make profiler output more readable by adding ``__init__`` modules' parents""" + """Make profiler output more readable by adding `__init__` modules' parents + """ filename, line, name = func_name if filename.endswith("__init__.py"): return os.path.basename(filename[:-12]) + filename[-12:], line, name @@ -49,14 +50,16 @@ except ImportError: profile = None pstats = None -import os, os.path +import os +import os.path import sys import warnings -from cherrypy._cpcompat import BytesIO +from cherrypy._cpcompat import StringIO _count = 0 + class Profiler(object): def __init__(self, path=None): @@ -85,7 +88,7 @@ class Profiler(object): def stats(self, filename, sortby='cumulative'): """:rtype stats(index): output of print_stats() for the given profile. """ - sio = BytesIO() + sio = StringIO() if sys.version_info >= (2, 5): s = pstats.Stats(os.path.join(self.path, filename), stream=sio) s.strip_dirs() @@ -124,7 +127,8 @@ class Profiler(object): runs = self.statfiles() runs.sort() for i in runs: - yield "%s
    " % (i, i) + yield "%s
    " % ( + i, i) menu.exposed = True def report(self, filename): @@ -142,14 +146,15 @@ class ProfileAggregator(Profiler): self.count = _count = _count + 1 self.profiler = profile.Profile() - def run(self, func, *args): + def run(self, func, *args, **params): path = os.path.join(self.path, "cp_%04d.prof" % self.count) - result = self.profiler.runcall(func, *args) + result = self.profiler.runcall(func, *args, **params) self.profiler.dump_stats(path) return result class make_app: + def __init__(self, nextapp, path=None, aggregate=False): """Make a WSGI middleware app which wraps 'nextapp' with profiling. @@ -167,9 +172,11 @@ class make_app: """ if profile is None or pstats is None: - msg = ("Your installation of Python does not have a profile module. " - "If you're on Debian, try `sudo apt-get install python-profiler`. " - "See http://www.cherrypy.org/wiki/ProfilingOnDebian for details.") + msg = ("Your installation of Python does not have a profile " + "module. If you're on Debian, try " + "`sudo apt-get install python-profiler`. " + "See http://www.cherrypy.org/wiki/ProfilingOnDebian " + "for details.") warnings.warn(msg) self.nextapp = nextapp @@ -191,8 +198,10 @@ class make_app: def serve(path=None, port=8080): if profile is None or pstats is None: msg = ("Your installation of Python does not have a profile module. " - "If you're on Debian, try `sudo apt-get install python-profiler`. " - "See http://www.cherrypy.org/wiki/ProfilingOnDebian for details.") + "If you're on Debian, try " + "`sudo apt-get install python-profiler`. " + "See http://www.cherrypy.org/wiki/ProfilingOnDebian " + "for details.") warnings.warn(msg) import cherrypy @@ -205,4 +214,3 @@ def serve(path=None, port=8080): if __name__ == "__main__": serve(*tuple(sys.argv[1:])) - diff --git a/lib/cherrypy/lib/reprconf.py b/lib/cherrypy/lib/reprconf.py index 502b2c4d..6e70b5ec 100644 --- a/lib/cherrypy/lib/reprconf.py +++ b/lib/cherrypy/lib/reprconf.py @@ -44,6 +44,7 @@ except ImportError: import operator as _operator import sys + def as_dict(config): """Return a dict from 'config' whether it is a dict, file, or filename.""" if isinstance(config, basestring): @@ -54,6 +55,7 @@ def as_dict(config): class NamespaceSet(dict): + """A dict of config namespace names and handlers. Each config entry should begin with a namespace name; the corresponding @@ -129,6 +131,7 @@ class NamespaceSet(dict): class Config(dict): + """A dict-like set of configuration data, with defaults and namespaces. May take a file, filename, or dict. @@ -180,6 +183,7 @@ class Config(dict): class Parser(ConfigParser): + """Sub-class of ConfigParser that keeps the case of options and that raises an exception if the file cannot be read. """ @@ -260,13 +264,31 @@ class _Builder2: return expr[subs] def build_CallFunc(self, o): - children = map(self.build, o.getChildren()) - callee = children.pop(0) - kwargs = children.pop() or {} - starargs = children.pop() or () - args = tuple(children) + tuple(starargs) + children = o.getChildren() + # Build callee from first child + callee = self.build(children[0]) + # Build args and kwargs from remaining children + args = [] + kwargs = {} + for child in children[1:]: + class_name = child.__class__.__name__ + # None is ignored + if class_name == 'NoneType': + continue + # Keywords become kwargs + if class_name == 'Keyword': + kwargs.update(self.build(child)) + # Everything else becomes args + else : + args.append(self.build(child)) return callee(*args, **kwargs) + def build_Keyword(self, o): + key, value_obj = o.getChildren() + value = self.build(value_obj) + kw_dict = {key: value} + return kw_dict + def build_List(self, o): return map(self.build, o.getChildren()) @@ -415,6 +437,9 @@ class _Builder3: raise TypeError("unrepr could not resolve the name %s" % repr(name)) + def build_NameConstant(self, o): + return o.value + def build_UnaryOp(self, o): op, operand = map(self.build, [o.op, o.operand]) return op(operand) @@ -454,14 +479,9 @@ def unrepr(s): def modules(modulePath): """Load a module and retrieve a reference to that module.""" - try: - mod = sys.modules[modulePath] - if mod is None: - raise KeyError() - except KeyError: - # The last [''] is important. - mod = __import__(modulePath, globals(), locals(), ['']) - return mod + __import__(modulePath) + return sys.modules[modulePath] + def attributes(full_attribute_name): """Load a module and retrieve an attribute of that module.""" @@ -481,5 +501,3 @@ def attributes(full_attribute_name): # Return a reference to the attribute. return attr - - diff --git a/lib/cherrypy/lib/sessions.py b/lib/cherrypy/lib/sessions.py index 9c5a4b27..37556363 100644 --- a/lib/cherrypy/lib/sessions.py +++ b/lib/cherrypy/lib/sessions.py @@ -8,10 +8,11 @@ You need to edit your config file to use sessions. Here's an example:: tools.sessions.storage_path = "/home/site/sessions" tools.sessions.timeout = 60 -This sets the session to be stored in files in the directory /home/site/sessions, -and the session timeout to 60 minutes. If you omit ``storage_type`` the sessions -will be saved in RAM. ``tools.sessions.on`` is the only required line for -working sessions, the rest are optional. +This sets the session to be stored in files in the directory +/home/site/sessions, and the session timeout to 60 minutes. If you omit +``storage_type`` the sessions will be saved in RAM. +``tools.sessions.on`` is the only required line for working sessions, +the rest are optional. By default, the session ID is passed in a cookie, so the client's browser must have cookies enabled for your site. @@ -25,9 +26,14 @@ Locking sessions ================ By default, the ``'locking'`` mode of sessions is ``'implicit'``, which means -the session is locked early and unlocked late. If you want to control when the -session data is locked and unlocked, set ``tools.sessions.locking = 'explicit'``. -Then call ``cherrypy.session.acquire_lock()`` and ``cherrypy.session.release_lock()``. +the session is locked early and unlocked late. Be mindful of this default mode +for any requests that take a long time to process (streaming responses, +expensive calculations, database lookups, API calls, etc), as other concurrent +requests that also utilize sessions will hang until the session is unlocked. + +If you want to control when the session data is locked and unlocked, +set ``tools.sessions.locking = 'explicit'``. Then call +``cherrypy.session.acquire_lock()`` and ``cherrypy.session.release_lock()``. Regardless of which mode you use, the session is guaranteed to be unlocked when the request is complete. @@ -83,23 +89,25 @@ On the other extreme, some users report Firefox sending cookies after their expiration date, although this was on a system with an inaccurate system time. Maybe FF doesn't trust system time. """ - +import sys import datetime import os -import random import time import threading import types -from warnings import warn import cherrypy from cherrypy._cpcompat import copyitems, pickle, random20, unicodestr from cherrypy.lib import httputil - +from cherrypy.lib import lockfile +from cherrypy.lib import locking +from cherrypy.lib import is_iterator missing = object() + class Session(object): + """A CherryPy dict-like Session object (one per request).""" _id = None @@ -109,6 +117,7 @@ class Session(object): def _get_id(self): return self._id + def _set_id(self, value): self._id = value for o in self.id_observers: @@ -145,7 +154,10 @@ class Session(object): True if the application called session.regenerate(). This is not set by internal calls to regenerate the session id.""" - debug=False + debug = False + "If True, log debug information." + + # --------------------- Session management methods --------------------- # def __init__(self, id=None, **kwargs): self.id_observers = [] @@ -162,12 +174,15 @@ class Session(object): self._regenerate() else: self.id = id - if not self._exists(): + if self._exists(): + if self.debug: + cherrypy.log('Set id to %s.' % id, 'TOOLS.SESSIONS') + else: if self.debug: cherrypy.log('Expired or malicious session %r; ' 'making a new one' % id, 'TOOLS.SESSIONS') # Expired or malicious session. Make a new one. - # See http://www.cherrypy.org/ticket/709. + # See https://bitbucket.org/cherrypy/cherrypy/issue/709. self.id = None self.missing = True self._regenerate() @@ -187,11 +202,18 @@ class Session(object): def _regenerate(self): if self.id is not None: + if self.debug: + cherrypy.log( + 'Deleting the existing session %r before ' + 'regeneration.' % self.id, + 'TOOLS.SESSIONS') self.delete() old_session_was_locked = self.locked if old_session_was_locked: self.release_lock() + if self.debug: + cherrypy.log('Old lock released.', 'TOOLS.SESSIONS') self.id = None while self.id is None: @@ -199,9 +221,14 @@ class Session(object): # Assert that the generated id is not already stored. if self._exists(): self.id = None + if self.debug: + cherrypy.log('Set id to generated %s.' % self.id, + 'TOOLS.SESSIONS') if old_session_was_locked: self.acquire_lock() + if self.debug: + cherrypy.log('Regenerated lock acquired.', 'TOOLS.SESSIONS') def clean_up(self): """Clean up expired sessions.""" @@ -217,17 +244,24 @@ class Session(object): # If session data has never been loaded then it's never been # accessed: no need to save it if self.loaded: - t = datetime.timedelta(seconds = self.timeout * 60) + t = datetime.timedelta(seconds=self.timeout * 60) expiration_time = self.now() + t if self.debug: - cherrypy.log('Saving with expiry %s' % expiration_time, + cherrypy.log('Saving session %r with expiry %s' % + (self.id, expiration_time), 'TOOLS.SESSIONS') self._save(expiration_time) - + else: + if self.debug: + cherrypy.log( + 'Skipping save of session %r (no session loaded).' % + self.id, 'TOOLS.SESSIONS') finally: if self.locked: # Always release the lock if the user didn't release it self.release_lock() + if self.debug: + cherrypy.log('Lock released after save.', 'TOOLS.SESSIONS') def load(self): """Copy stored session data into this session instance.""" @@ -235,9 +269,13 @@ class Session(object): # data is either None or a tuple (session_data, expiration_time) if data is None or data[1] < self.now(): if self.debug: - cherrypy.log('Expired session, flushing data', 'TOOLS.SESSIONS') + cherrypy.log('Expired session %r, flushing data.' % self.id, + 'TOOLS.SESSIONS') self._data = {} else: + if self.debug: + cherrypy.log('Data loaded for session %r.' % self.id, + 'TOOLS.SESSIONS') self._data = data[0] self.loaded = True @@ -245,7 +283,7 @@ class Session(object): # The instances are created and destroyed per-request. cls = self.__class__ if self.clean_freq and not cls.clean_thread: - # clean_up is in instancemethod and not a classmethod, + # clean_up is an instancemethod and not a classmethod, # so that tool config can be accessed inside the method. t = cherrypy.process.plugins.Monitor( cherrypy.engine, self.clean_up, self.clean_freq * 60, @@ -253,21 +291,31 @@ class Session(object): t.subscribe() cls.clean_thread = t t.start() + if self.debug: + cherrypy.log('Started cleanup thread.', 'TOOLS.SESSIONS') def delete(self): """Delete stored session data.""" self._delete() + if self.debug: + cherrypy.log('Deleted session %s.' % self.id, + 'TOOLS.SESSIONS') + + # -------------------- Application accessor methods -------------------- # def __getitem__(self, key): - if not self.loaded: self.load() + if not self.loaded: + self.load() return self._data[key] def __setitem__(self, key, value): - if not self.loaded: self.load() + if not self.loaded: + self.load() self._data[key] = value def __delitem__(self, key): - if not self.loaded: self.load() + if not self.loaded: + self.load() del self._data[key] def pop(self, key, default=missing): @@ -275,55 +323,65 @@ class Session(object): If key is not found, default is returned if given, otherwise KeyError is raised. """ - if not self.loaded: self.load() + if not self.loaded: + self.load() if default is missing: return self._data.pop(key) else: return self._data.pop(key, default) def __contains__(self, key): - if not self.loaded: self.load() + if not self.loaded: + self.load() return key in self._data if hasattr({}, 'has_key'): def has_key(self, key): """D.has_key(k) -> True if D has a key k, else False.""" - if not self.loaded: self.load() + if not self.loaded: + self.load() return key in self._data def get(self, key, default=None): """D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.""" - if not self.loaded: self.load() + if not self.loaded: + self.load() return self._data.get(key, default) def update(self, d): """D.update(E) -> None. Update D from E: for k in E: D[k] = E[k].""" - if not self.loaded: self.load() + if not self.loaded: + self.load() self._data.update(d) def setdefault(self, key, default=None): """D.setdefault(k[,d]) -> D.get(k,d), also set D[k]=d if k not in D.""" - if not self.loaded: self.load() + if not self.loaded: + self.load() return self._data.setdefault(key, default) def clear(self): """D.clear() -> None. Remove all items from D.""" - if not self.loaded: self.load() + if not self.loaded: + self.load() self._data.clear() def keys(self): """D.keys() -> list of D's keys.""" - if not self.loaded: self.load() + if not self.loaded: + self.load() return self._data.keys() def items(self): """D.items() -> list of D's (key, value) pairs, as 2-tuples.""" - if not self.loaded: self.load() + if not self.loaded: + self.load() return self._data.items() def values(self): """D.values() -> list of D's values.""" - if not self.loaded: self.load() + if not self.loaded: + self.load() return self._data.values() @@ -335,22 +393,26 @@ class RamSession(Session): def clean_up(self): """Clean up expired sessions.""" + now = self.now() - for id, (data, expiration_time) in copyitems(self.cache): + for _id, (data, expiration_time) in copyitems(self.cache): if expiration_time <= now: try: - del self.cache[id] + del self.cache[_id] except KeyError: pass try: - del self.locks[id] + if self.locks[_id].acquire(blocking=False): + lock = self.locks.pop(_id) + lock.release() except KeyError: pass # added to remove obsolete lock objects - for id in list(self.locks): - if id not in self.cache: - self.locks.pop(id, None) + for _id in list(self.locks): + if _id not in self.cache and self.locks[_id].acquire(blocking=False): + lock = self.locks.pop(_id) + lock.release() def _exists(self): return self.id in self.cache @@ -380,6 +442,7 @@ class RamSession(Session): class FileSession(Session): + """Implementation of the File backend for sessions storage_path @@ -387,6 +450,10 @@ class FileSession(Session): will be saved as pickle.dump(data, expiration_time) in its own file; the filename will be self.SESSION_PREFIX + self.id. + lock_timeout + A timedelta or numeric seconds indicating how long + to block acquiring a lock. If None (default), acquiring a lock + will block indefinitely. """ SESSION_PREFIX = 'session-' @@ -396,8 +463,17 @@ class FileSession(Session): def __init__(self, id=None, **kwargs): # The 'storage_path' arg is required for file-based sessions. kwargs['storage_path'] = os.path.abspath(kwargs['storage_path']) + kwargs.setdefault('lock_timeout', None) + Session.__init__(self, id=id, **kwargs) + # validate self.lock_timeout + if isinstance(self.lock_timeout, (int, float)): + self.lock_timeout = datetime.timedelta(seconds=self.lock_timeout) + if not isinstance(self.lock_timeout, (datetime.timedelta, type(None))): + raise ValueError("Lock timeout must be numeric seconds or " + "a timedelta instance.") + def setup(cls, **kwargs): """Set up the storage system for file-based sessions. @@ -409,17 +485,6 @@ class FileSession(Session): for k, v in kwargs.items(): setattr(cls, k, v) - - # Warn if any lock files exist at startup. - lockfiles = [fname for fname in os.listdir(cls.storage_path) - if (fname.startswith(cls.SESSION_PREFIX) - and fname.endswith(cls.LOCK_SUFFIX))] - if lockfiles: - plural = ('', 's')[len(lockfiles) > 1] - warn("%s session lockfile%s found at startup. If you are " - "only running one process, then you may need to " - "manually delete the lockfiles found at %r." - % (len(lockfiles), plural, cls.storage_path)) setup = classmethod(setup) def _get_file_path(self): @@ -433,6 +498,8 @@ class FileSession(Session): return os.path.exists(path) def _load(self, path=None): + assert self.locked, ("The session load without being locked. " + "Check your tools' priority levels.") if path is None: path = self._get_file_path() try: @@ -442,9 +509,15 @@ class FileSession(Session): finally: f.close() except (IOError, EOFError): + e = sys.exc_info()[1] + if self.debug: + cherrypy.log("Error loading the session pickle: %s" % + e, 'TOOLS.SESSIONS') return None def _save(self, expiration_time): + assert self.locked, ("The session was saved without being locked. " + "Check your tools' priority levels.") f = open(self._get_file_path(), "wb") try: pickle.dump((self._data, expiration_time), f, self.pickle_protocol) @@ -452,6 +525,8 @@ class FileSession(Session): f.close() def _delete(self): + assert self.locked, ("The session deletion without being locked. " + "Check your tools' priority levels.") try: os.unlink(self._get_file_path()) except OSError: @@ -462,21 +537,22 @@ class FileSession(Session): if path is None: path = self._get_file_path() path += self.LOCK_SUFFIX - while True: + checker = locking.LockChecker(self.id, self.lock_timeout) + while not checker.expired(): try: - lockfd = os.open(path, os.O_CREAT|os.O_WRONLY|os.O_EXCL) - except OSError: + self.lock = lockfile.LockFile(path) + except lockfile.LockError: time.sleep(0.1) else: - os.close(lockfd) break self.locked = True + if self.debug: + cherrypy.log('Lock acquired.', 'TOOLS.SESSIONS') def release_lock(self, path=None): """Release the lock on the currently-loaded session data.""" - if path is None: - path = self._get_file_path() - os.unlink(path + self.LOCK_SUFFIX) + self.lock.release() + self.lock.remove() self.locked = False def clean_up(self): @@ -485,11 +561,18 @@ class FileSession(Session): # Iterate over all session files in self.storage_path for fname in os.listdir(self.storage_path): if (fname.startswith(self.SESSION_PREFIX) - and not fname.endswith(self.LOCK_SUFFIX)): + and not fname.endswith(self.LOCK_SUFFIX)): # We have a session file: lock and load it and check # if it's expired. If it fails, nevermind. path = os.path.join(self.storage_path, fname) self.acquire_lock(path) + if self.debug: + # This is a bit of a hack, since we're calling clean_up + # on the first instance rather than the entire class, + # so depending on whether you have "debug" set on the + # path of the first session called, this may not run. + cherrypy.log('Cleanup lock acquired.', 'TOOLS.SESSIONS') + try: contents = self._load(path) # _load returns None on IOError @@ -509,6 +592,7 @@ class FileSession(Session): class PostgresqlSession(Session): + """ Implementation of the PostgreSQL backend for sessions. It assumes a table like this:: @@ -578,6 +662,8 @@ class PostgresqlSession(Session): self.locked = True self.cursor.execute('select id from session where id=%s for update', (self.id,)) + if self.debug: + cherrypy.log('Lock acquired.', 'TOOLS.SESSIONS') def release_lock(self): """Release the lock on the currently-loaded session data.""" @@ -618,6 +704,7 @@ class MemcachedSession(Session): def _get_id(self): return self._id + def _set_id(self, value): # This encode() call is where we differ from the superclass. # Memcache keys MUST be byte strings, not unicode. @@ -649,7 +736,8 @@ class MemcachedSession(Session): self.mc_lock.acquire() try: if not self.cache.set(self.id, (self._data, expiration_time), td): - raise AssertionError("Session data for id %r not set." % self.id) + raise AssertionError( + "Session data for id %r not set." % self.id) finally: self.mc_lock.release() @@ -660,6 +748,8 @@ class MemcachedSession(Session): """Acquire an exclusive lock on the currently-loaded session data.""" self.locked = True self.locks.setdefault(self.id, threading.RLock()).acquire() + if self.debug: + cherrypy.log('Lock acquired.', 'TOOLS.SESSIONS') def release_lock(self): """Release the lock on the currently-loaded session data.""" @@ -693,17 +783,20 @@ def save(): else: # If the body is not being streamed, we save the data now # (so we can release the lock). - if isinstance(response.body, types.GeneratorType): + if is_iterator(response.body): response.collapse_body() cherrypy.session.save() save.failsafe = True + def close(): """Close the session object for this request.""" sess = getattr(cherrypy.serving, "session", None) if getattr(sess, "locked", False): # If the session is still locked we release the lock sess.release_lock() + if sess.debug: + cherrypy.log('Lock released on close.', 'TOOLS.SESSIONS') close.failsafe = True close.priority = 90 @@ -787,6 +880,7 @@ def init(storage_type='ram', path=None, path_header=None, name='session_id', kwargs['clean_freq'] = clean_freq cherrypy.serving.session = sess = storage_class(id, **kwargs) sess.debug = debug + def update_cookie(id): """Update the cookie every time the session id changes.""" cherrypy.serving.response.cookie[name] = id @@ -841,8 +935,11 @@ def set_response_cookie(path=None, path_header=None, name='session_id', # Set response cookie cookie = cherrypy.serving.response.cookie cookie[name] = cherrypy.serving.session.id - cookie[name]['path'] = (path or cherrypy.serving.request.headers.get(path_header) - or '/') + cookie[name]['path'] = ( + path or + cherrypy.serving.request.headers.get(path_header) or + '/' + ) # We'd like to use the "max-age" param as indicated in # http://www.faqs.org/rfcs/rfc2109.html but IE doesn't @@ -861,11 +958,11 @@ def set_response_cookie(path=None, path_header=None, name='session_id', raise ValueError("The httponly cookie token is not supported.") cookie[name]['httponly'] = 1 + def expire(): """Expire the current session cookie.""" - name = cherrypy.serving.request.config.get('tools.sessions.name', 'session_id') + name = cherrypy.serving.request.config.get( + 'tools.sessions.name', 'session_id') one_year = 60 * 60 * 24 * 365 e = time.time() - one_year cherrypy.serving.response.cookie[name]['expires'] = httputil.HTTPDate(e) - - diff --git a/lib/cherrypy/lib/static.py b/lib/cherrypy/lib/static.py index f55dec1d..a630dae6 100644 --- a/lib/cherrypy/lib/static.py +++ b/lib/cherrypy/lib/static.py @@ -1,26 +1,27 @@ +import os +import re +import stat +import mimetypes + try: from io import UnsupportedOperation except ImportError: UnsupportedOperation = object() -import logging -import mimetypes -mimetypes.init() -mimetypes.types_map['.dwg']='image/x-dwg' -mimetypes.types_map['.ico']='image/x-icon' -mimetypes.types_map['.bz2']='application/x-bzip2' -mimetypes.types_map['.gz']='application/x-gzip' - -import os -import re -import stat -import time import cherrypy from cherrypy._cpcompat import ntob, unquote from cherrypy.lib import cptools, httputil, file_generator_limited -def serve_file(path, content_type=None, disposition=None, name=None, debug=False): +mimetypes.init() +mimetypes.types_map['.dwg'] = 'image/x-dwg' +mimetypes.types_map['.ico'] = 'image/x-icon' +mimetypes.types_map['.bz2'] = 'application/x-bzip2' +mimetypes.types_map['.gz'] = 'application/x-gzip' + + +def serve_file(path, content_type=None, disposition=None, name=None, + debug=False): """Set status, headers, and body in order to serve the given path. The Content-Type header will be set to the content_type arg, if provided. @@ -92,6 +93,7 @@ def serve_file(path, content_type=None, disposition=None, name=None, debug=False fileobj = open(path, 'rb') return _serve_fileobj(fileobj, content_type, content_length, debug=debug) + def serve_fileobj(fileobj, content_type=None, disposition=None, name=None, debug=False): """Set status, headers, and body in order to serve the given file object. @@ -145,6 +147,7 @@ def serve_fileobj(fileobj, content_type=None, disposition=None, name=None, return _serve_fileobj(fileobj, content_type, content_length, debug=debug) + def _serve_fileobj(fileobj, content_type, content_length, debug=False): """Internal. Set response.body to the given file object, perhaps ranged.""" response = cherrypy.serving.response @@ -156,7 +159,8 @@ def _serve_fileobj(fileobj, content_type, content_length, debug=False): r = httputil.get_ranges(request.headers.get('Range'), content_length) if r == []: response.headers['Content-Range'] = "bytes */%s" % content_length - message = "Invalid Range (first-byte-pos greater than Content-Length)" + message = ("Invalid Range (first-byte-pos greater than " + "Content-Length)") if debug: cherrypy.log(message, 'TOOLS.STATIC') raise cherrypy.HTTPError(416, message) @@ -169,8 +173,9 @@ def _serve_fileobj(fileobj, content_type, content_length, debug=False): stop = content_length r_len = stop - start if debug: - cherrypy.log('Single part; start: %r, stop: %r' % (start, stop), - 'TOOLS.STATIC') + cherrypy.log( + 'Single part; start: %r, stop: %r' % (start, stop), + 'TOOLS.STATIC') response.status = "206 Partial Content" response.headers['Content-Range'] = ( "bytes %s-%s/%s" % (start, stop - 1, content_length)) @@ -182,11 +187,11 @@ def _serve_fileobj(fileobj, content_type, content_length, debug=False): response.status = "206 Partial Content" try: # Python 3 - from email.generator import _make_boundary as choose_boundary + from email.generator import _make_boundary as make_boundary except ImportError: # Python 2 - from mimetools import choose_boundary - boundary = choose_boundary() + from mimetools import choose_boundary as make_boundary + boundary = make_boundary() ct = "multipart/byteranges; boundary=%s" % boundary response.headers['Content-Type'] = ct if "Content-Length" in response.headers: @@ -199,14 +204,20 @@ def _serve_fileobj(fileobj, content_type, content_length, debug=False): for start, stop in r: if debug: - cherrypy.log('Multipart; start: %r, stop: %r' % (start, stop), - 'TOOLS.STATIC') + cherrypy.log( + 'Multipart; start: %r, stop: %r' % ( + start, stop), + 'TOOLS.STATIC') yield ntob("--" + boundary, 'ascii') - yield ntob("\r\nContent-type: %s" % content_type, 'ascii') - yield ntob("\r\nContent-range: bytes %s-%s/%s\r\n\r\n" - % (start, stop - 1, content_length), 'ascii') + yield ntob("\r\nContent-type: %s" % content_type, + 'ascii') + yield ntob( + "\r\nContent-range: bytes %s-%s/%s\r\n\r\n" % ( + start, stop - 1, content_length), + 'ascii') fileobj.seek(start) - for chunk in file_generator_limited(fileobj, stop-start): + gen = file_generator_limited(fileobj, stop - start) + for chunk in gen: yield chunk yield ntob("\r\n") # Final boundary @@ -226,6 +237,7 @@ def _serve_fileobj(fileobj, content_type, content_length, debug=False): response.body = fileobj return response.body + def serve_download(path, name=None): """Serve 'path' as an application/x-download attachment.""" # This is such a common idiom I felt it deserved its own wrapper. @@ -252,6 +264,7 @@ def _attempt(filename, content_types, debug=False): cherrypy.log('NotFound', 'TOOLS.STATICFILE') return False + def staticdir(section, dir, root="", match="", content_types=None, index="", debug=False): """Serve a static resource from the given (root +) dir. @@ -314,7 +327,7 @@ def staticdir(section, dir, root="", match="", content_types=None, index="", # have ".." or similar uplevel attacks in it. Check that the final # filename is a child of dir. if not os.path.normpath(filename).startswith(os.path.normpath(dir)): - raise cherrypy.HTTPError(403) # Forbidden + raise cherrypy.HTTPError(403) # Forbidden handled = _attempt(filename, content_types) if not handled: @@ -325,6 +338,7 @@ def staticdir(section, dir, root="", match="", content_types=None, index="", request.is_index = filename[-1] in (r"\/") return handled + def staticfile(filename, root=None, match="", content_types=None, debug=False): """Serve a static resource from the given (root +) filename. @@ -354,7 +368,8 @@ def staticfile(filename, root=None, match="", content_types=None, debug=False): # If filename is relative, make absolute using "root". if not os.path.isabs(filename): if not root: - msg = "Static tool requires an absolute filename (got '%s')." % filename + msg = "Static tool requires an absolute filename (got '%s')." % ( + filename,) if debug: cherrypy.log(msg, 'TOOLS.STATICFILE') raise ValueError(msg) diff --git a/lib/cherrypy/lib/xmlrpcutil.py b/lib/cherrypy/lib/xmlrpcutil.py index 9a44464b..9fc9564f 100644 --- a/lib/cherrypy/lib/xmlrpcutil.py +++ b/lib/cherrypy/lib/xmlrpcutil.py @@ -3,6 +3,7 @@ import sys import cherrypy from cherrypy._cpcompat import ntob + def get_xmlrpclib(): try: import xmlrpc.client as x @@ -10,6 +11,7 @@ def get_xmlrpclib(): import xmlrpclib as x return x + def process_body(): """Return (params, method) from request body.""" try: @@ -48,8 +50,8 @@ def respond(body, encoding='utf-8', allow_none=0): encoding=encoding, allow_none=allow_none)) + def on_error(*args, **kwargs): body = str(sys.exc_info()[1]) xmlrpclib = get_xmlrpclib() _set_response(xmlrpclib.dumps(xmlrpclib.Fault(1, body))) - diff --git a/lib/cherrypy/process/plugins.py b/lib/cherrypy/process/plugins.py index 57b665b3..c787ba92 100644 --- a/lib/cherrypy/process/plugins.py +++ b/lib/cherrypy/process/plugins.py @@ -7,7 +7,8 @@ import sys import time import threading -from cherrypy._cpcompat import basestring, get_daemon, get_thread_ident, ntob, set +from cherrypy._cpcompat import basestring, get_daemon, get_thread_ident +from cherrypy._cpcompat import ntob, set, Timer, SetDaemonProperty # _module__file__base is used by Autoreload to make # absolute any filenames retrieved from sys.modules which are not @@ -19,8 +20,8 @@ from cherrypy._cpcompat import basestring, get_daemon, get_thread_ident, ntob, s # changes the current directory by executing os.chdir(), then the next time # Autoreload runs, it will not be able to find any filenames which are # not absolute paths, because the current directory is not the same as when the -# module was first imported. Autoreload will then wrongly conclude the file has -# "changed", and initiate the shutdown/re-exec sequence. +# module was first imported. Autoreload will then wrongly conclude the file +# has "changed", and initiate the shutdown/re-exec sequence. # See ticket #917. # For this workaround to have a decent probability of success, this module # needs to be imported as early as possible, before the app has much chance @@ -29,10 +30,12 @@ _module__file__base = os.getcwd() class SimplePlugin(object): + """Plugin base class which auto-subscribes methods for known channels.""" bus = None - """A :class:`Bus `, usually cherrypy.engine.""" + """A :class:`Bus `, usually cherrypy.engine. + """ def __init__(self, bus): self.bus = bus @@ -54,8 +57,8 @@ class SimplePlugin(object): self.bus.unsubscribe(channel, method) - class SignalHandler(object): + """Register bus channels (and listeners) for system signals. You can modify what signals your application listens for, and what it does @@ -74,9 +77,9 @@ class SignalHandler(object): if the process is attached to a TTY. This is because Unix window managers tend to send SIGHUP to terminal windows when the user closes them. - Feel free to add signals which are not available on every platform. The - :class:`SignalHandler` will ignore errors raised from attempting to register - handlers for unknown signals. + Feel free to add signals which are not available on every platform. + The :class:`SignalHandler` will ignore errors raised from attempting + to register handlers for unknown signals. """ handlers = {} @@ -187,15 +190,17 @@ class SignalHandler(object): try: - import pwd, grp + import pwd + import grp except ImportError: pwd, grp = None, None class DropPrivileges(SimplePlugin): + """Drop privileges. uid/gid arguments not available on Windows. - Special thanks to Gavin Baker: http://antonym.org/node/100. + Special thanks to `Gavin Baker `_ """ def __init__(self, bus, umask=None, uid=None, gid=None): @@ -207,6 +212,7 @@ class DropPrivileges(SimplePlugin): def _get_uid(self): return self._uid + def _set_uid(self, val): if val is not None: if pwd is None: @@ -217,10 +223,11 @@ class DropPrivileges(SimplePlugin): val = pwd.getpwnam(val)[2] self._uid = val uid = property(_get_uid, _set_uid, - doc="The uid under which to run. Availability: Unix.") + doc="The uid under which to run. Availability: Unix.") def _get_gid(self): return self._gid + def _set_gid(self, val): if val is not None: if grp is None: @@ -231,10 +238,11 @@ class DropPrivileges(SimplePlugin): val = grp.getgrnam(val)[2] self._gid = val gid = property(_get_gid, _set_gid, - doc="The gid under which to run. Availability: Unix.") + doc="The gid under which to run. Availability: Unix.") def _get_umask(self): return self._umask + def _set_umask(self, val): if val is not None: try: @@ -244,8 +252,11 @@ class DropPrivileges(SimplePlugin): level=30) val = None self._umask = val - umask = property(_get_umask, _set_umask, - doc="""The default permission mode for newly created files and directories. + umask = property( + _get_umask, + _set_umask, + doc="""The default permission mode for newly created files and + directories. Usually expressed in octal format, for example, ``0644``. Availability: Unix, Windows. @@ -299,6 +310,7 @@ class DropPrivileges(SimplePlugin): class Daemonizer(SimplePlugin): + """Daemonize the running script. Use this with a Web Site Process Bus via:: @@ -368,7 +380,7 @@ class Daemonizer(SimplePlugin): pid = os.fork() if pid > 0: self.bus.log('Forking twice.') - os._exit(0) # Exit second parent + os._exit(0) # Exit second parent except OSError: exc = sys.exc_info()[1] sys.exit("%s: fork #2 failed: (%d) %s\n" @@ -394,6 +406,7 @@ class Daemonizer(SimplePlugin): class PIDFile(SimplePlugin): + """Maintain a PID file via a WSPBus.""" def __init__(self, bus, pidfile): @@ -406,7 +419,7 @@ class PIDFile(SimplePlugin): if self.finalized: self.bus.log('PID %r already written to %r.' % (pid, self.pidfile)) else: - open(self.pidfile, "wb").write(ntob("%s" % pid, 'utf8')) + open(self.pidfile, "wb").write(ntob("%s\n" % pid, 'utf8')) self.bus.log('PID %r written to %r.' % (pid, self.pidfile)) self.finalized = True start.priority = 70 @@ -421,14 +434,20 @@ class PIDFile(SimplePlugin): pass -class PerpetualTimer(threading._Timer): - """A responsive subclass of threading._Timer whose run() method repeats. +class PerpetualTimer(Timer): + + """A responsive subclass of threading.Timer whose run() method repeats. Use this timer only when you really need a very interruptible timer; this checks its 'finished' condition up to 20 times a second, which can results in pretty high CPU usage """ + def __init__(self, *args, **kwargs): + "Override parent constructor to allow 'bus' to be provided." + self.bus = kwargs.pop('bus', None) + super(PerpetualTimer, self).__init__(*args, **kwargs) + def run(self): while True: self.finished.wait(self.interval) @@ -437,13 +456,16 @@ class PerpetualTimer(threading._Timer): try: self.function(*self.args, **self.kwargs) except Exception: - self.bus.log("Error in perpetual timer thread function %r." % - self.function, level=40, traceback=True) + if self.bus: + self.bus.log( + "Error in perpetual timer thread function %r." % + self.function, level=40, traceback=True) # Quit on first error to avoid massive logs. raise -class BackgroundTask(threading.Thread): +class BackgroundTask(SetDaemonProperty, threading.Thread): + """A subclass of threading.Thread whose run() method repeats. Use this class for most repeating tasks. It uses time.sleep() to wait @@ -462,6 +484,9 @@ class BackgroundTask(threading.Thread): self.running = False self.bus = bus + # default to daemonic + self.daemon = True + def cancel(self): self.running = False @@ -480,11 +505,9 @@ class BackgroundTask(threading.Thread): # Quit on first error to avoid massive logs. raise - def _set_daemon(self): - return True - class Monitor(SimplePlugin): + """WSPBus listener to periodically run a callback in its own thread.""" callback = None @@ -494,7 +517,9 @@ class Monitor(SimplePlugin): """The time in seconds between callback runs.""" thread = None - """A :class:`BackgroundTask` thread.""" + """A :class:`BackgroundTask` + thread. + """ def __init__(self, bus, callback, frequency=60, name=None): SimplePlugin.__init__(self, bus) @@ -509,7 +534,7 @@ class Monitor(SimplePlugin): threadname = self.name or self.__class__.__name__ if self.thread is None: self.thread = BackgroundTask(self.frequency, self.callback, - bus = self.bus) + bus=self.bus) self.thread.setName(threadname) self.thread.start() self.bus.log("Started monitor thread %r." % threadname) @@ -520,7 +545,8 @@ class Monitor(SimplePlugin): def stop(self): """Stop our callback's background task thread.""" if self.thread is None: - self.bus.log("No thread running for %s." % self.name or self.__class__.__name__) + self.bus.log("No thread running for %s." % + self.name or self.__class__.__name__) else: if self.thread is not threading.currentThread(): name = self.thread.getName() @@ -538,6 +564,7 @@ class Monitor(SimplePlugin): class Autoreloader(Monitor): + """Monitor which re-executes the process when files change. This :ref:`plugin` restarts the process (via :func:`os.execv`) @@ -547,9 +574,9 @@ class Autoreloader(Monitor): cherrypy.engine.autoreload.files.add(myFile) - If there are imported files you do *not* wish to monitor, you can adjust the - ``match`` attribute, a regular expression. For example, to stop monitoring - cherrypy itself:: + If there are imported files you do *not* wish to monitor, you can + adjust the ``match`` attribute, a regular expression. For example, + to stop monitoring cherrypy itself:: cherrypy.engine.autoreload.match = r'^(?!cherrypy).+' @@ -583,15 +610,20 @@ class Autoreloader(Monitor): def sysfiles(self): """Return a Set of sys.modules filenames to monitor.""" files = set() - for k, m in sys.modules.items(): + for k, m in list(sys.modules.items()): if re.match(self.match, k): - if hasattr(m, '__loader__') and hasattr(m.__loader__, 'archive'): + if ( + hasattr(m, '__loader__') and + hasattr(m.__loader__, 'archive') + ): f = m.__loader__.archive else: f = getattr(m, '__file__', None) if f is not None and not os.path.isabs(f): - # ensure absolute paths so a os.chdir() in the app doesn't break me - f = os.path.normpath(os.path.join(_module__file__base, f)) + # ensure absolute paths so a os.chdir() in the app + # doesn't break me + f = os.path.normpath( + os.path.join(_module__file__base, f)) files.add(f) return files @@ -619,14 +651,17 @@ class Autoreloader(Monitor): else: if mtime is None or mtime > oldtime: # The file has been deleted or modified. - self.bus.log("Restarting because %s changed." % filename) + self.bus.log("Restarting because %s changed." % + filename) self.thread.cancel() - self.bus.log("Stopped thread %r." % self.thread.getName()) + self.bus.log("Stopped thread %r." % + self.thread.getName()) self.bus.restart() return class ThreadManager(SimplePlugin): + """Manager for HTTP request threads. If you have control over thread creation and destruction, publish to @@ -680,4 +715,3 @@ class ThreadManager(SimplePlugin): self.bus.publish('stop_thread', i) self.threads.clear() graceful = stop - diff --git a/lib/cherrypy/process/servers.py b/lib/cherrypy/process/servers.py index e353a11a..fef37f77 100644 --- a/lib/cherrypy/process/servers.py +++ b/lib/cherrypy/process/servers.py @@ -13,7 +13,9 @@ protocols, etc.), you can manually register each one and then start them all with engine.start:: s1 = ServerAdapter(cherrypy.engine, MyWSGIServer(host='0.0.0.0', port=80)) - s2 = ServerAdapter(cherrypy.engine, another.HTTPServer(host='127.0.0.1', SSL=True)) + s2 = ServerAdapter(cherrypy.engine, + another.HTTPServer(host='127.0.0.1', + SSL=True)) s1.subscribe() s2.subscribe() cherrypy.engine.start() @@ -63,7 +65,7 @@ hello.py:: cherrypy.tree.mount(HelloWorld()) # CherryPy autoreload must be disabled for the flup server to work - cherrypy.config.update({'engine.autoreload_on':False}) + cherrypy.config.update({'engine.autoreload.on':False}) Then run :doc:`/deployguide/cherryd` with the '-f' arg:: @@ -107,15 +109,17 @@ directive, configure your fastcgi script like the following:: } # end of $HTTP["url"] =~ "^/" Please see `Lighttpd FastCGI Docs -`_ for an explanation -of the possible configuration options. +`_ for +an explanation of the possible configuration options. """ import sys import time +import warnings class ServerAdapter(object): + """Adapter for an HTTP server. If you need to start more than one HTTP server (to serve on multiple @@ -149,8 +153,7 @@ class ServerAdapter(object): if self.bind_addr is None: on_what = "unknown interface (dynamic?)" elif isinstance(self.bind_addr, tuple): - host, port = self.bind_addr - on_what = "%s:%s" % (host, port) + on_what = self._get_base() else: on_what = "socket file: %s" % self.bind_addr @@ -176,6 +179,21 @@ class ServerAdapter(object): self.bus.log("Serving on %s" % on_what) start.priority = 75 + def _get_base(self): + if not self.httpserver: + return '' + host, port = self.bind_addr + if getattr(self.httpserver, 'ssl_certificate', None): + scheme = "https" + if port != 443: + host += ":%s" % port + else: + scheme = "http" + if port != 80: + host += ":%s" % port + + return "%s://%s" % (scheme, host) + def _start_http_thread(self): """HTTP servers MUST be running in new threads, so that the main thread persists to receive KeyboardInterrupt's. If an @@ -234,6 +252,7 @@ class ServerAdapter(object): class FlupCGIServer(object): + """Adapter for a flup.server.cgi.WSGIServer.""" def __init__(self, *args, **kwargs): @@ -257,6 +276,7 @@ class FlupCGIServer(object): class FlupFCGIServer(object): + """Adapter for a flup.server.fcgi.WSGIServer.""" def __init__(self, *args, **kwargs): @@ -296,11 +316,13 @@ class FlupFCGIServer(object): # Forcibly stop the fcgi server main event loop. self.fcgiserver._keepGoing = False # Force all worker threads to die off. - self.fcgiserver._threadPool.maxSpare = self.fcgiserver._threadPool._idleCount + self.fcgiserver._threadPool.maxSpare = ( + self.fcgiserver._threadPool._idleCount) self.ready = False class FlupSCGIServer(object): + """Adapter for a flup.server.scgi.WSGIServer.""" def __init__(self, *args, **kwargs): @@ -344,10 +366,12 @@ def client_host(server_host): return '127.0.0.1' if server_host in ('::', '::0', '::0.0.0.0'): # :: is IN6ADDR_ANY, which should answer on localhost. - # ::0 and ::0.0.0.0 are non-canonical but common ways to write IN6ADDR_ANY. + # ::0 and ::0.0.0.0 are non-canonical but common + # ways to write IN6ADDR_ANY. return '::1' return server_host + def check_port(host, port, timeout=1.0): """Raise an error if the given port is not free on the given host.""" if not host: @@ -364,7 +388,9 @@ def check_port(host, port, timeout=1.0): socket.SOCK_STREAM) except socket.gaierror: if ':' in host: - info = [(socket.AF_INET6, socket.SOCK_STREAM, 0, "", (host, port, 0, 0))] + info = [( + socket.AF_INET6, socket.SOCK_STREAM, 0, "", (host, port, 0, 0) + )] else: info = [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (host, port))] @@ -378,18 +404,20 @@ def check_port(host, port, timeout=1.0): s.settimeout(timeout) s.connect((host, port)) s.close() - raise IOError("Port %s is in use on %s; perhaps the previous " - "httpserver did not shut down properly." % - (repr(port), repr(host))) except socket.error: if s: s.close() + else: + raise IOError("Port %s is in use on %s; perhaps the previous " + "httpserver did not shut down properly." % + (repr(port), repr(host))) # Feel free to increase these defaults on slow systems: free_port_timeout = 0.1 occupied_port_timeout = 1.0 + def wait_for_free_port(host, port, timeout=None): """Wait for the specified port to become free (drop requests).""" if not host: @@ -409,6 +437,7 @@ def wait_for_free_port(host, port, timeout=None): raise IOError("Port %r not free on %r" % (port, host)) + def wait_for_occupied_port(host, port, timeout=None): """Wait for the specified port to become active (receive requests).""" if not host: @@ -420,8 +449,17 @@ def wait_for_occupied_port(host, port, timeout=None): try: check_port(host, port, timeout=timeout) except IOError: + # port is occupied return else: time.sleep(timeout) - raise IOError("Port %r not bound on %r" % (port, host)) + if host == client_host(host): + raise IOError("Port %r not bound on %r" % (port, host)) + + # On systems where a loopback interface is not available and the + # server is bound to all interfaces, it's difficult to determine + # whether the server is in fact occupying the port. In this case, + # just issue a warning and move on. See issue #1100. + msg = "Unable to verify that the server is bound on %r" % port + warnings.warn(msg) diff --git a/lib/cherrypy/process/win32.py b/lib/cherrypy/process/win32.py index 6f135177..4afd3f14 100644 --- a/lib/cherrypy/process/win32.py +++ b/lib/cherrypy/process/win32.py @@ -11,6 +11,7 @@ from cherrypy.process import wspbus, plugins class ConsoleCtrlHandler(plugins.SimplePlugin): + """A WSPBus plugin for handling Win32 console events (like Ctrl-C).""" def __init__(self, bus): @@ -68,6 +69,7 @@ class ConsoleCtrlHandler(plugins.SimplePlugin): class Win32Bus(wspbus.Bus): + """A Web Site Process Bus implementation for Win32. Instead of time.sleep, this bus blocks using native win32event objects. @@ -90,6 +92,7 @@ class Win32Bus(wspbus.Bus): def _get_state(self): return self._state + def _set_state(self, value): self._state = value event = self._get_state_event(value) @@ -106,7 +109,8 @@ class Win32Bus(wspbus.Bus): # Don't wait for an event that beat us to the punch ;) if self.state not in state: events = tuple([self._get_state_event(s) for s in state]) - win32event.WaitForMultipleObjects(events, 0, win32event.INFINITE) + win32event.WaitForMultipleObjects( + events, 0, win32event.INFINITE) else: # Don't wait for an event that beat us to the punch ;) if self.state != state: @@ -115,6 +119,7 @@ class Win32Bus(wspbus.Bus): class _ControlCodes(dict): + """Control codes used to "signal" a service via ControlService. User-defined control codes are in the range 128-255. We generally use @@ -145,6 +150,7 @@ def signal_child(service, command): class PyWebService(win32serviceutil.ServiceFramework): + """Python Web Service.""" _svc_name_ = "Python Web Service" diff --git a/lib/cherrypy/process/wspbus.py b/lib/cherrypy/process/wspbus.py index 3ef0217c..5409d038 100644 --- a/lib/cherrypy/process/wspbus.py +++ b/lib/cherrypy/process/wspbus.py @@ -78,13 +78,16 @@ from cherrypy._cpcompat import set # sys.executable is a relative-path, and/or cause other problems). _startup_cwd = os.getcwd() + class ChannelFailures(Exception): - """Exception raised when errors occur in a listener during Bus.publish().""" + + """Exception raised when errors occur in a listener during Bus.publish(). + """ delimiter = '\n' def __init__(self, *args, **kwargs): # Don't use 'super' here; Exceptions are old-style in Py2.4 - # See http://www.cherrypy.org/ticket/959 + # See https://bitbucket.org/cherrypy/cherrypy/issue/959 Exception.__init__(self, *args, **kwargs) self._exceptions = list() @@ -107,9 +110,13 @@ class ChannelFailures(Exception): __nonzero__ = __bool__ # Use a flag to indicate the state of the bus. + + class _StateEnum(object): + class State(object): name = None + def __repr__(self): return "states.%s" % self.name @@ -137,6 +144,7 @@ else: class Bus(object): + """Process state-machine and messenger for HTTP site deployment. All listeners for a given channel are guaranteed to be called even @@ -266,14 +274,14 @@ class Bus(object): # signal handler, console handler, or atexit handler), so we # can't just let exceptions propagate out unhandled. # Assume it's been logged and just die. - os._exit(70) # EX_SOFTWARE + os._exit(70) # EX_SOFTWARE if exitstate == states.STARTING: # exit() was called before start() finished, possibly due to # Ctrl-C because a start listener got stuck. In this case, # we could get stuck in a loop where Ctrl-C never exits the # process, so we just call os.exit here. - os._exit(70) # EX_SOFTWARE + os._exit(70) # EX_SOFTWARE def restart(self): """Restart the process (may close connections). @@ -311,13 +319,21 @@ class Bus(object): raise # Waiting for ALL child threads to finish is necessary on OS X. - # See http://www.cherrypy.org/ticket/581. + # See https://bitbucket.org/cherrypy/cherrypy/issue/581. # It's also good to let them all shut down before allowing # the main thread to call atexit handlers. - # See http://www.cherrypy.org/ticket/751. + # See https://bitbucket.org/cherrypy/cherrypy/issue/751. self.log("Waiting for child threads to terminate...") for t in threading.enumerate(): - if t != threading.currentThread() and t.isAlive(): + # Validate the we're not trying to join the MainThread + # that will cause a deadlock and the case exist when + # implemented as a windows service and in any other case + # that another thread executes cherrypy.engine.exit() + if ( + t != threading.currentThread() and + t.isAlive() and + not isinstance(t, threading._MainThread) + ): # Note that any dummy (external) threads are always daemonic. if hasattr(threading.Thread, "daemon"): # Python 2.6+ @@ -389,7 +405,7 @@ class Bus(object): Set self.max_cloexec_files to 0 to disable this behavior. """ - for fd in range(3, self.max_cloexec_files): # skip stdin/out/err + for fd in range(3, self.max_cloexec_files): # skip stdin/out/err try: flags = fcntl.fcntl(fd, fcntl.F_GETFD) except IOError: diff --git a/lib/cherrypy/scaffold/__init__.py b/lib/cherrypy/scaffold/__init__.py index 9a04796d..50de34bb 100644 --- a/lib/cherrypy/scaffold/__init__.py +++ b/lib/cherrypy/scaffold/__init__.py @@ -47,11 +47,11 @@ Or, just look at the pretty picture:
    other.exposed = True files = cherrypy.tools.staticdir.handler( - section="/files", - dir=os.path.join(local_dir, "static"), - # Ignore .php files, etc. + section="/files", + dir=os.path.join(local_dir, "static"), + # Ignore .php files, etc. match=r'\.(css|gif|html?|ico|jpe?g|js|png|swf|xml)$', - ) + ) root = Root() diff --git a/lib/cherrypy/wsgiserver/ssl_builtin.py b/lib/cherrypy/wsgiserver/ssl_builtin.py index 7148dfda..2c74ad84 100644 --- a/lib/cherrypy/wsgiserver/ssl_builtin.py +++ b/lib/cherrypy/wsgiserver/ssl_builtin.py @@ -25,6 +25,7 @@ from cherrypy import wsgiserver class BuiltinSSLAdapter(wsgiserver.SSLAdapter): + """A wrapper for integrating Python's builtin ssl module with CherryPy.""" certificate = None @@ -48,8 +49,9 @@ class BuiltinSSLAdapter(wsgiserver.SSLAdapter): """Wrap and return the given socket, plus WSGI environ entries.""" try: s = ssl.wrap_socket(sock, do_handshake_on_connect=True, - server_side=True, certfile=self.certificate, - keyfile=self.private_key, ssl_version=ssl.PROTOCOL_SSLv23) + server_side=True, certfile=self.certificate, + keyfile=self.private_key, + ssl_version=ssl.PROTOCOL_SSLv23) except ssl.SSLError: e = sys.exc_info()[1] if e.errno == ssl.SSL_ERROR_EOF: @@ -77,9 +79,9 @@ class BuiltinSSLAdapter(wsgiserver.SSLAdapter): "HTTPS": "on", 'SSL_PROTOCOL': cipher[1], 'SSL_CIPHER': cipher[0] -## SSL_VERSION_INTERFACE string The mod_ssl program version -## SSL_VERSION_LIBRARY string The OpenSSL program version - } + # SSL_VERSION_INTERFACE string The mod_ssl program version + # SSL_VERSION_LIBRARY string The OpenSSL program version + } return ssl_environ if sys.version_info >= (3, 0): @@ -88,4 +90,3 @@ class BuiltinSSLAdapter(wsgiserver.SSLAdapter): else: def makefile(self, sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): return wsgiserver.CP_fileobject(sock, mode, bufsize) - diff --git a/lib/cherrypy/wsgiserver/ssl_pyopenssl.py b/lib/cherrypy/wsgiserver/ssl_pyopenssl.py index 42745fbc..f8f2dafe 100644 --- a/lib/cherrypy/wsgiserver/ssl_pyopenssl.py +++ b/lib/cherrypy/wsgiserver/ssl_pyopenssl.py @@ -1,7 +1,7 @@ """A library for integrating pyOpenSSL with CherryPy. The OpenSSL module must be importable for SSL functionality. -You can obtain it from http://pyopenssl.sourceforge.net/ +You can obtain it from `here `_. To use this module, set CherryPyWSGIServer.ssl_adapter to an instance of SSLAdapter. There are two ways to use SSL: @@ -44,6 +44,7 @@ except ImportError: class SSL_fileobject(wsgiserver.CP_fileobject): + """SSL file object attached to a socket object.""" ssl_timeout = 3 @@ -96,15 +97,8 @@ class SSL_fileobject(wsgiserver.CP_fileobject): if time.time() - start > self.ssl_timeout: raise socket.timeout("timed out") - def recv(self, *args, **kwargs): - buf = [] - r = super(SSL_fileobject, self).recv - while True: - data = self._safe_call(True, r, *args, **kwargs) - buf.append(data) - p = self._sock.pending() - if not p: - return "".join(buf) + def recv(self, size): + return self._safe_call(True, super(SSL_fileobject, self).recv, size) def sendall(self, *args, **kwargs): return self._safe_call(False, super(SSL_fileobject, self).sendall, @@ -116,6 +110,7 @@ class SSL_fileobject(wsgiserver.CP_fileobject): class SSLConnection: + """A thread-safe wrapper for an SSL.Connection. ``*args``: the arguments to create the wrapped ``SSL.Connection(*args)``. @@ -151,6 +146,7 @@ class SSLConnection: class pyOpenSSLAdapter(wsgiserver.SSLAdapter): + """A wrapper for integrating pyOpenSSL with CherryPy.""" context = None @@ -205,11 +201,11 @@ class pyOpenSSLAdapter(wsgiserver.SSLAdapter): ssl_environ = { "HTTPS": "on", # pyOpenSSL doesn't provide access to any of these AFAICT -## 'SSL_PROTOCOL': 'SSLv2', -## SSL_CIPHER string The cipher specification name -## SSL_VERSION_INTERFACE string The mod_ssl program version -## SSL_VERSION_LIBRARY string The OpenSSL program version - } + # 'SSL_PROTOCOL': 'SSLv2', + # SSL_CIPHER string The cipher specification name + # SSL_VERSION_INTERFACE string The mod_ssl program version + # SSL_VERSION_LIBRARY string The OpenSSL program version + } if self.certificate: # Server certificate attributes @@ -218,9 +214,11 @@ class pyOpenSSLAdapter(wsgiserver.SSLAdapter): ssl_environ.update({ 'SSL_SERVER_M_VERSION': cert.get_version(), 'SSL_SERVER_M_SERIAL': cert.get_serial_number(), -## 'SSL_SERVER_V_START': Validity of server's certificate (start time), -## 'SSL_SERVER_V_END': Validity of server's certificate (end time), - }) + # 'SSL_SERVER_V_START': + # Validity of server's certificate (start time), + # 'SSL_SERVER_V_END': + # Validity of server's certificate (end time), + }) for prefix, dn in [("I", cert.get_issuer()), ("S", cert.get_subject())]: @@ -253,4 +251,3 @@ class pyOpenSSLAdapter(wsgiserver.SSLAdapter): return f else: return wsgiserver.CP_fileobject(sock, mode, bufsize) - diff --git a/lib/cherrypy/wsgiserver/wsgiserver2.py b/lib/cherrypy/wsgiserver/wsgiserver2.py index 2422fadb..c7f08350 100644 --- a/lib/cherrypy/wsgiserver/wsgiserver2.py +++ b/lib/cherrypy/wsgiserver/wsgiserver2.py @@ -86,19 +86,34 @@ import re import rfc822 import socket import sys -if 'win' in sys.platform and not hasattr(socket, 'IPPROTO_IPV6'): - socket.IPPROTO_IPV6 = 41 +if 'win' in sys.platform and hasattr(socket, "AF_INET6"): + if not hasattr(socket, 'IPPROTO_IPV6'): + socket.IPPROTO_IPV6 = 41 + if not hasattr(socket, 'IPV6_V6ONLY'): + socket.IPV6_V6ONLY = 27 try: import cStringIO as StringIO except ImportError: import StringIO DEFAULT_BUFFER_SIZE = -1 -_fileobject_uses_str_type = isinstance(socket._fileobject(None)._rbuf, basestring) + +class FauxSocket(object): + + """Faux socket with the minimal interface required by pypy""" + + def _reuse(self): + pass + +_fileobject_uses_str_type = isinstance( + socket._fileobject(FauxSocket())._rbuf, basestring) +del FauxSocket # this class is not longer required for anything. import threading import time import traceback + + def format_exc(limit=None): """Like print_exc() but return a string. Backport for Python 2.3.""" try: @@ -107,25 +122,31 @@ def format_exc(limit=None): finally: etype = value = tb = None +import operator from urllib import unquote -from urlparse import urlparse import warnings if sys.version_info >= (3, 0): bytestr = bytes unicodestr = str basestring = (bytes, str) + def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given encoding.""" + """Return the given native string as a byte string in the given + encoding. + """ # In Python 3, the native string type is unicode return n.encode(encoding) else: bytestr = str unicodestr = unicode basestring = basestring + def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given encoding.""" + """Return the given native string as a byte string in the given + encoding. + """ # In Python 2, the native string type is bytes. Assume it's already # in the given encoding, which for ISO-8859-1 is almost always what # was intended. @@ -146,6 +167,7 @@ quoted_slash = re.compile(ntob("(?i)%2F")) import errno + def plat_specific_errors(*errnames): """Return error numbers for all errors in errnames on this platform. @@ -170,24 +192,27 @@ socket_errors_to_ignore = plat_specific_errors( "ECONNABORTED", "WSAECONNABORTED", "ENETRESET", "WSAENETRESET", "EHOSTDOWN", "EHOSTUNREACH", - ) +) socket_errors_to_ignore.append("timed out") socket_errors_to_ignore.append("The read operation timed out") socket_errors_nonblocking = plat_specific_errors( 'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK') -comma_separated_headers = [ntob(h) for h in +comma_separated_headers = [ + ntob(h) for h in ['Accept', 'Accept-Charset', 'Accept-Encoding', 'Accept-Language', 'Accept-Ranges', 'Allow', 'Cache-Control', 'Connection', 'Content-Encoding', 'Content-Language', 'Expect', 'If-Match', 'If-None-Match', 'Pragma', 'Proxy-Authenticate', 'TE', 'Trailer', 'Transfer-Encoding', 'Upgrade', 'Vary', 'Via', 'Warning', - 'WWW-Authenticate']] + 'WWW-Authenticate'] +] import logging -if not hasattr(logging, 'statistics'): logging.statistics = {} +if not hasattr(logging, 'statistics'): + logging.statistics = {} def read_headers(rfile, hdict=None): @@ -242,7 +267,9 @@ def read_headers(rfile, hdict=None): class MaxSizeExceeded(Exception): pass + class SizeCheckWrapper(object): + """Wraps a file-like object, raising MaxSizeExceeded if too large.""" def __init__(self, rfile, maxlen): @@ -275,8 +302,8 @@ class SizeCheckWrapper(object): self.bytes_read += len(data) self._check_length() res.append(data) - # See http://www.cherrypy.org/ticket/421 - if len(data) < 256 or data[-1:] == "\n": + # See https://bitbucket.org/cherrypy/cherrypy/issue/421 + if len(data) < 256 or data[-1:] == LF: return EMPTY.join(res) def readlines(self, sizehint=0): @@ -312,6 +339,7 @@ class SizeCheckWrapper(object): class KnownLengthRFile(object): + """Wraps a file-like object, returning an empty string when exhausted.""" def __init__(self, rfile, content_length): @@ -368,6 +396,7 @@ class KnownLengthRFile(object): class ChunkedRFile(object): + """Wraps a file-like object, returning an empty string when exhausted. This class is intended to provide a conforming wsgi.input value for @@ -417,8 +446,8 @@ class ChunkedRFile(object): crlf = self.rfile.read(2) if crlf != CRLF: raise ValueError( - "Bad chunked transfer coding (expected '\\r\\n', " - "got " + repr(crlf) + ")") + "Bad chunked transfer coding (expected '\\r\\n', " + "got " + repr(crlf) + ")") def read(self, size=None): data = EMPTY @@ -520,6 +549,7 @@ class ChunkedRFile(object): class HTTPRequest(object): + """An HTTP Request (and response). A single HTTP connection may consist of multiple request/response pairs. @@ -553,7 +583,7 @@ class HTTPRequest(object): This value is set automatically inside send_headers.""" def __init__(self, server, conn): - self.server= server + self.server = server self.conn = conn self.ready = False @@ -579,7 +609,8 @@ class HTTPRequest(object): try: success = self.read_request_line() except MaxSizeExceeded: - self.simple_response("414 Request-URI Too Long", + self.simple_response( + "414 Request-URI Too Long", "The Request-URI sent with the request exceeds the maximum " "allowed bytes.") return @@ -590,7 +621,8 @@ class HTTPRequest(object): try: success = self.read_request_headers() except MaxSizeExceeded: - self.simple_response("413 Request Entity Too Large", + self.simple_response( + "413 Request Entity Too Large", "The headers sent with the request exceed the maximum " "allowed bytes.") return @@ -626,7 +658,8 @@ class HTTPRequest(object): return False if not request_line.endswith(CRLF): - self.simple_response("400 Bad Request", "HTTP requires CRLF terminators") + self.simple_response( + "400 Bad Request", "HTTP requires CRLF terminators") return False try: @@ -709,7 +742,8 @@ class HTTPRequest(object): mrbs = self.server.max_request_body_size if mrbs and int(self.inheaders.get("Content-Length", 0)) > mrbs: - self.simple_response("413 Request Entity Too Large", + self.simple_response( + "413 Request Entity Too Large", "The entity sent with the request exceeds the maximum " "allowed bytes.") return False @@ -763,7 +797,8 @@ class HTTPRequest(object): # but it seems like it would be a big slowdown for such a rare case. if self.inheaders.get("Expect", "") == "100-continue": # Don't use simple_response here, because it emits headers - # we don't want. See http://www.cherrypy.org/ticket/951 + # we don't want. See + # https://bitbucket.org/cherrypy/cherrypy/issue/951 msg = self.server.protocol + " 100 Continue\r\n\r\n" try: self.conn.wfile.sendall(msg) @@ -800,7 +835,8 @@ class HTTPRequest(object): if i > 0 and QUESTION_MARK not in uri[:i]: # An absoluteURI. # If there's a scheme (and it must be http or https), then: - # http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query ]] + # http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query + # ]] scheme, remainder = uri[:i].lower(), uri[i + 3:] authority, path = remainder.split(FORWARD_SLASH, 1) path = FORWARD_SLASH + path @@ -822,7 +858,8 @@ class HTTPRequest(object): cl = int(self.inheaders.get("Content-Length", 0)) if mrbs and mrbs < cl: if not self.sent_headers: - self.simple_response("413 Request Entity Too Large", + self.simple_response( + "413 Request Entity Too Large", "The entity sent with the request exceeds the maximum " "allowed bytes.") return @@ -897,7 +934,7 @@ class HTTPRequest(object): pass else: if (self.response_protocol == 'HTTP/1.1' - and self.method != 'HEAD'): + and self.method != 'HEAD'): # Use the chunked transfer-coding self.chunked_write = True self.outheaders.append(("Transfer-Encoding", "chunked")) @@ -946,16 +983,19 @@ class HTTPRequest(object): class NoSSLError(Exception): + """Exception raised when a client speaks HTTP to an HTTPS socket.""" pass class FatalSSLAlert(Exception): + """Exception raised when the SSL implementation signals a fatal alert.""" pass class CP_fileobject(socket._fileobject): + """Faux file object attached to a socket object.""" def __init__(self, *args, **kwargs): @@ -992,23 +1032,26 @@ class CP_fileobject(socket._fileobject): return data except socket.error, e: if (e.args[0] not in socket_errors_nonblocking - and e.args[0] not in socket_error_eintr): + and e.args[0] not in socket_error_eintr): raise if not _fileobject_uses_str_type: def read(self, size=-1): - # Use max, disallow tiny reads in a loop as they are very inefficient. - # We never leave read() with any leftover data from a new recv() call - # in our internal buffer. + # Use max, disallow tiny reads in a loop as they are very + # inefficient. + # We never leave read() with any leftover data from a new recv() + # call in our internal buffer. rbufsize = max(self._rbufsize, self.default_bufsize) - # Our use of StringIO rather than lists of string objects returned by - # recv() minimizes memory usage and fragmentation that occurs when - # rbufsize is large compared to the typical return value of recv(). + # Our use of StringIO rather than lists of string objects returned + # by recv() minimizes memory usage and fragmentation that occurs + # when rbufsize is large compared to the typical return value of + # recv(). buf = self._rbuf buf.seek(0, 2) # seek end if size < 0: # Read until EOF - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. + # reset _rbuf. we consume it via buf. + self._rbuf = StringIO.StringIO() while True: data = self.recv(rbufsize) if not data: @@ -1019,14 +1062,16 @@ class CP_fileobject(socket._fileobject): # Read until size bytes or EOF seen, whichever comes first buf_len = buf.tell() if buf_len >= size: - # Already have size bytes in our buffer? Extract and return. + # Already have size bytes in our buffer? Extract and + # return. buf.seek(0) rv = buf.read(size) self._rbuf = StringIO.StringIO() self._rbuf.write(buf.read()) return rv - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. + # reset _rbuf. we consume it via buf. + self._rbuf = StringIO.StringIO() while True: left = size - buf_len # recv() will malloc the amount of memory given as its @@ -1074,7 +1119,8 @@ class CP_fileobject(socket._fileobject): # Speed up unbuffered case buf.seek(0) buffers = [buf.read()] - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. + # reset _rbuf. we consume it via buf. + self._rbuf = StringIO.StringIO() data = None recv = self.recv while data != "\n": @@ -1085,7 +1131,8 @@ class CP_fileobject(socket._fileobject): return "".join(buffers) buf.seek(0, 2) # seek end - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. + # reset _rbuf. we consume it via buf. + self._rbuf = StringIO.StringIO() while True: data = self.recv(self._rbufsize) if not data: @@ -1100,7 +1147,8 @@ class CP_fileobject(socket._fileobject): buf.write(data) return buf.getvalue() else: - # Read until size bytes or \n or EOF seen, whichever comes first + # Read until size bytes or \n or EOF seen, whichever comes + # first buf.seek(0, 2) # seek end buf_len = buf.tell() if buf_len >= size: @@ -1109,7 +1157,8 @@ class CP_fileobject(socket._fileobject): self._rbuf = StringIO.StringIO() self._rbuf.write(buf.read()) return rv - self._rbuf = StringIO.StringIO() # reset _rbuf. we consume it via buf. + # reset _rbuf. we consume it via buf. + self._rbuf = StringIO.StringIO() while True: data = self.recv(self._rbufsize) if not data: @@ -1125,8 +1174,8 @@ class CP_fileobject(socket._fileobject): buf.write(data[:nl]) break else: - # Shortcut. Avoid data copy through buf when returning - # a substring of our first recv(). + # Shortcut. Avoid data copy through buf when + # returning a substring of our first recv(). return data[:nl] n = len(data) if n == size and not buf_len: @@ -1220,7 +1269,8 @@ class CP_fileobject(socket._fileobject): break return "".join(buffers) else: - # Read until size bytes or \n or EOF seen, whichever comes first + # Read until size bytes or \n or EOF seen, whichever comes + # first nl = data.find('\n', 0, size) if nl >= 0: nl += 1 @@ -1256,6 +1306,7 @@ class CP_fileobject(socket._fileobject): class HTTPConnection(object): + """An HTTP connection (active socket). server: the Server object which received this connection. @@ -1273,8 +1324,8 @@ class HTTPConnection(object): def __init__(self, server, sock, makefile=CP_fileobject): self.server = server self.socket = sock - self.rfile = makefile(sock, "rb", self.rbufsize) - self.wfile = makefile(sock, "wb", self.wbufsize) + self.rfile = makefile(sock._sock, "rb", self.rbufsize) + self.wfile = makefile(sock._sock, "wb", self.wbufsize) self.requests_seen = 0 def communicate(self): @@ -1306,11 +1357,14 @@ class HTTPConnection(object): e = sys.exc_info()[1] errnum = e.args[0] # sadly SSL sockets return a different (longer) time out string - if errnum == 'timed out' or errnum == 'The read operation timed out': + if ( + errnum == 'timed out' or + errnum == 'The read operation timed out' + ): # Don't error if we're between requests; only error # if 1) no request has been started at all, or 2) we're # in the middle of a request. - # See http://www.cherrypy.org/ticket/853 + # See https://bitbucket.org/cherrypy/cherrypy/issue/853 if (not request_seen) or (req and req.started_request): # Don't bother writing the 408 if the response # has already started being written. @@ -1338,8 +1392,10 @@ class HTTPConnection(object): except NoSSLError: if req and not req.sent_headers: # Unwrap our wfile - self.wfile = CP_fileobject(self.socket._sock, "wb", self.wbufsize) - req.simple_response("400 Bad Request", + self.wfile = CP_fileobject( + self.socket._sock, "wb", self.wbufsize) + req.simple_response( + "400 Bad Request", "The client sent a plain HTTP request, but " "this server only speaks HTTPS on this port.") self.linger = True @@ -1360,11 +1416,12 @@ class HTTPConnection(object): self.rfile.close() if not self.linger: - # Python's socket module does NOT call close on the kernel socket - # when you call socket.close(). We do so manually here because we - # want this server to send a FIN TCP segment immediately. Note this - # must be called *before* calling socket.close(), because the latter - # drops its reference to the kernel socket. + # Python's socket module does NOT call close on the kernel + # socket when you call socket.close(). We do so manually here + # because we want this server to send a FIN TCP segment + # immediately. Note this must be called *before* calling + # socket.close(), because the latter drops its reference to + # the kernel socket. if hasattr(self.socket, '_sock'): self.socket._sock.close() self.socket.close() @@ -1379,9 +1436,13 @@ class HTTPConnection(object): class TrueyZero(object): - """An object which equals and does math like the integer '0' but evals True.""" + + """An object which equals and does math like the integer 0 but evals True. + """ + def __add__(self, other): return other + def __radd__(self, other): return other trueyzero = TrueyZero() @@ -1389,7 +1450,9 @@ trueyzero = TrueyZero() _SHUTDOWNREQUEST = None + class WorkerThread(threading.Thread): + """Thread which continuously polls a Queue for Connection objects. Due to the timing issues of polling a Queue, a WorkerThread does not @@ -1409,7 +1472,6 @@ class WorkerThread(threading.Thread): """A simple flag for the calling server to know when this thread has begun polling the Queue.""" - def __init__(self, server): self.ready = False self.server = server @@ -1420,12 +1482,30 @@ class WorkerThread(threading.Thread): self.start_time = None self.work_time = 0 self.stats = { - 'Requests': lambda s: self.requests_seen + ((self.start_time is None) and trueyzero or self.conn.requests_seen), - 'Bytes Read': lambda s: self.bytes_read + ((self.start_time is None) and trueyzero or self.conn.rfile.bytes_read), - 'Bytes Written': lambda s: self.bytes_written + ((self.start_time is None) and trueyzero or self.conn.wfile.bytes_written), - 'Work Time': lambda s: self.work_time + ((self.start_time is None) and trueyzero or time.time() - self.start_time), - 'Read Throughput': lambda s: s['Bytes Read'](s) / (s['Work Time'](s) or 1e-6), - 'Write Throughput': lambda s: s['Bytes Written'](s) / (s['Work Time'](s) or 1e-6), + 'Requests': lambda s: self.requests_seen + ( + (self.start_time is None) and + trueyzero or + self.conn.requests_seen + ), + 'Bytes Read': lambda s: self.bytes_read + ( + (self.start_time is None) and + trueyzero or + self.conn.rfile.bytes_read + ), + 'Bytes Written': lambda s: self.bytes_written + ( + (self.start_time is None) and + trueyzero or + self.conn.wfile.bytes_written + ), + 'Work Time': lambda s: self.work_time + ( + (self.start_time is None) and + trueyzero or + time.time() - self.start_time + ), + 'Read Throughput': lambda s: s['Bytes Read'](s) / ( + s['Work Time'](s) or 1e-6), + 'Write Throughput': lambda s: s['Bytes Written'](s) / ( + s['Work Time'](s) or 1e-6), } threading.Thread.__init__(self) @@ -1458,18 +1538,21 @@ class WorkerThread(threading.Thread): class ThreadPool(object): + """A Request Queue for an HTTPServer which pools threads. ThreadPool objects must provide min, get(), put(obj), start() and stop(timeout) attributes. """ - def __init__(self, server, min=10, max=-1): + def __init__(self, server, min=10, max=-1, + accepted_queue_size=-1, accepted_queue_timeout=10): self.server = server self.min = min self.max = max self._threads = [] - self._queue = queue.Queue() + self._queue = queue.Queue(maxsize=accepted_queue_size) + self._queue_put_timeout = accepted_queue_timeout self.get = self._queue.get def start(self): @@ -1489,19 +1572,35 @@ class ThreadPool(object): idle = property(_get_idle, doc=_get_idle.__doc__) def put(self, obj): - self._queue.put(obj) + self._queue.put(obj, block=True, timeout=self._queue_put_timeout) if obj is _SHUTDOWNREQUEST: return def grow(self, amount): """Spawn new worker threads (not above self.max).""" - for i in range(amount): - if self.max > 0 and len(self._threads) >= self.max: - break - worker = WorkerThread(self.server) - worker.setName("CP Server " + worker.getName()) - self._threads.append(worker) - worker.start() + if self.max > 0: + budget = max(self.max - len(self._threads), 0) + else: + # self.max <= 0 indicates no maximum + budget = float('inf') + + n_new = min(amount, budget) + + workers = [self._spawn_worker() for i in range(n_new)] + while not self._all(operator.attrgetter('ready'), workers): + time.sleep(.1) + self._threads.extend(workers) + + def _spawn_worker(self): + worker = WorkerThread(self.server) + worker.setName("CP Server " + worker.getName()) + worker.start() + return worker + + def _all(func, items): + results = [func(item) for item in items] + return reduce(operator.and_, results, True) + _all = staticmethod(_all) def shrink(self, amount): """Kill off worker threads (not below self.min).""" @@ -1512,13 +1611,17 @@ class ThreadPool(object): self._threads.remove(t) amount -= 1 - if amount > 0: - for i in range(min(amount, len(self._threads) - self.min)): - # Put a number of shutdown requests on the queue equal - # to 'amount'. Once each of those is processed by a worker, - # that worker will terminate and be culled from our list - # in self.put. - self._queue.put(_SHUTDOWNREQUEST) + # calculate the number of threads above the minimum + n_extra = max(len(self._threads) - self.min, 0) + + # don't remove more than amount + n_to_remove = min(amount, n_extra) + + # put shutdown requests on the queue equal to the number of threads + # to remove. As each request is processed by a worker, that worker + # will terminate and be culled from the list. + for n in range(n_to_remove): + self._queue.put(_SHUTDOWNREQUEST) def stop(self, timeout=5): # Must shut down threads here so the code that calls @@ -1553,7 +1656,8 @@ class ThreadPool(object): worker.join() except (AssertionError, # Ignore repeated Ctrl-C. - # See http://www.cherrypy.org/ticket/691. + # See + # https://bitbucket.org/cherrypy/cherrypy/issue/691. KeyboardInterrupt): pass @@ -1562,12 +1666,19 @@ class ThreadPool(object): qsize = property(_get_qsize) - try: import fcntl except ImportError: try: from ctypes import windll, WinError + import ctypes.wintypes + _SetHandleInformation = windll.kernel32.SetHandleInformation + _SetHandleInformation.argtypes = [ + ctypes.wintypes.HANDLE, + ctypes.wintypes.DWORD, + ctypes.wintypes.DWORD, + ] + _SetHandleInformation.restype = ctypes.wintypes.BOOL except ImportError: def prevent_socket_inheritance(sock): """Dummy function, since neither fcntl nor ctypes are available.""" @@ -1575,7 +1686,7 @@ except ImportError: else: def prevent_socket_inheritance(sock): """Mark the given socket fd as non-inheritable (Windows).""" - if not windll.kernel32.SetHandleInformation(sock.fileno(), 1, 0): + if not _SetHandleInformation(sock.fileno(), 1, 0): raise WinError() else: def prevent_socket_inheritance(sock): @@ -1586,12 +1697,14 @@ else: class SSLAdapter(object): + """Base class for SSL driver library adapters. Required methods: * ``wrap(sock) -> (wrapped socket, ssl environ dict)`` - * ``makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE) -> socket file object`` + * ``makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE) -> + socket file object`` """ def __init__(self, certificate, private_key, certificate_chain=None): @@ -1607,6 +1720,7 @@ class SSLAdapter(object): class HTTPServer(object): + """An HTTP server.""" _bind_addr = "127.0.0.1" @@ -1619,7 +1733,8 @@ class HTTPServer(object): """The minimum number of worker threads to create (default 10).""" maxthreads = None - """The maximum number of worker threads to create (default -1 = no limit).""" + """The maximum number of worker threads to create (default -1 = no limit). + """ server_name = None """The name of the server; defaults to socket.gethostname().""" @@ -1631,15 +1746,18 @@ class HTTPServer(object): features used in the response.""" request_queue_size = 5 - """The 'backlog' arg to socket.listen(); max queued connections (default 5).""" + """The 'backlog' arg to socket.listen(); max queued connections + (default 5). + """ shutdown_timeout = 5 - """The total time, in seconds, to wait for worker threads to cleanly exit.""" + """The total time, in seconds, to wait for worker threads to cleanly exit. + """ timeout = 10 """The timeout in seconds for accepted connections (default 10).""" - version = "CherryPy/3.2.2" + version = "CherryPy/3.6.0" """A version string for the HTTPServer.""" software = None @@ -1648,7 +1766,8 @@ class HTTPServer(object): If None, this defaults to ``'%s Server' % self.version``.""" ready = False - """An internal flag which marks whether the socket is accepting connections.""" + """An internal flag which marks whether the socket is accepting connections + """ max_request_header_size = 0 """The maximum size, in bytes, for request headers, or 0 for no limit.""" @@ -1692,14 +1811,15 @@ class HTTPServer(object): 'Threads': lambda s: len(getattr(self.requests, "_threads", [])), 'Threads Idle': lambda s: getattr(self.requests, "idle", None), 'Socket Errors': 0, - 'Requests': lambda s: (not s['Enabled']) and -1 or sum([w['Requests'](w) for w - in s['Worker Threads'].values()], 0), - 'Bytes Read': lambda s: (not s['Enabled']) and -1 or sum([w['Bytes Read'](w) for w - in s['Worker Threads'].values()], 0), - 'Bytes Written': lambda s: (not s['Enabled']) and -1 or sum([w['Bytes Written'](w) for w - in s['Worker Threads'].values()], 0), - 'Work Time': lambda s: (not s['Enabled']) and -1 or sum([w['Work Time'](w) for w - in s['Worker Threads'].values()], 0), + 'Requests': lambda s: (not s['Enabled']) and -1 or sum( + [w['Requests'](w) for w in s['Worker Threads'].values()], 0), + 'Bytes Read': lambda s: (not s['Enabled']) and -1 or sum( + [w['Bytes Read'](w) for w in s['Worker Threads'].values()], 0), + 'Bytes Written': lambda s: (not s['Enabled']) and -1 or sum( + [w['Bytes Written'](w) for w in s['Worker Threads'].values()], + 0), + 'Work Time': lambda s: (not s['Enabled']) and -1 or sum( + [w['Work Time'](w) for w in s['Worker Threads'].values()], 0), 'Read Throughput': lambda s: (not s['Enabled']) and -1 or sum( [w['Bytes Read'](w) / (w['Work Time'](w) or 1e-6) for w in s['Worker Threads'].values()], 0), @@ -1707,7 +1827,7 @@ class HTTPServer(object): [w['Bytes Written'](w) / (w['Work Time'](w) or 1e-6) for w in s['Worker Threads'].values()], 0), 'Worker Threads': {}, - } + } logging.statistics["CherryPy HTTPServer %d" % id(self)] = self.stats def runtime(self): @@ -1722,6 +1842,7 @@ class HTTPServer(object): def _get_bind_addr(self): return self._bind_addr + def _set_bind_addr(self, value): if isinstance(value, tuple) and value[0] in ('', None): # Despite the socket module docs, using '' does not @@ -1738,7 +1859,9 @@ class HTTPServer(object): "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead " "to listen on all active interfaces.") self._bind_addr = value - bind_addr = property(_get_bind_addr, _set_bind_addr, + bind_addr = property( + _get_bind_addr, + _set_bind_addr, doc="""The interface on which to listen for connections. For TCP sockets, a (host, port) tuple. Host values may be any IPv4 @@ -1763,14 +1886,14 @@ class HTTPServer(object): # SSL backward compatibility if (self.ssl_adapter is None and - getattr(self, 'ssl_certificate', None) and - getattr(self, 'ssl_private_key', None)): + getattr(self, 'ssl_certificate', None) and + getattr(self, 'ssl_private_key', None)): warnings.warn( - "SSL attributes are deprecated in CherryPy 3.2, and will " - "be removed in CherryPy 3.3. Use an ssl_adapter attribute " - "instead.", - DeprecationWarning - ) + "SSL attributes are deprecated in CherryPy 3.2, and will " + "be removed in CherryPy 3.3. Use an ssl_adapter attribute " + "instead.", + DeprecationWarning + ) try: from cherrypy.wsgiserver.ssl_pyopenssl import pyOpenSSLAdapter except ImportError: @@ -1785,21 +1908,28 @@ class HTTPServer(object): # AF_UNIX socket # So we can reuse the socket... - try: os.unlink(self.bind_addr) - except: pass + try: + os.unlink(self.bind_addr) + except: + pass # So everyone can access the socket... - try: os.chmod(self.bind_addr, 511) # 0777 - except: pass + try: + os.chmod(self.bind_addr, 511) # 0777 + except: + pass - info = [(socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)] + info = [ + (socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)] else: # AF_INET or AF_INET6 socket - # Get the correct address family for our host (allows IPv6 addresses) + # Get the correct address family for our host (allows IPv6 + # addresses) host, port = self.bind_addr try: - info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM, 0, socket.AI_PASSIVE) + info = socket.getaddrinfo( + host, port, socket.AF_UNSPEC, + socket.SOCK_STREAM, 0, socket.AI_PASSIVE) except socket.gaierror: if ':' in self.bind_addr[0]: info = [(socket.AF_INET6, socket.SOCK_STREAM, @@ -1814,7 +1944,8 @@ class HTTPServer(object): af, socktype, proto, canonname, sa = res try: self.bind(af, socktype, proto) - except socket.error: + except socket.error, serr: + msg = "%s -- (%s: %s)" % (msg, sa, serr) if self.socket: self.socket.close() self.socket = None @@ -1869,11 +2000,13 @@ class HTTPServer(object): self.socket = self.ssl_adapter.bind(self.socket) # If listening on the IPV6 any address ('::' = IN6ADDR_ANY), - # activate dual-stack. See http://www.cherrypy.org/ticket/871. + # activate dual-stack. See + # https://bitbucket.org/cherrypy/cherrypy/issue/871. if (hasattr(socket, 'AF_INET6') and family == socket.AF_INET6 - and self.bind_addr[0] in ('::', '::0', '::0.0.0.0')): + and self.bind_addr[0] in ('::', '::0', '::0.0.0.0')): try: - self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + self.socket.setsockopt( + socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) except (AttributeError, socket.error): # Apparently, the socket option is not available in # this machine's TCP stack @@ -1908,7 +2041,7 @@ class HTTPServer(object): "Content-Type: text/plain\r\n\r\n", msg] - wfile = makefile(s, "wb", DEFAULT_BUFFER_SIZE) + wfile = makefile(s._sock, "wb", DEFAULT_BUFFER_SIZE) try: wfile.sendall("".join(buf)) except socket.error: @@ -1928,7 +2061,7 @@ class HTTPServer(object): if not isinstance(self.bind_addr, basestring): # optional values # Until we do DNS lookups, omit REMOTE_HOST - if addr is None: # sometimes this can happen + if addr is None: # sometimes this can happen # figure out if AF_INET or AF_INET6. if len(s.getsockname()) == 2: # AF_INET @@ -1941,7 +2074,12 @@ class HTTPServer(object): conn.ssl_env = ssl_env - self.requests.put(conn) + try: + self.requests.put(conn) + except queue.Full: + # Just drop the conn. TODO: write 503 back? + conn.close() + return except socket.timeout: # The only reason for the timeout in start() is so we can # notice keyboard interrupts on Win32, which don't interrupt @@ -1956,19 +2094,22 @@ class HTTPServer(object): # is received during the accept() call; all docs say retry # the call, and I *think* I'm reading it right that Python # will then go ahead and poll for and handle the signal - # elsewhere. See http://www.cherrypy.org/ticket/707. + # elsewhere. See + # https://bitbucket.org/cherrypy/cherrypy/issue/707. return if x.args[0] in socket_errors_nonblocking: - # Just try again. See http://www.cherrypy.org/ticket/479. + # Just try again. See + # https://bitbucket.org/cherrypy/cherrypy/issue/479. return if x.args[0] in socket_errors_to_ignore: # Our socket was closed. - # See http://www.cherrypy.org/ticket/686. + # See https://bitbucket.org/cherrypy/cherrypy/issue/686. return raise def _get_interrupt(self): return self._interrupt + def _set_interrupt(self, interrupt): self._interrupt = True self.stop() @@ -1994,7 +2135,8 @@ class HTTPServer(object): x = sys.exc_info()[1] if x.args[0] not in socket_errors_to_ignore: # Changed to use error code and not message - # See http://www.cherrypy.org/ticket/860. + # See + # https://bitbucket.org/cherrypy/cherrypy/issue/860. raise else: # Note that we're explicitly NOT using AI_PASSIVE, @@ -2007,8 +2149,9 @@ class HTTPServer(object): s = None try: s = socket.socket(af, socktype, proto) - # See http://groups.google.com/group/cherrypy-users/ - # browse_frm/thread/bbfe5eb39c904fe0 + # See + # http://groups.google.com/group/cherrypy-users/ + # browse_frm/thread/bbfe5eb39c904fe0 s.settimeout(1.0) s.connect((host, port)) s.close() @@ -2023,7 +2166,9 @@ class HTTPServer(object): class Gateway(object): - """A base class to interface HTTPServer with other systems, such as WSGI.""" + + """A base class to interface HTTPServer with other systems, such as WSGI. + """ def __init__(self, req): self.req = req @@ -2038,7 +2183,8 @@ class Gateway(object): ssl_adapters = { 'builtin': 'cherrypy.wsgiserver.ssl_builtin.BuiltinSSLAdapter', 'pyopenssl': 'cherrypy.wsgiserver.ssl_pyopenssl.pyOpenSSLAdapter', - } +} + def get_ssl_adapter_class(name='pyopenssl'): """Return an SSL adapter class for the given name.""" @@ -2065,18 +2211,22 @@ def get_ssl_adapter_class(name='pyopenssl'): return adapter -# -------------------------------- WSGI Stuff -------------------------------- # +# ------------------------------- WSGI Stuff -------------------------------- # class CherryPyWSGIServer(HTTPServer): + """A subclass of HTTPServer which calls a WSGI application.""" wsgi_version = (1, 0) """The version of WSGI to produce.""" def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, - max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5): - self.requests = ThreadPool(self, min=numthreads or 1, max=max) + max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5, + accepted_queue_size=-1, accepted_queue_timeout=10): + self.requests = ThreadPool(self, min=numthreads or 1, max=max, + accepted_queue_size=accepted_queue_size, + accepted_queue_timeout=accepted_queue_timeout) self.wsgi_app = wsgi_app self.gateway = wsgi_gateways[self.wsgi_version] @@ -2092,12 +2242,14 @@ class CherryPyWSGIServer(HTTPServer): def _get_numthreads(self): return self.requests.min + def _set_numthreads(self, value): self.requests.min = value numthreads = property(_get_numthreads, _set_numthreads) class WSGIGateway(Gateway): + """A base class to interface HTTPServer with WSGI.""" def __init__(self, req): @@ -2129,7 +2281,7 @@ class WSGIGateway(Gateway): if hasattr(response, "close"): response.close() - def start_response(self, status, headers, exc_info = None): + def start_response(self, status, headers, exc_info=None): """WSGI callable to begin the HTTP response.""" # "The application may call start_response more than once, # if and only if the exc_info argument is provided." @@ -2150,9 +2302,11 @@ class WSGIGateway(Gateway): self.req.status = status for k, v in headers: if not isinstance(k, str): - raise TypeError("WSGI response header key %r is not of type str." % k) + raise TypeError( + "WSGI response header key %r is not of type str." % k) if not isinstance(v, str): - raise TypeError("WSGI response header value %r is not of type str." % v) + raise TypeError( + "WSGI response header value %r is not of type str." % v) if k.lower() == 'content-length': self.remaining_bytes_out = int(v) self.req.outheaders.extend(headers) @@ -2173,7 +2327,8 @@ class WSGIGateway(Gateway): if rbo is not None and chunklen > rbo: if not self.req.sent_headers: # Whew. We can send a 500 to the client. - self.req.simple_response("500 Internal Server Error", + self.req.simple_response( + "500 Internal Server Error", "The requested resource returned more bytes than the " "declared Content-Length.") else: @@ -2195,6 +2350,7 @@ class WSGIGateway(Gateway): class WSGIGateway_10(WSGIGateway): + """A Gateway class to interface HTTPServer with WSGI 1.0.x.""" def get_environ(self): @@ -2223,7 +2379,7 @@ class WSGIGateway_10(WSGIGateway): 'wsgi.run_once': False, 'wsgi.url_scheme': req.scheme, 'wsgi.version': (1, 0), - } + } if isinstance(req.server.bind_addr, basestring): # AF_UNIX. This isn't really allowed by WSGI, which doesn't @@ -2251,17 +2407,19 @@ class WSGIGateway_10(WSGIGateway): class WSGIGateway_u0(WSGIGateway_10): + """A Gateway class to interface HTTPServer with WSGI u.0. - WSGI u.0 is an experimental protocol, which uses unicode for keys and values - in both Python 2 and Python 3. + WSGI u.0 is an experimental protocol, which uses unicode for keys and + values in both Python 2 and Python 3. """ def get_environ(self): """Return a new environ dict targeting the given wsgi.version""" req = self.req env_10 = WSGIGateway_10.get_environ(self) - env = dict([(k.decode('ISO-8859-1'), v) for k, v in env_10.iteritems()]) + env = dict([(k.decode('ISO-8859-1'), v) + for k, v in env_10.iteritems()]) env[u'wsgi.version'] = ('u', 0) # Request-URI @@ -2286,7 +2444,9 @@ wsgi_gateways = { ('u', 0): WSGIGateway_u0, } + class WSGIPathInfoDispatcher(object): + """A WSGI dispatcher for dispatch based on the PATH_INFO. apps: a dict or list of (path_prefix, app) pairs. @@ -2299,7 +2459,7 @@ class WSGIPathInfoDispatcher(object): pass # Sort the apps by len(path), descending - apps.sort(cmp=lambda x,y: cmp(len(x[0]), len(y[0]))) + apps.sort(cmp=lambda x, y: cmp(len(x[0]), len(y[0]))) apps.reverse() # The path_prefix strings must start, but not end, with a slash. @@ -2319,4 +2479,3 @@ class WSGIPathInfoDispatcher(object): start_response('404 Not Found', [('Content-Type', 'text/plain'), ('Content-Length', '0')]) return [''] - diff --git a/lib/cherrypy/wsgiserver/wsgiserver3.py b/lib/cherrypy/wsgiserver/wsgiserver3.py index 1550ee51..8bba6261 100644 --- a/lib/cherrypy/wsgiserver/wsgiserver3.py +++ b/lib/cherrypy/wsgiserver/wsgiserver3.py @@ -86,9 +86,12 @@ import re import email.utils import socket import sys -if 'win' in sys.platform and not hasattr(socket, 'IPPROTO_IPV6'): - socket.IPPROTO_IPV6 = 41 -if sys.version_info < (3,1): +if 'win' in sys.platform and hasattr(socket, "AF_INET6"): + if not hasattr(socket, 'IPPROTO_IPV6'): + socket.IPPROTO_IPV6 = 41 + if not hasattr(socket, 'IPV6_V6ONLY'): + socket.IPV6_V6ONLY = 27 +if sys.version_info < (3, 1): import io else: import _pyio as io @@ -97,25 +100,27 @@ DEFAULT_BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE import threading import time from traceback import format_exc -from urllib.parse import unquote -from urllib.parse import urlparse -from urllib.parse import scheme_chars -import warnings if sys.version_info >= (3, 0): bytestr = bytes unicodestr = str basestring = (bytes, str) + def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given encoding.""" + """Return the given native string as a byte string in the given + encoding. + """ # In Python 3, the native string type is unicode return n.encode(encoding) else: bytestr = str unicodestr = unicode basestring = basestring + def ntob(n, encoding='ISO-8859-1'): - """Return the given native string as a byte string in the given encoding.""" + """Return the given native string as a byte string in the given + encoding. + """ # In Python 2, the native string type is bytes. Assume it's already # in the given encoding, which for ISO-8859-1 is almost always what # was intended. @@ -136,6 +141,7 @@ quoted_slash = re.compile(ntob("(?i)%2F")) import errno + def plat_specific_errors(*errnames): """Return error numbers for all errors in errnames on this platform. @@ -160,24 +166,27 @@ socket_errors_to_ignore = plat_specific_errors( "ECONNABORTED", "WSAECONNABORTED", "ENETRESET", "WSAENETRESET", "EHOSTDOWN", "EHOSTUNREACH", - ) +) socket_errors_to_ignore.append("timed out") socket_errors_to_ignore.append("The read operation timed out") socket_errors_nonblocking = plat_specific_errors( 'EAGAIN', 'EWOULDBLOCK', 'WSAEWOULDBLOCK') -comma_separated_headers = [ntob(h) for h in +comma_separated_headers = [ + ntob(h) for h in ['Accept', 'Accept-Charset', 'Accept-Encoding', 'Accept-Language', 'Accept-Ranges', 'Allow', 'Cache-Control', 'Connection', 'Content-Encoding', 'Content-Language', 'Expect', 'If-Match', 'If-None-Match', 'Pragma', 'Proxy-Authenticate', 'TE', 'Trailer', 'Transfer-Encoding', 'Upgrade', 'Vary', 'Via', 'Warning', - 'WWW-Authenticate']] + 'WWW-Authenticate'] +] import logging -if not hasattr(logging, 'statistics'): logging.statistics = {} +if not hasattr(logging, 'statistics'): + logging.statistics = {} def read_headers(rfile, hdict=None): @@ -232,7 +241,9 @@ def read_headers(rfile, hdict=None): class MaxSizeExceeded(Exception): pass + class SizeCheckWrapper(object): + """Wraps a file-like object, raising MaxSizeExceeded if too large.""" def __init__(self, rfile, maxlen): @@ -265,8 +276,8 @@ class SizeCheckWrapper(object): self.bytes_read += len(data) self._check_length() res.append(data) - # See http://www.cherrypy.org/ticket/421 - if len(data) < 256 or data[-1:] == "\n": + # See https://bitbucket.org/cherrypy/cherrypy/issue/421 + if len(data) < 256 or data[-1:] == LF: return EMPTY.join(res) def readlines(self, sizehint=0): @@ -302,6 +313,7 @@ class SizeCheckWrapper(object): class KnownLengthRFile(object): + """Wraps a file-like object, returning an empty string when exhausted.""" def __init__(self, rfile, content_length): @@ -358,6 +370,7 @@ class KnownLengthRFile(object): class ChunkedRFile(object): + """Wraps a file-like object, returning an empty string when exhausted. This class is intended to provide a conforming wsgi.input value for @@ -407,8 +420,8 @@ class ChunkedRFile(object): crlf = self.rfile.read(2) if crlf != CRLF: raise ValueError( - "Bad chunked transfer coding (expected '\\r\\n', " - "got " + repr(crlf) + ")") + "Bad chunked transfer coding (expected '\\r\\n', " + "got " + repr(crlf) + ")") def read(self, size=None): data = EMPTY @@ -510,6 +523,7 @@ class ChunkedRFile(object): class HTTPRequest(object): + """An HTTP Request (and response). A single HTTP connection may consist of multiple request/response pairs. @@ -543,7 +557,7 @@ class HTTPRequest(object): This value is set automatically inside send_headers.""" def __init__(self, server, conn): - self.server= server + self.server = server self.conn = conn self.ready = False @@ -569,7 +583,8 @@ class HTTPRequest(object): try: success = self.read_request_line() except MaxSizeExceeded: - self.simple_response("414 Request-URI Too Long", + self.simple_response( + "414 Request-URI Too Long", "The Request-URI sent with the request exceeds the maximum " "allowed bytes.") return @@ -580,7 +595,8 @@ class HTTPRequest(object): try: success = self.read_request_headers() except MaxSizeExceeded: - self.simple_response("413 Request Entity Too Large", + self.simple_response( + "413 Request Entity Too Large", "The headers sent with the request exceed the maximum " "allowed bytes.") return @@ -616,12 +632,14 @@ class HTTPRequest(object): return False if not request_line.endswith(CRLF): - self.simple_response("400 Bad Request", "HTTP requires CRLF terminators") + self.simple_response( + "400 Bad Request", "HTTP requires CRLF terminators") return False try: method, uri, req_protocol = request_line.strip().split(SPACE, 2) - # The [x:y] slicing is necessary for byte strings to avoid getting ord's + # The [x:y] slicing is necessary for byte strings to avoid getting + # ord's rp = int(req_protocol[5:6]), int(req_protocol[7:8]) except ValueError: self.simple_response("400 Bad Request", "Malformed Request-Line") @@ -676,7 +694,8 @@ class HTTPRequest(object): # Notice that, in (b), the response will be "HTTP/1.1" even though # the client only understands 1.0. RFC 2616 10.5.6 says we should # only return 505 if the _major_ version is different. - # The [x:y] slicing is necessary for byte strings to avoid getting ord's + # The [x:y] slicing is necessary for byte strings to avoid getting + # ord's sp = int(self.server.protocol[5:6]), int(self.server.protocol[7:8]) if sp[0] != rp[0]: @@ -700,7 +719,8 @@ class HTTPRequest(object): mrbs = self.server.max_request_body_size if mrbs and int(self.inheaders.get(b"Content-Length", 0)) > mrbs: - self.simple_response("413 Request Entity Too Large", + self.simple_response( + "413 Request Entity Too Large", "The entity sent with the request exceeds the maximum " "allowed bytes.") return False @@ -754,8 +774,10 @@ class HTTPRequest(object): # but it seems like it would be a big slowdown for such a rare case. if self.inheaders.get(b"Expect", b"") == b"100-continue": # Don't use simple_response here, because it emits headers - # we don't want. See http://www.cherrypy.org/ticket/951 - msg = self.server.protocol.encode('ascii') + b" 100 Continue\r\n\r\n" + # we don't want. See + # https://bitbucket.org/cherrypy/cherrypy/issue/951 + msg = self.server.protocol.encode( + 'ascii') + b" 100 Continue\r\n\r\n" try: self.conn.wfile.write(msg) except socket.error: @@ -791,9 +813,10 @@ class HTTPRequest(object): if sep and QUESTION_MARK not in scheme: # An absoluteURI. # If there's a scheme (and it must be http or https), then: - # http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query ]] + # http_URL = "http:" "//" host [ ":" port ] [ abs_path [ "?" query + # ]] authority, path_a, path_b = remainder.partition(FORWARD_SLASH) - return scheme.lower(), authority, path_a+path_b + return scheme.lower(), authority, path_a + path_b if uri.startswith(FORWARD_SLASH): # An abs_path. @@ -823,9 +846,10 @@ class HTTPRequest(object): cl = int(self.inheaders.get(b"Content-Length", 0)) if mrbs and mrbs < cl: if not self.sent_headers: - self.simple_response("413 Request Entity Too Large", - "The entity sent with the request exceeds the maximum " - "allowed bytes.") + self.simple_response( + "413 Request Entity Too Large", + "The entity sent with the request exceeds the " + "maximum allowed bytes.") return self.rfile = KnownLengthRFile(self.conn.rfile, cl) @@ -898,7 +922,7 @@ class HTTPRequest(object): pass else: if (self.response_protocol == 'HTTP/1.1' - and self.method != b'HEAD'): + and self.method != b'HEAD'): # Use the chunked transfer-coding self.chunked_write = True self.outheaders.append((b"Transfer-Encoding", b"chunked")) @@ -934,14 +958,17 @@ class HTTPRequest(object): self.rfile.read(remaining) if b"date" not in hkeys: - self.outheaders.append( - (b"Date", email.utils.formatdate(usegmt=True).encode('ISO-8859-1'))) + self.outheaders.append(( + b"Date", + email.utils.formatdate(usegmt=True).encode('ISO-8859-1') + )) if b"server" not in hkeys: self.outheaders.append( (b"Server", self.server.server_name.encode('ISO-8859-1'))) - buf = [self.server.protocol.encode('ascii') + SPACE + self.status + CRLF] + buf = [self.server.protocol.encode( + 'ascii') + SPACE + self.status + CRLF] for k, v in self.outheaders: buf.append(k + COLON + SPACE + v + CRLF) buf.append(CRLF) @@ -949,16 +976,19 @@ class HTTPRequest(object): class NoSSLError(Exception): + """Exception raised when a client speaks HTTP to an HTTPS socket.""" pass class FatalSSLAlert(Exception): + """Exception raised when the SSL implementation signals a fatal alert.""" pass class CP_BufferedWriter(io.BufferedWriter): + """Faux file object attached to a socket object.""" def write(self, b): @@ -989,7 +1019,9 @@ def CP_makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE): else: return CP_BufferedWriter(socket.SocketIO(sock, mode), bufsize) + class HTTPConnection(object): + """An HTTP connection (active socket). server: the Server object which received this connection. @@ -1040,11 +1072,14 @@ class HTTPConnection(object): e = sys.exc_info()[1] errnum = e.args[0] # sadly SSL sockets return a different (longer) time out string - if errnum == 'timed out' or errnum == 'The read operation timed out': + if ( + errnum == 'timed out' or + errnum == 'The read operation timed out' + ): # Don't error if we're between requests; only error # if 1) no request has been started at all, or 2) we're # in the middle of a request. - # See http://www.cherrypy.org/ticket/853 + # See https://bitbucket.org/cherrypy/cherrypy/issue/853 if (not request_seen) or (req and req.started_request): # Don't bother writing the 408 if the response # has already started being written. @@ -1072,10 +1107,12 @@ class HTTPConnection(object): except NoSSLError: if req and not req.sent_headers: # Unwrap our wfile - self.wfile = CP_makefile(self.socket._sock, "wb", self.wbufsize) - req.simple_response("400 Bad Request", - "The client sent a plain HTTP request, but " - "this server only speaks HTTPS on this port.") + self.wfile = CP_makefile( + self.socket._sock, "wb", self.wbufsize) + req.simple_response( + "400 Bad Request", + "The client sent a plain HTTP request, but this server " + "only speaks HTTPS on this port.") self.linger = True except Exception: e = sys.exc_info()[1] @@ -1094,13 +1131,15 @@ class HTTPConnection(object): self.rfile.close() if not self.linger: - # Python's socket module does NOT call close on the kernel socket - # when you call socket.close(). We do so manually here because we - # want this server to send a FIN TCP segment immediately. Note this - # must be called *before* calling socket.close(), because the latter - # drops its reference to the kernel socket. - # Python 3 *probably* fixed this with socket._real_close; hard to tell. -## self.socket._sock.close() + # Python's socket module does NOT call close on the kernel + # socket when you call socket.close(). We do so manually here + # because we want this server to send a FIN TCP segment + # immediately. Note this must be called *before* calling + # socket.close(), because the latter drops its reference to + # the kernel socket. + # Python 3 *probably* fixed this with socket._real_close; + # hard to tell. +# self.socket._sock.close() self.socket.close() else: # On the other hand, sometimes we want to hang around for a bit @@ -1113,9 +1152,13 @@ class HTTPConnection(object): class TrueyZero(object): - """An object which equals and does math like the integer '0' but evals True.""" + + """An object which equals and does math like the integer 0 but evals True. + """ + def __add__(self, other): return other + def __radd__(self, other): return other trueyzero = TrueyZero() @@ -1123,7 +1166,9 @@ trueyzero = TrueyZero() _SHUTDOWNREQUEST = None + class WorkerThread(threading.Thread): + """Thread which continuously polls a Queue for Connection objects. Due to the timing issues of polling a Queue, a WorkerThread does not @@ -1143,7 +1188,6 @@ class WorkerThread(threading.Thread): """A simple flag for the calling server to know when this thread has begun polling the Queue.""" - def __init__(self, server): self.ready = False self.server = server @@ -1154,12 +1198,30 @@ class WorkerThread(threading.Thread): self.start_time = None self.work_time = 0 self.stats = { - 'Requests': lambda s: self.requests_seen + ((self.start_time is None) and trueyzero or self.conn.requests_seen), - 'Bytes Read': lambda s: self.bytes_read + ((self.start_time is None) and trueyzero or self.conn.rfile.bytes_read), - 'Bytes Written': lambda s: self.bytes_written + ((self.start_time is None) and trueyzero or self.conn.wfile.bytes_written), - 'Work Time': lambda s: self.work_time + ((self.start_time is None) and trueyzero or time.time() - self.start_time), - 'Read Throughput': lambda s: s['Bytes Read'](s) / (s['Work Time'](s) or 1e-6), - 'Write Throughput': lambda s: s['Bytes Written'](s) / (s['Work Time'](s) or 1e-6), + 'Requests': lambda s: self.requests_seen + ( + (self.start_time is None) and + trueyzero or + self.conn.requests_seen + ), + 'Bytes Read': lambda s: self.bytes_read + ( + (self.start_time is None) and + trueyzero or + self.conn.rfile.bytes_read + ), + 'Bytes Written': lambda s: self.bytes_written + ( + (self.start_time is None) and + trueyzero or + self.conn.wfile.bytes_written + ), + 'Work Time': lambda s: self.work_time + ( + (self.start_time is None) and + trueyzero or + time.time() - self.start_time + ), + 'Read Throughput': lambda s: s['Bytes Read'](s) / ( + s['Work Time'](s) or 1e-6), + 'Write Throughput': lambda s: s['Bytes Written'](s) / ( + s['Work Time'](s) or 1e-6), } threading.Thread.__init__(self) @@ -1192,18 +1254,21 @@ class WorkerThread(threading.Thread): class ThreadPool(object): + """A Request Queue for an HTTPServer which pools threads. ThreadPool objects must provide min, get(), put(obj), start() and stop(timeout) attributes. """ - def __init__(self, server, min=10, max=-1): + def __init__(self, server, min=10, max=-1, + accepted_queue_size=-1, accepted_queue_timeout=10): self.server = server self.min = min self.max = max self._threads = [] - self._queue = queue.Queue() + self._queue = queue.Queue(maxsize=accepted_queue_size) + self._queue_put_timeout = accepted_queue_timeout self.get = self._queue.get def start(self): @@ -1223,19 +1288,30 @@ class ThreadPool(object): idle = property(_get_idle, doc=_get_idle.__doc__) def put(self, obj): - self._queue.put(obj) + self._queue.put(obj, block=True, timeout=self._queue_put_timeout) if obj is _SHUTDOWNREQUEST: return def grow(self, amount): """Spawn new worker threads (not above self.max).""" - for i in range(amount): - if self.max > 0 and len(self._threads) >= self.max: - break - worker = WorkerThread(self.server) - worker.setName("CP Server " + worker.getName()) - self._threads.append(worker) - worker.start() + if self.max > 0: + budget = max(self.max - len(self._threads), 0) + else: + # self.max <= 0 indicates no maximum + budget = float('inf') + + n_new = min(amount, budget) + + workers = [self._spawn_worker() for i in range(n_new)] + while not all(worker.ready for worker in workers): + time.sleep(.1) + self._threads.extend(workers) + + def _spawn_worker(self): + worker = WorkerThread(self.server) + worker.setName("CP Server " + worker.getName()) + worker.start() + return worker def shrink(self, amount): """Kill off worker threads (not below self.min).""" @@ -1246,13 +1322,17 @@ class ThreadPool(object): self._threads.remove(t) amount -= 1 - if amount > 0: - for i in range(min(amount, len(self._threads) - self.min)): - # Put a number of shutdown requests on the queue equal - # to 'amount'. Once each of those is processed by a worker, - # that worker will terminate and be culled from our list - # in self.put. - self._queue.put(_SHUTDOWNREQUEST) + # calculate the number of threads above the minimum + n_extra = max(len(self._threads) - self.min, 0) + + # don't remove more than amount + n_to_remove = min(amount, n_extra) + + # put shutdown requests on the queue equal to the number of threads + # to remove. As each request is processed by a worker, that worker + # will terminate and be culled from the list. + for n in range(n_to_remove): + self._queue.put(_SHUTDOWNREQUEST) def stop(self, timeout=5): # Must shut down threads here so the code that calls @@ -1287,7 +1367,8 @@ class ThreadPool(object): worker.join() except (AssertionError, # Ignore repeated Ctrl-C. - # See http://www.cherrypy.org/ticket/691. + # See + # https://bitbucket.org/cherrypy/cherrypy/issue/691. KeyboardInterrupt): pass @@ -1296,12 +1377,19 @@ class ThreadPool(object): qsize = property(_get_qsize) - try: import fcntl except ImportError: try: from ctypes import windll, WinError + import ctypes.wintypes + _SetHandleInformation = windll.kernel32.SetHandleInformation + _SetHandleInformation.argtypes = [ + ctypes.wintypes.HANDLE, + ctypes.wintypes.DWORD, + ctypes.wintypes.DWORD, + ] + _SetHandleInformation.restype = ctypes.wintypes.BOOL except ImportError: def prevent_socket_inheritance(sock): """Dummy function, since neither fcntl nor ctypes are available.""" @@ -1309,7 +1397,7 @@ except ImportError: else: def prevent_socket_inheritance(sock): """Mark the given socket fd as non-inheritable (Windows).""" - if not windll.kernel32.SetHandleInformation(sock.fileno(), 1, 0): + if not _SetHandleInformation(sock.fileno(), 1, 0): raise WinError() else: def prevent_socket_inheritance(sock): @@ -1320,12 +1408,14 @@ else: class SSLAdapter(object): + """Base class for SSL driver library adapters. Required methods: * ``wrap(sock) -> (wrapped socket, ssl environ dict)`` - * ``makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE) -> socket file object`` + * ``makefile(sock, mode='r', bufsize=DEFAULT_BUFFER_SIZE) -> + socket file object`` """ def __init__(self, certificate, private_key, certificate_chain=None): @@ -1341,6 +1431,7 @@ class SSLAdapter(object): class HTTPServer(object): + """An HTTP server.""" _bind_addr = "127.0.0.1" @@ -1353,7 +1444,8 @@ class HTTPServer(object): """The minimum number of worker threads to create (default 10).""" maxthreads = None - """The maximum number of worker threads to create (default -1 = no limit).""" + """The maximum number of worker threads to create (default -1 = no limit). + """ server_name = None """The name of the server; defaults to socket.gethostname().""" @@ -1365,15 +1457,18 @@ class HTTPServer(object): features used in the response.""" request_queue_size = 5 - """The 'backlog' arg to socket.listen(); max queued connections (default 5).""" + """The 'backlog' arg to socket.listen(); max queued connections + (default 5). + """ shutdown_timeout = 5 - """The total time, in seconds, to wait for worker threads to cleanly exit.""" + """The total time, in seconds, to wait for worker threads to cleanly exit. + """ timeout = 10 """The timeout in seconds for accepted connections (default 10).""" - version = "CherryPy/3.2.2" + version = "CherryPy/3.6.0" """A version string for the HTTPServer.""" software = None @@ -1382,7 +1477,9 @@ class HTTPServer(object): If None, this defaults to ``'%s Server' % self.version``.""" ready = False - """An internal flag which marks whether the socket is accepting connections.""" + """An internal flag which marks whether the socket is accepting + connections. + """ max_request_header_size = 0 """The maximum size, in bytes, for request headers, or 0 for no limit.""" @@ -1426,14 +1523,15 @@ class HTTPServer(object): 'Threads': lambda s: len(getattr(self.requests, "_threads", [])), 'Threads Idle': lambda s: getattr(self.requests, "idle", None), 'Socket Errors': 0, - 'Requests': lambda s: (not s['Enabled']) and -1 or sum([w['Requests'](w) for w - in s['Worker Threads'].values()], 0), - 'Bytes Read': lambda s: (not s['Enabled']) and -1 or sum([w['Bytes Read'](w) for w - in s['Worker Threads'].values()], 0), - 'Bytes Written': lambda s: (not s['Enabled']) and -1 or sum([w['Bytes Written'](w) for w - in s['Worker Threads'].values()], 0), - 'Work Time': lambda s: (not s['Enabled']) and -1 or sum([w['Work Time'](w) for w - in s['Worker Threads'].values()], 0), + 'Requests': lambda s: (not s['Enabled']) and -1 or sum( + [w['Requests'](w) for w in s['Worker Threads'].values()], 0), + 'Bytes Read': lambda s: (not s['Enabled']) and -1 or sum( + [w['Bytes Read'](w) for w in s['Worker Threads'].values()], 0), + 'Bytes Written': lambda s: (not s['Enabled']) and -1 or sum( + [w['Bytes Written'](w) for w in s['Worker Threads'].values()], + 0), + 'Work Time': lambda s: (not s['Enabled']) and -1 or sum( + [w['Work Time'](w) for w in s['Worker Threads'].values()], 0), 'Read Throughput': lambda s: (not s['Enabled']) and -1 or sum( [w['Bytes Read'](w) / (w['Work Time'](w) or 1e-6) for w in s['Worker Threads'].values()], 0), @@ -1441,7 +1539,7 @@ class HTTPServer(object): [w['Bytes Written'](w) / (w['Work Time'](w) or 1e-6) for w in s['Worker Threads'].values()], 0), 'Worker Threads': {}, - } + } logging.statistics["CherryPy HTTPServer %d" % id(self)] = self.stats def runtime(self): @@ -1456,6 +1554,7 @@ class HTTPServer(object): def _get_bind_addr(self): return self._bind_addr + def _set_bind_addr(self, value): if isinstance(value, tuple) and value[0] in ('', None): # Despite the socket module docs, using '' does not @@ -1472,7 +1571,9 @@ class HTTPServer(object): "Use '0.0.0.0' (IPv4) or '::' (IPv6) instead " "to listen on all active interfaces.") self._bind_addr = value - bind_addr = property(_get_bind_addr, _set_bind_addr, + bind_addr = property( + _get_bind_addr, + _set_bind_addr, doc="""The interface on which to listen for connections. For TCP sockets, a (host, port) tuple. Host values may be any IPv4 @@ -1500,21 +1601,28 @@ class HTTPServer(object): # AF_UNIX socket # So we can reuse the socket... - try: os.unlink(self.bind_addr) - except: pass + try: + os.unlink(self.bind_addr) + except: + pass # So everyone can access the socket... - try: os.chmod(self.bind_addr, 511) # 0777 - except: pass + try: + os.chmod(self.bind_addr, 511) # 0777 + except: + pass - info = [(socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)] + info = [ + (socket.AF_UNIX, socket.SOCK_STREAM, 0, "", self.bind_addr)] else: # AF_INET or AF_INET6 socket - # Get the correct address family for our host (allows IPv6 addresses) + # Get the correct address family for our host (allows IPv6 + # addresses) host, port = self.bind_addr try: info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM, 0, socket.AI_PASSIVE) + socket.SOCK_STREAM, 0, + socket.AI_PASSIVE) except socket.gaierror: if ':' in self.bind_addr[0]: info = [(socket.AF_INET6, socket.SOCK_STREAM, @@ -1529,7 +1637,8 @@ class HTTPServer(object): af, socktype, proto, canonname, sa = res try: self.bind(af, socktype, proto) - except socket.error: + except socket.error as serr: + msg = "%s -- (%s: %s)" % (msg, sa, serr) if self.socket: self.socket.close() self.socket = None @@ -1583,11 +1692,13 @@ class HTTPServer(object): self.socket = self.ssl_adapter.bind(self.socket) # If listening on the IPV6 any address ('::' = IN6ADDR_ANY), - # activate dual-stack. See http://www.cherrypy.org/ticket/871. + # activate dual-stack. See + # https://bitbucket.org/cherrypy/cherrypy/issue/871. if (hasattr(socket, 'AF_INET6') and family == socket.AF_INET6 - and self.bind_addr[0] in ('::', '::0', '::0.0.0.0')): + and self.bind_addr[0] in ('::', '::0', '::0.0.0.0')): try: - self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + self.socket.setsockopt( + socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) except (AttributeError, socket.error): # Apparently, the socket option is not available in # this machine's TCP stack @@ -1642,7 +1753,7 @@ class HTTPServer(object): if not isinstance(self.bind_addr, basestring): # optional values # Until we do DNS lookups, omit REMOTE_HOST - if addr is None: # sometimes this can happen + if addr is None: # sometimes this can happen # figure out if AF_INET or AF_INET6. if len(s.getsockname()) == 2: # AF_INET @@ -1655,7 +1766,12 @@ class HTTPServer(object): conn.ssl_env = ssl_env - self.requests.put(conn) + try: + self.requests.put(conn) + except queue.Full: + # Just drop the conn. TODO: write 503 back? + conn.close() + return except socket.timeout: # The only reason for the timeout in start() is so we can # notice keyboard interrupts on Win32, which don't interrupt @@ -1670,19 +1786,22 @@ class HTTPServer(object): # is received during the accept() call; all docs say retry # the call, and I *think* I'm reading it right that Python # will then go ahead and poll for and handle the signal - # elsewhere. See http://www.cherrypy.org/ticket/707. + # elsewhere. See + # https://bitbucket.org/cherrypy/cherrypy/issue/707. return if x.args[0] in socket_errors_nonblocking: - # Just try again. See http://www.cherrypy.org/ticket/479. + # Just try again. See + # https://bitbucket.org/cherrypy/cherrypy/issue/479. return if x.args[0] in socket_errors_to_ignore: # Our socket was closed. - # See http://www.cherrypy.org/ticket/686. + # See https://bitbucket.org/cherrypy/cherrypy/issue/686. return raise def _get_interrupt(self): return self._interrupt + def _set_interrupt(self, interrupt): self._interrupt = True self.stop() @@ -1708,7 +1827,8 @@ class HTTPServer(object): x = sys.exc_info()[1] if x.args[0] not in socket_errors_to_ignore: # Changed to use error code and not message - # See http://www.cherrypy.org/ticket/860. + # See + # https://bitbucket.org/cherrypy/cherrypy/issue/860. raise else: # Note that we're explicitly NOT using AI_PASSIVE, @@ -1721,8 +1841,9 @@ class HTTPServer(object): s = None try: s = socket.socket(af, socktype, proto) - # See http://groups.google.com/group/cherrypy-users/ - # browse_frm/thread/bbfe5eb39c904fe0 + # See + # http://groups.google.com/group/cherrypy-users/ + # browse_frm/thread/bbfe5eb39c904fe0 s.settimeout(1.0) s.connect((host, port)) s.close() @@ -1737,7 +1858,9 @@ class HTTPServer(object): class Gateway(object): - """A base class to interface HTTPServer with other systems, such as WSGI.""" + + """A base class to interface HTTPServer with other systems, such as WSGI. + """ def __init__(self, req): self.req = req @@ -1751,7 +1874,8 @@ class Gateway(object): # of such classes (in which case they will be lazily loaded). ssl_adapters = { 'builtin': 'cherrypy.wsgiserver.ssl_builtin.BuiltinSSLAdapter', - } +} + def get_ssl_adapter_class(name='builtin'): """Return an SSL adapter class for the given name.""" @@ -1778,18 +1902,22 @@ def get_ssl_adapter_class(name='builtin'): return adapter -# -------------------------------- WSGI Stuff -------------------------------- # +# ------------------------------- WSGI Stuff -------------------------------- # class CherryPyWSGIServer(HTTPServer): + """A subclass of HTTPServer which calls a WSGI application.""" wsgi_version = (1, 0) """The version of WSGI to produce.""" def __init__(self, bind_addr, wsgi_app, numthreads=10, server_name=None, - max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5): - self.requests = ThreadPool(self, min=numthreads or 1, max=max) + max=-1, request_queue_size=5, timeout=10, shutdown_timeout=5, + accepted_queue_size=-1, accepted_queue_timeout=10): + self.requests = ThreadPool(self, min=numthreads or 1, max=max, + accepted_queue_size=accepted_queue_size, + accepted_queue_timeout=accepted_queue_timeout) self.wsgi_app = wsgi_app self.gateway = wsgi_gateways[self.wsgi_version] @@ -1805,12 +1933,14 @@ class CherryPyWSGIServer(HTTPServer): def _get_numthreads(self): return self.requests.min + def _set_numthreads(self, value): self.requests.min = value numthreads = property(_get_numthreads, _set_numthreads) class WSGIGateway(Gateway): + """A base class to interface HTTPServer with WSGI.""" def __init__(self, req): @@ -1842,7 +1972,7 @@ class WSGIGateway(Gateway): if hasattr(response, "close"): response.close() - def start_response(self, status, headers, exc_info = None): + def start_response(self, status, headers, exc_info=None): """WSGI callable to begin the HTTP response.""" # "The application may call start_response more than once, # if and only if the exc_info argument is provided." @@ -1870,12 +2000,15 @@ class WSGIGateway(Gateway): for k, v in headers: if not isinstance(k, str): - raise TypeError("WSGI response header key %r is not of type str." % k) + raise TypeError( + "WSGI response header key %r is not of type str." % k) if not isinstance(v, str): - raise TypeError("WSGI response header value %r is not of type str." % v) + raise TypeError( + "WSGI response header value %r is not of type str." % v) if k.lower() == 'content-length': self.remaining_bytes_out = int(v) - self.req.outheaders.append((k.encode('ISO-8859-1'), v.encode('ISO-8859-1'))) + self.req.outheaders.append( + (k.encode('ISO-8859-1'), v.encode('ISO-8859-1'))) return self.write @@ -1894,8 +2027,9 @@ class WSGIGateway(Gateway): if not self.req.sent_headers: # Whew. We can send a 500 to the client. self.req.simple_response("500 Internal Server Error", - "The requested resource returned more bytes than the " - "declared Content-Length.") + "The requested resource returned " + "more bytes than the declared " + "Content-Length.") else: # Dang. We have probably already sent data. Truncate the chunk # to fit (so the client doesn't hang) and raise an error later. @@ -1915,6 +2049,7 @@ class WSGIGateway(Gateway): class WSGIGateway_10(WSGIGateway): + """A Gateway class to interface HTTPServer with WSGI 1.0.x.""" def get_environ(self): @@ -1930,7 +2065,7 @@ class WSGIGateway_10(WSGIGateway): 'REMOTE_ADDR': req.conn.remote_addr or '', 'REMOTE_PORT': str(req.conn.remote_port or ''), 'REQUEST_METHOD': req.method.decode('ISO-8859-1'), - 'REQUEST_URI': req.uri, + 'REQUEST_URI': req.uri.decode('ISO-8859-1'), 'SCRIPT_NAME': '', 'SERVER_NAME': req.server.server_name, # Bah. "SERVER_PROTOCOL" is actually the REQUEST protocol. @@ -1943,8 +2078,7 @@ class WSGIGateway_10(WSGIGateway): 'wsgi.run_once': False, 'wsgi.url_scheme': req.scheme.decode('ISO-8859-1'), 'wsgi.version': (1, 0), - } - + } if isinstance(req.server.bind_addr, basestring): # AF_UNIX. This isn't really allowed by WSGI, which doesn't # address unix domain sockets. But it's better than nothing. @@ -1972,10 +2106,11 @@ class WSGIGateway_10(WSGIGateway): class WSGIGateway_u0(WSGIGateway_10): + """A Gateway class to interface HTTPServer with WSGI u.0. - WSGI u.0 is an experimental protocol, which uses unicode for keys and values - in both Python 2 and Python 3. + WSGI u.0 is an experimental protocol, which uses unicode for keys + and values in both Python 2 and Python 3. """ def get_environ(self): @@ -2004,7 +2139,9 @@ wsgi_gateways = { ('u', 0): WSGIGateway_u0, } + class WSGIPathInfoDispatcher(object): + """A WSGI dispatcher for dispatch based on the PATH_INFO. apps: a dict or list of (path_prefix, app) pairs. @@ -2037,4 +2174,3 @@ class WSGIPathInfoDispatcher(object): start_response('404 Not Found', [('Content-Type', 'text/plain'), ('Content-Length', '0')]) return [''] - diff --git a/lib/concurrent/LICENSE b/lib/concurrent/LICENSE new file mode 100644 index 00000000..c430db0f --- /dev/null +++ b/lib/concurrent/LICENSE @@ -0,0 +1,21 @@ +Copyright 2009 Brian Quinlan. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY BRIAN QUINLAN "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +HALL THE FREEBSD PROJECT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/lib/concurrent/__init__.py b/lib/concurrent/__init__.py new file mode 100644 index 00000000..b36383a6 --- /dev/null +++ b/lib/concurrent/__init__.py @@ -0,0 +1,3 @@ +from pkgutil import extend_path + +__path__ = extend_path(__path__, __name__) diff --git a/lib/concurrent/futures/__init__.py b/lib/concurrent/futures/__init__.py new file mode 100644 index 00000000..fef52819 --- /dev/null +++ b/lib/concurrent/futures/__init__.py @@ -0,0 +1,23 @@ +# Copyright 2009 Brian Quinlan. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Execute computations asynchronously using threads or processes.""" + +__author__ = 'Brian Quinlan (brian@sweetapp.com)' + +from concurrent.futures._base import (FIRST_COMPLETED, + FIRST_EXCEPTION, + ALL_COMPLETED, + CancelledError, + TimeoutError, + Future, + Executor, + wait, + as_completed) +from concurrent.futures.thread import ThreadPoolExecutor + +# Jython doesn't have multiprocessing +try: + from concurrent.futures.process import ProcessPoolExecutor +except ImportError: + pass diff --git a/lib/concurrent/futures/_base.py b/lib/concurrent/futures/_base.py new file mode 100644 index 00000000..6f0c0f3b --- /dev/null +++ b/lib/concurrent/futures/_base.py @@ -0,0 +1,605 @@ +# Copyright 2009 Brian Quinlan. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +from __future__ import with_statement +import logging +import threading +import time + +from concurrent.futures._compat import reraise + +try: + from collections import namedtuple +except ImportError: + from concurrent.futures._compat import namedtuple + +__author__ = 'Brian Quinlan (brian@sweetapp.com)' + +FIRST_COMPLETED = 'FIRST_COMPLETED' +FIRST_EXCEPTION = 'FIRST_EXCEPTION' +ALL_COMPLETED = 'ALL_COMPLETED' +_AS_COMPLETED = '_AS_COMPLETED' + +# Possible future states (for internal use by the futures package). +PENDING = 'PENDING' +RUNNING = 'RUNNING' +# The future was cancelled by the user... +CANCELLED = 'CANCELLED' +# ...and _Waiter.add_cancelled() was called by a worker. +CANCELLED_AND_NOTIFIED = 'CANCELLED_AND_NOTIFIED' +FINISHED = 'FINISHED' + +_FUTURE_STATES = [ + PENDING, + RUNNING, + CANCELLED, + CANCELLED_AND_NOTIFIED, + FINISHED +] + +_STATE_TO_DESCRIPTION_MAP = { + PENDING: "pending", + RUNNING: "running", + CANCELLED: "cancelled", + CANCELLED_AND_NOTIFIED: "cancelled", + FINISHED: "finished" +} + +# Logger for internal use by the futures package. +LOGGER = logging.getLogger("concurrent.futures") + +class Error(Exception): + """Base class for all future-related exceptions.""" + pass + +class CancelledError(Error): + """The Future was cancelled.""" + pass + +class TimeoutError(Error): + """The operation exceeded the given deadline.""" + pass + +class _Waiter(object): + """Provides the event that wait() and as_completed() block on.""" + def __init__(self): + self.event = threading.Event() + self.finished_futures = [] + + def add_result(self, future): + self.finished_futures.append(future) + + def add_exception(self, future): + self.finished_futures.append(future) + + def add_cancelled(self, future): + self.finished_futures.append(future) + +class _AsCompletedWaiter(_Waiter): + """Used by as_completed().""" + + def __init__(self): + super(_AsCompletedWaiter, self).__init__() + self.lock = threading.Lock() + + def add_result(self, future): + with self.lock: + super(_AsCompletedWaiter, self).add_result(future) + self.event.set() + + def add_exception(self, future): + with self.lock: + super(_AsCompletedWaiter, self).add_exception(future) + self.event.set() + + def add_cancelled(self, future): + with self.lock: + super(_AsCompletedWaiter, self).add_cancelled(future) + self.event.set() + +class _FirstCompletedWaiter(_Waiter): + """Used by wait(return_when=FIRST_COMPLETED).""" + + def add_result(self, future): + super(_FirstCompletedWaiter, self).add_result(future) + self.event.set() + + def add_exception(self, future): + super(_FirstCompletedWaiter, self).add_exception(future) + self.event.set() + + def add_cancelled(self, future): + super(_FirstCompletedWaiter, self).add_cancelled(future) + self.event.set() + +class _AllCompletedWaiter(_Waiter): + """Used by wait(return_when=FIRST_EXCEPTION and ALL_COMPLETED).""" + + def __init__(self, num_pending_calls, stop_on_exception): + self.num_pending_calls = num_pending_calls + self.stop_on_exception = stop_on_exception + self.lock = threading.Lock() + super(_AllCompletedWaiter, self).__init__() + + def _decrement_pending_calls(self): + with self.lock: + self.num_pending_calls -= 1 + if not self.num_pending_calls: + self.event.set() + + def add_result(self, future): + super(_AllCompletedWaiter, self).add_result(future) + self._decrement_pending_calls() + + def add_exception(self, future): + super(_AllCompletedWaiter, self).add_exception(future) + if self.stop_on_exception: + self.event.set() + else: + self._decrement_pending_calls() + + def add_cancelled(self, future): + super(_AllCompletedWaiter, self).add_cancelled(future) + self._decrement_pending_calls() + +class _AcquireFutures(object): + """A context manager that does an ordered acquire of Future conditions.""" + + def __init__(self, futures): + self.futures = sorted(futures, key=id) + + def __enter__(self): + for future in self.futures: + future._condition.acquire() + + def __exit__(self, *args): + for future in self.futures: + future._condition.release() + +def _create_and_install_waiters(fs, return_when): + if return_when == _AS_COMPLETED: + waiter = _AsCompletedWaiter() + elif return_when == FIRST_COMPLETED: + waiter = _FirstCompletedWaiter() + else: + pending_count = sum( + f._state not in [CANCELLED_AND_NOTIFIED, FINISHED] for f in fs) + + if return_when == FIRST_EXCEPTION: + waiter = _AllCompletedWaiter(pending_count, stop_on_exception=True) + elif return_when == ALL_COMPLETED: + waiter = _AllCompletedWaiter(pending_count, stop_on_exception=False) + else: + raise ValueError("Invalid return condition: %r" % return_when) + + for f in fs: + f._waiters.append(waiter) + + return waiter + +def as_completed(fs, timeout=None): + """An iterator over the given futures that yields each as it completes. + + Args: + fs: The sequence of Futures (possibly created by different Executors) to + iterate over. + timeout: The maximum number of seconds to wait. If None, then there + is no limit on the wait time. + + Returns: + An iterator that yields the given Futures as they complete (finished or + cancelled). + + Raises: + TimeoutError: If the entire result iterator could not be generated + before the given timeout. + """ + if timeout is not None: + end_time = timeout + time.time() + + with _AcquireFutures(fs): + finished = set( + f for f in fs + if f._state in [CANCELLED_AND_NOTIFIED, FINISHED]) + pending = set(fs) - finished + waiter = _create_and_install_waiters(fs, _AS_COMPLETED) + + try: + for future in finished: + yield future + + while pending: + if timeout is None: + wait_timeout = None + else: + wait_timeout = end_time - time.time() + if wait_timeout < 0: + raise TimeoutError( + '%d (of %d) futures unfinished' % ( + len(pending), len(fs))) + + waiter.event.wait(wait_timeout) + + with waiter.lock: + finished = waiter.finished_futures + waiter.finished_futures = [] + waiter.event.clear() + + for future in finished: + yield future + pending.remove(future) + + finally: + for f in fs: + f._waiters.remove(waiter) + +DoneAndNotDoneFutures = namedtuple( + 'DoneAndNotDoneFutures', 'done not_done') +def wait(fs, timeout=None, return_when=ALL_COMPLETED): + """Wait for the futures in the given sequence to complete. + + Args: + fs: The sequence of Futures (possibly created by different Executors) to + wait upon. + timeout: The maximum number of seconds to wait. If None, then there + is no limit on the wait time. + return_when: Indicates when this function should return. The options + are: + + FIRST_COMPLETED - Return when any future finishes or is + cancelled. + FIRST_EXCEPTION - Return when any future finishes by raising an + exception. If no future raises an exception + then it is equivalent to ALL_COMPLETED. + ALL_COMPLETED - Return when all futures finish or are cancelled. + + Returns: + A named 2-tuple of sets. The first set, named 'done', contains the + futures that completed (is finished or cancelled) before the wait + completed. The second set, named 'not_done', contains uncompleted + futures. + """ + with _AcquireFutures(fs): + done = set(f for f in fs + if f._state in [CANCELLED_AND_NOTIFIED, FINISHED]) + not_done = set(fs) - done + + if (return_when == FIRST_COMPLETED) and done: + return DoneAndNotDoneFutures(done, not_done) + elif (return_when == FIRST_EXCEPTION) and done: + if any(f for f in done + if not f.cancelled() and f.exception() is not None): + return DoneAndNotDoneFutures(done, not_done) + + if len(done) == len(fs): + return DoneAndNotDoneFutures(done, not_done) + + waiter = _create_and_install_waiters(fs, return_when) + + waiter.event.wait(timeout) + for f in fs: + f._waiters.remove(waiter) + + done.update(waiter.finished_futures) + return DoneAndNotDoneFutures(done, set(fs) - done) + +class Future(object): + """Represents the result of an asynchronous computation.""" + + def __init__(self): + """Initializes the future. Should not be called by clients.""" + self._condition = threading.Condition() + self._state = PENDING + self._result = None + self._exception = None + self._traceback = None + self._waiters = [] + self._done_callbacks = [] + + def _invoke_callbacks(self): + for callback in self._done_callbacks: + try: + callback(self) + except Exception: + LOGGER.exception('exception calling callback for %r', self) + + def __repr__(self): + with self._condition: + if self._state == FINISHED: + if self._exception: + return '' % ( + hex(id(self)), + _STATE_TO_DESCRIPTION_MAP[self._state], + self._exception.__class__.__name__) + else: + return '' % ( + hex(id(self)), + _STATE_TO_DESCRIPTION_MAP[self._state], + self._result.__class__.__name__) + return '' % ( + hex(id(self)), + _STATE_TO_DESCRIPTION_MAP[self._state]) + + def cancel(self): + """Cancel the future if possible. + + Returns True if the future was cancelled, False otherwise. A future + cannot be cancelled if it is running or has already completed. + """ + with self._condition: + if self._state in [RUNNING, FINISHED]: + return False + + if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: + return True + + self._state = CANCELLED + self._condition.notify_all() + + self._invoke_callbacks() + return True + + def cancelled(self): + """Return True if the future has cancelled.""" + with self._condition: + return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED] + + def running(self): + """Return True if the future is currently executing.""" + with self._condition: + return self._state == RUNNING + + def done(self): + """Return True of the future was cancelled or finished executing.""" + with self._condition: + return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED] + + def __get_result(self): + if self._exception: + reraise(self._exception, self._traceback) + else: + return self._result + + def add_done_callback(self, fn): + """Attaches a callable that will be called when the future finishes. + + Args: + fn: A callable that will be called with this future as its only + argument when the future completes or is cancelled. The callable + will always be called by a thread in the same process in which + it was added. If the future has already completed or been + cancelled then the callable will be called immediately. These + callables are called in the order that they were added. + """ + with self._condition: + if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]: + self._done_callbacks.append(fn) + return + fn(self) + + def result(self, timeout=None): + """Return the result of the call that the future represents. + + Args: + timeout: The number of seconds to wait for the result if the future + isn't done. If None, then there is no limit on the wait time. + + Returns: + The result of the call that the future represents. + + Raises: + CancelledError: If the future was cancelled. + TimeoutError: If the future didn't finish executing before the given + timeout. + Exception: If the call raised then that exception will be raised. + """ + with self._condition: + if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: + raise CancelledError() + elif self._state == FINISHED: + return self.__get_result() + + self._condition.wait(timeout) + + if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: + raise CancelledError() + elif self._state == FINISHED: + return self.__get_result() + else: + raise TimeoutError() + + def exception_info(self, timeout=None): + """Return a tuple of (exception, traceback) raised by the call that the + future represents. + + Args: + timeout: The number of seconds to wait for the exception if the + future isn't done. If None, then there is no limit on the wait + time. + + Returns: + The exception raised by the call that the future represents or None + if the call completed without raising. + + Raises: + CancelledError: If the future was cancelled. + TimeoutError: If the future didn't finish executing before the given + timeout. + """ + with self._condition: + if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: + raise CancelledError() + elif self._state == FINISHED: + return self._exception, self._traceback + + self._condition.wait(timeout) + + if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: + raise CancelledError() + elif self._state == FINISHED: + return self._exception, self._traceback + else: + raise TimeoutError() + + def exception(self, timeout=None): + """Return the exception raised by the call that the future represents. + + Args: + timeout: The number of seconds to wait for the exception if the + future isn't done. If None, then there is no limit on the wait + time. + + Returns: + The exception raised by the call that the future represents or None + if the call completed without raising. + + Raises: + CancelledError: If the future was cancelled. + TimeoutError: If the future didn't finish executing before the given + timeout. + """ + return self.exception_info(timeout)[0] + + # The following methods should only be used by Executors and in tests. + def set_running_or_notify_cancel(self): + """Mark the future as running or process any cancel notifications. + + Should only be used by Executor implementations and unit tests. + + If the future has been cancelled (cancel() was called and returned + True) then any threads waiting on the future completing (though calls + to as_completed() or wait()) are notified and False is returned. + + If the future was not cancelled then it is put in the running state + (future calls to running() will return True) and True is returned. + + This method should be called by Executor implementations before + executing the work associated with this future. If this method returns + False then the work should not be executed. + + Returns: + False if the Future was cancelled, True otherwise. + + Raises: + RuntimeError: if this method was already called or if set_result() + or set_exception() was called. + """ + with self._condition: + if self._state == CANCELLED: + self._state = CANCELLED_AND_NOTIFIED + for waiter in self._waiters: + waiter.add_cancelled(self) + # self._condition.notify_all() is not necessary because + # self.cancel() triggers a notification. + return False + elif self._state == PENDING: + self._state = RUNNING + return True + else: + LOGGER.critical('Future %s in unexpected state: %s', + id(self.future), + self.future._state) + raise RuntimeError('Future in unexpected state') + + def set_result(self, result): + """Sets the return value of work associated with the future. + + Should only be used by Executor implementations and unit tests. + """ + with self._condition: + self._result = result + self._state = FINISHED + for waiter in self._waiters: + waiter.add_result(self) + self._condition.notify_all() + self._invoke_callbacks() + + def set_exception_info(self, exception, traceback): + """Sets the result of the future as being the given exception + and traceback. + + Should only be used by Executor implementations and unit tests. + """ + with self._condition: + self._exception = exception + self._traceback = traceback + self._state = FINISHED + for waiter in self._waiters: + waiter.add_exception(self) + self._condition.notify_all() + self._invoke_callbacks() + + def set_exception(self, exception): + """Sets the result of the future as being the given exception. + + Should only be used by Executor implementations and unit tests. + """ + self.set_exception_info(exception, None) + +class Executor(object): + """This is an abstract base class for concrete asynchronous executors.""" + + def submit(self, fn, *args, **kwargs): + """Submits a callable to be executed with the given arguments. + + Schedules the callable to be executed as fn(*args, **kwargs) and returns + a Future instance representing the execution of the callable. + + Returns: + A Future representing the given call. + """ + raise NotImplementedError() + + def map(self, fn, *iterables, **kwargs): + """Returns a iterator equivalent to map(fn, iter). + + Args: + fn: A callable that will take as many arguments as there are + passed iterables. + timeout: The maximum number of seconds to wait. If None, then there + is no limit on the wait time. + + Returns: + An iterator equivalent to: map(func, *iterables) but the calls may + be evaluated out-of-order. + + Raises: + TimeoutError: If the entire result iterator could not be generated + before the given timeout. + Exception: If fn(*args) raises for any values. + """ + timeout = kwargs.get('timeout') + if timeout is not None: + end_time = timeout + time.time() + + fs = [self.submit(fn, *args) for args in zip(*iterables)] + + try: + for future in fs: + if timeout is None: + yield future.result() + else: + yield future.result(end_time - time.time()) + finally: + for future in fs: + future.cancel() + + def shutdown(self, wait=True): + """Clean-up the resources associated with the Executor. + + It is safe to call this method several times. Otherwise, no other + methods can be called after this one. + + Args: + wait: If True then shutdown will not return until all running + futures have finished executing and the resources used by the + executor have been reclaimed. + """ + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.shutdown(wait=True) + return False diff --git a/lib/concurrent/futures/_compat.py b/lib/concurrent/futures/_compat.py new file mode 100644 index 00000000..e77cf0e5 --- /dev/null +++ b/lib/concurrent/futures/_compat.py @@ -0,0 +1,111 @@ +from keyword import iskeyword as _iskeyword +from operator import itemgetter as _itemgetter +import sys as _sys + + +def namedtuple(typename, field_names): + """Returns a new subclass of tuple with named fields. + + >>> Point = namedtuple('Point', 'x y') + >>> Point.__doc__ # docstring for the new class + 'Point(x, y)' + >>> p = Point(11, y=22) # instantiate with positional args or keywords + >>> p[0] + p[1] # indexable like a plain tuple + 33 + >>> x, y = p # unpack like a regular tuple + >>> x, y + (11, 22) + >>> p.x + p.y # fields also accessable by name + 33 + >>> d = p._asdict() # convert to a dictionary + >>> d['x'] + 11 + >>> Point(**d) # convert from a dictionary + Point(x=11, y=22) + >>> p._replace(x=100) # _replace() is like str.replace() but targets named fields + Point(x=100, y=22) + + """ + + # Parse and validate the field names. Validation serves two purposes, + # generating informative error messages and preventing template injection attacks. + if isinstance(field_names, basestring): + field_names = field_names.replace(',', ' ').split() # names separated by whitespace and/or commas + field_names = tuple(map(str, field_names)) + for name in (typename,) + field_names: + if not all(c.isalnum() or c=='_' for c in name): + raise ValueError('Type names and field names can only contain alphanumeric characters and underscores: %r' % name) + if _iskeyword(name): + raise ValueError('Type names and field names cannot be a keyword: %r' % name) + if name[0].isdigit(): + raise ValueError('Type names and field names cannot start with a number: %r' % name) + seen_names = set() + for name in field_names: + if name.startswith('_'): + raise ValueError('Field names cannot start with an underscore: %r' % name) + if name in seen_names: + raise ValueError('Encountered duplicate field name: %r' % name) + seen_names.add(name) + + # Create and fill-in the class template + numfields = len(field_names) + argtxt = repr(field_names).replace("'", "")[1:-1] # tuple repr without parens or quotes + reprtxt = ', '.join('%s=%%r' % name for name in field_names) + dicttxt = ', '.join('%r: t[%d]' % (name, pos) for pos, name in enumerate(field_names)) + template = '''class %(typename)s(tuple): + '%(typename)s(%(argtxt)s)' \n + __slots__ = () \n + _fields = %(field_names)r \n + def __new__(_cls, %(argtxt)s): + return _tuple.__new__(_cls, (%(argtxt)s)) \n + @classmethod + def _make(cls, iterable, new=tuple.__new__, len=len): + 'Make a new %(typename)s object from a sequence or iterable' + result = new(cls, iterable) + if len(result) != %(numfields)d: + raise TypeError('Expected %(numfields)d arguments, got %%d' %% len(result)) + return result \n + def __repr__(self): + return '%(typename)s(%(reprtxt)s)' %% self \n + def _asdict(t): + 'Return a new dict which maps field names to their values' + return {%(dicttxt)s} \n + def _replace(_self, **kwds): + 'Return a new %(typename)s object replacing specified fields with new values' + result = _self._make(map(kwds.pop, %(field_names)r, _self)) + if kwds: + raise ValueError('Got unexpected field names: %%r' %% kwds.keys()) + return result \n + def __getnewargs__(self): + return tuple(self) \n\n''' % locals() + for i, name in enumerate(field_names): + template += ' %s = _property(_itemgetter(%d))\n' % (name, i) + + # Execute the template string in a temporary namespace and + # support tracing utilities by setting a value for frame.f_globals['__name__'] + namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename, + _property=property, _tuple=tuple) + try: + exec(template, namespace) + except SyntaxError: + e = _sys.exc_info()[1] + raise SyntaxError(e.message + ':\n' + template) + result = namespace[typename] + + # For pickling to work, the __module__ variable needs to be set to the frame + # where the named tuple is created. Bypass this step in enviroments where + # sys._getframe is not defined (Jython for example). + if hasattr(_sys, '_getframe'): + result.__module__ = _sys._getframe(1).f_globals.get('__name__', '__main__') + + return result + + +if _sys.version_info[0] < 3: + def reraise(exc, traceback): + locals_ = {'exc_type': type(exc), 'exc_value': exc, 'traceback': traceback} + exec('raise exc_type, exc_value, traceback', {}, locals_) +else: + def reraise(exc, traceback): + # Tracebacks are embedded in exceptions in Python 3 + raise exc diff --git a/lib/concurrent/futures/process.py b/lib/concurrent/futures/process.py new file mode 100644 index 00000000..98684f8e --- /dev/null +++ b/lib/concurrent/futures/process.py @@ -0,0 +1,363 @@ +# Copyright 2009 Brian Quinlan. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Implements ProcessPoolExecutor. + +The follow diagram and text describe the data-flow through the system: + +|======================= In-process =====================|== Out-of-process ==| + ++----------+ +----------+ +--------+ +-----------+ +---------+ +| | => | Work Ids | => | | => | Call Q | => | | +| | +----------+ | | +-----------+ | | +| | | ... | | | | ... | | | +| | | 6 | | | | 5, call() | | | +| | | 7 | | | | ... | | | +| Process | | ... | | Local | +-----------+ | Process | +| Pool | +----------+ | Worker | | #1..n | +| Executor | | Thread | | | +| | +----------- + | | +-----------+ | | +| | <=> | Work Items | <=> | | <= | Result Q | <= | | +| | +------------+ | | +-----------+ | | +| | | 6: call() | | | | ... | | | +| | | future | | | | 4, result | | | +| | | ... | | | | 3, except | | | ++----------+ +------------+ +--------+ +-----------+ +---------+ + +Executor.submit() called: +- creates a uniquely numbered _WorkItem and adds it to the "Work Items" dict +- adds the id of the _WorkItem to the "Work Ids" queue + +Local worker thread: +- reads work ids from the "Work Ids" queue and looks up the corresponding + WorkItem from the "Work Items" dict: if the work item has been cancelled then + it is simply removed from the dict, otherwise it is repackaged as a + _CallItem and put in the "Call Q". New _CallItems are put in the "Call Q" + until "Call Q" is full. NOTE: the size of the "Call Q" is kept small because + calls placed in the "Call Q" can no longer be cancelled with Future.cancel(). +- reads _ResultItems from "Result Q", updates the future stored in the + "Work Items" dict and deletes the dict entry + +Process #1..n: +- reads _CallItems from "Call Q", executes the calls, and puts the resulting + _ResultItems in "Request Q" +""" + +from __future__ import with_statement +import atexit +import multiprocessing +import threading +import weakref +import sys + +from concurrent.futures import _base + +try: + import queue +except ImportError: + import Queue as queue + +__author__ = 'Brian Quinlan (brian@sweetapp.com)' + +# Workers are created as daemon threads and processes. This is done to allow the +# interpreter to exit when there are still idle processes in a +# ProcessPoolExecutor's process pool (i.e. shutdown() was not called). However, +# allowing workers to die with the interpreter has two undesirable properties: +# - The workers would still be running during interpretor shutdown, +# meaning that they would fail in unpredictable ways. +# - The workers could be killed while evaluating a work item, which could +# be bad if the callable being evaluated has external side-effects e.g. +# writing to a file. +# +# To work around this problem, an exit handler is installed which tells the +# workers to exit when their work queues are empty and then waits until the +# threads/processes finish. + +_threads_queues = weakref.WeakKeyDictionary() +_shutdown = False + +def _python_exit(): + global _shutdown + _shutdown = True + items = list(_threads_queues.items()) + for t, q in items: + q.put(None) + for t, q in items: + t.join() + +# Controls how many more calls than processes will be queued in the call queue. +# A smaller number will mean that processes spend more time idle waiting for +# work while a larger number will make Future.cancel() succeed less frequently +# (Futures in the call queue cannot be cancelled). +EXTRA_QUEUED_CALLS = 1 + +class _WorkItem(object): + def __init__(self, future, fn, args, kwargs): + self.future = future + self.fn = fn + self.args = args + self.kwargs = kwargs + +class _ResultItem(object): + def __init__(self, work_id, exception=None, result=None): + self.work_id = work_id + self.exception = exception + self.result = result + +class _CallItem(object): + def __init__(self, work_id, fn, args, kwargs): + self.work_id = work_id + self.fn = fn + self.args = args + self.kwargs = kwargs + +def _process_worker(call_queue, result_queue): + """Evaluates calls from call_queue and places the results in result_queue. + + This worker is run in a separate process. + + Args: + call_queue: A multiprocessing.Queue of _CallItems that will be read and + evaluated by the worker. + result_queue: A multiprocessing.Queue of _ResultItems that will written + to by the worker. + shutdown: A multiprocessing.Event that will be set as a signal to the + worker that it should exit when call_queue is empty. + """ + while True: + call_item = call_queue.get(block=True) + if call_item is None: + # Wake up queue management thread + result_queue.put(None) + return + try: + r = call_item.fn(*call_item.args, **call_item.kwargs) + except BaseException: + e = sys.exc_info()[1] + result_queue.put(_ResultItem(call_item.work_id, + exception=e)) + else: + result_queue.put(_ResultItem(call_item.work_id, + result=r)) + +def _add_call_item_to_queue(pending_work_items, + work_ids, + call_queue): + """Fills call_queue with _WorkItems from pending_work_items. + + This function never blocks. + + Args: + pending_work_items: A dict mapping work ids to _WorkItems e.g. + {5: <_WorkItem...>, 6: <_WorkItem...>, ...} + work_ids: A queue.Queue of work ids e.g. Queue([5, 6, ...]). Work ids + are consumed and the corresponding _WorkItems from + pending_work_items are transformed into _CallItems and put in + call_queue. + call_queue: A multiprocessing.Queue that will be filled with _CallItems + derived from _WorkItems. + """ + while True: + if call_queue.full(): + return + try: + work_id = work_ids.get(block=False) + except queue.Empty: + return + else: + work_item = pending_work_items[work_id] + + if work_item.future.set_running_or_notify_cancel(): + call_queue.put(_CallItem(work_id, + work_item.fn, + work_item.args, + work_item.kwargs), + block=True) + else: + del pending_work_items[work_id] + continue + +def _queue_management_worker(executor_reference, + processes, + pending_work_items, + work_ids_queue, + call_queue, + result_queue): + """Manages the communication between this process and the worker processes. + + This function is run in a local thread. + + Args: + executor_reference: A weakref.ref to the ProcessPoolExecutor that owns + this thread. Used to determine if the ProcessPoolExecutor has been + garbage collected and that this function can exit. + process: A list of the multiprocessing.Process instances used as + workers. + pending_work_items: A dict mapping work ids to _WorkItems e.g. + {5: <_WorkItem...>, 6: <_WorkItem...>, ...} + work_ids_queue: A queue.Queue of work ids e.g. Queue([5, 6, ...]). + call_queue: A multiprocessing.Queue that will be filled with _CallItems + derived from _WorkItems for processing by the process workers. + result_queue: A multiprocessing.Queue of _ResultItems generated by the + process workers. + """ + nb_shutdown_processes = [0] + def shutdown_one_process(): + """Tell a worker to terminate, which will in turn wake us again""" + call_queue.put(None) + nb_shutdown_processes[0] += 1 + while True: + _add_call_item_to_queue(pending_work_items, + work_ids_queue, + call_queue) + + result_item = result_queue.get(block=True) + if result_item is not None: + work_item = pending_work_items[result_item.work_id] + del pending_work_items[result_item.work_id] + + if result_item.exception: + work_item.future.set_exception(result_item.exception) + else: + work_item.future.set_result(result_item.result) + # Check whether we should start shutting down. + executor = executor_reference() + # No more work items can be added if: + # - The interpreter is shutting down OR + # - The executor that owns this worker has been collected OR + # - The executor that owns this worker has been shutdown. + if _shutdown or executor is None or executor._shutdown_thread: + # Since no new work items can be added, it is safe to shutdown + # this thread if there are no pending work items. + if not pending_work_items: + while nb_shutdown_processes[0] < len(processes): + shutdown_one_process() + # If .join() is not called on the created processes then + # some multiprocessing.Queue methods may deadlock on Mac OS + # X. + for p in processes: + p.join() + call_queue.close() + return + del executor + +_system_limits_checked = False +_system_limited = None +def _check_system_limits(): + global _system_limits_checked, _system_limited + if _system_limits_checked: + if _system_limited: + raise NotImplementedError(_system_limited) + _system_limits_checked = True + try: + import os + nsems_max = os.sysconf("SC_SEM_NSEMS_MAX") + except (AttributeError, ValueError): + # sysconf not available or setting not available + return + if nsems_max == -1: + # indetermine limit, assume that limit is determined + # by available memory only + return + if nsems_max >= 256: + # minimum number of semaphores available + # according to POSIX + return + _system_limited = "system provides too few semaphores (%d available, 256 necessary)" % nsems_max + raise NotImplementedError(_system_limited) + +class ProcessPoolExecutor(_base.Executor): + def __init__(self, max_workers=None): + """Initializes a new ProcessPoolExecutor instance. + + Args: + max_workers: The maximum number of processes that can be used to + execute the given calls. If None or not given then as many + worker processes will be created as the machine has processors. + """ + _check_system_limits() + + if max_workers is None: + self._max_workers = multiprocessing.cpu_count() + else: + self._max_workers = max_workers + + # Make the call queue slightly larger than the number of processes to + # prevent the worker processes from idling. But don't make it too big + # because futures in the call queue cannot be cancelled. + self._call_queue = multiprocessing.Queue(self._max_workers + + EXTRA_QUEUED_CALLS) + self._result_queue = multiprocessing.Queue() + self._work_ids = queue.Queue() + self._queue_management_thread = None + self._processes = set() + + # Shutdown is a two-step process. + self._shutdown_thread = False + self._shutdown_lock = threading.Lock() + self._queue_count = 0 + self._pending_work_items = {} + + def _start_queue_management_thread(self): + # When the executor gets lost, the weakref callback will wake up + # the queue management thread. + def weakref_cb(_, q=self._result_queue): + q.put(None) + if self._queue_management_thread is None: + self._queue_management_thread = threading.Thread( + target=_queue_management_worker, + args=(weakref.ref(self, weakref_cb), + self._processes, + self._pending_work_items, + self._work_ids, + self._call_queue, + self._result_queue)) + self._queue_management_thread.daemon = True + self._queue_management_thread.start() + _threads_queues[self._queue_management_thread] = self._result_queue + + def _adjust_process_count(self): + for _ in range(len(self._processes), self._max_workers): + p = multiprocessing.Process( + target=_process_worker, + args=(self._call_queue, + self._result_queue)) + p.start() + self._processes.add(p) + + def submit(self, fn, *args, **kwargs): + with self._shutdown_lock: + if self._shutdown_thread: + raise RuntimeError('cannot schedule new futures after shutdown') + + f = _base.Future() + w = _WorkItem(f, fn, args, kwargs) + + self._pending_work_items[self._queue_count] = w + self._work_ids.put(self._queue_count) + self._queue_count += 1 + # Wake up queue management thread + self._result_queue.put(None) + + self._start_queue_management_thread() + self._adjust_process_count() + return f + submit.__doc__ = _base.Executor.submit.__doc__ + + def shutdown(self, wait=True): + with self._shutdown_lock: + self._shutdown_thread = True + if self._queue_management_thread: + # Wake up queue management thread + self._result_queue.put(None) + if wait: + self._queue_management_thread.join() + # To reduce the risk of openning too many files, remove references to + # objects that use file descriptors. + self._queue_management_thread = None + self._call_queue = None + self._result_queue = None + self._processes = None + shutdown.__doc__ = _base.Executor.shutdown.__doc__ + +atexit.register(_python_exit) diff --git a/lib/concurrent/futures/thread.py b/lib/concurrent/futures/thread.py new file mode 100644 index 00000000..930d1673 --- /dev/null +++ b/lib/concurrent/futures/thread.py @@ -0,0 +1,138 @@ +# Copyright 2009 Brian Quinlan. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Implements ThreadPoolExecutor.""" + +from __future__ import with_statement +import atexit +import threading +import weakref +import sys + +from concurrent.futures import _base + +try: + import queue +except ImportError: + import Queue as queue + +__author__ = 'Brian Quinlan (brian@sweetapp.com)' + +# Workers are created as daemon threads. This is done to allow the interpreter +# to exit when there are still idle threads in a ThreadPoolExecutor's thread +# pool (i.e. shutdown() was not called). However, allowing workers to die with +# the interpreter has two undesirable properties: +# - The workers would still be running during interpretor shutdown, +# meaning that they would fail in unpredictable ways. +# - The workers could be killed while evaluating a work item, which could +# be bad if the callable being evaluated has external side-effects e.g. +# writing to a file. +# +# To work around this problem, an exit handler is installed which tells the +# workers to exit when their work queues are empty and then waits until the +# threads finish. + +_threads_queues = weakref.WeakKeyDictionary() +_shutdown = False + +def _python_exit(): + global _shutdown + _shutdown = True + items = list(_threads_queues.items()) + for t, q in items: + q.put(None) + for t, q in items: + t.join() + +atexit.register(_python_exit) + +class _WorkItem(object): + def __init__(self, future, fn, args, kwargs): + self.future = future + self.fn = fn + self.args = args + self.kwargs = kwargs + + def run(self): + if not self.future.set_running_or_notify_cancel(): + return + + try: + result = self.fn(*self.args, **self.kwargs) + except BaseException: + e, tb = sys.exc_info()[1:] + self.future.set_exception_info(e, tb) + else: + self.future.set_result(result) + +def _worker(executor_reference, work_queue): + try: + while True: + work_item = work_queue.get(block=True) + if work_item is not None: + work_item.run() + continue + executor = executor_reference() + # Exit if: + # - The interpreter is shutting down OR + # - The executor that owns the worker has been collected OR + # - The executor that owns the worker has been shutdown. + if _shutdown or executor is None or executor._shutdown: + # Notice other workers + work_queue.put(None) + return + del executor + except BaseException: + _base.LOGGER.critical('Exception in worker', exc_info=True) + +class ThreadPoolExecutor(_base.Executor): + def __init__(self, max_workers): + """Initializes a new ThreadPoolExecutor instance. + + Args: + max_workers: The maximum number of threads that can be used to + execute the given calls. + """ + self._max_workers = max_workers + self._work_queue = queue.Queue() + self._threads = set() + self._shutdown = False + self._shutdown_lock = threading.Lock() + + def submit(self, fn, *args, **kwargs): + with self._shutdown_lock: + if self._shutdown: + raise RuntimeError('cannot schedule new futures after shutdown') + + f = _base.Future() + w = _WorkItem(f, fn, args, kwargs) + + self._work_queue.put(w) + self._adjust_thread_count() + return f + submit.__doc__ = _base.Executor.submit.__doc__ + + def _adjust_thread_count(self): + # When the executor gets lost, the weakref callback will wake up + # the worker threads. + def weakref_cb(_, q=self._work_queue): + q.put(None) + # TODO(bquinlan): Should avoid creating new threads if there are more + # idle threads than items in the work queue. + if len(self._threads) < self._max_workers: + t = threading.Thread(target=_worker, + args=(weakref.ref(self, weakref_cb), + self._work_queue)) + t.daemon = True + t.start() + self._threads.add(t) + _threads_queues[t] = self._work_queue + + def shutdown(self, wait=True): + with self._shutdown_lock: + self._shutdown = True + self._work_queue.put(None) + if wait: + for t in self._threads: + t.join() + shutdown.__doc__ = _base.Executor.shutdown.__doc__ diff --git a/lib/feedparser.py b/lib/feedparser.py index ed5695bf..15fdc95b 100644 --- a/lib/feedparser.py +++ b/lib/feedparser.py @@ -1736,7 +1736,7 @@ if _XML_AVAILABLE: else: givenprefix = None prefix = self._matchnamespaces.get(lowernamespace, givenprefix) - if givenprefix and (prefix == None or (prefix == '' and lowernamespace == '')) and not self.namespacesInUse.has_key(givenprefix): + if givenprefix and (prefix is None or (prefix == '' and lowernamespace == '')) and not self.namespacesInUse.has_key(givenprefix): raise UndeclaredNamespace, "'%s' is not associated with a namespace" % givenprefix localname = str(localname).lower() diff --git a/lib/httplib2/__init__.py b/lib/httplib2/__init__.py index 441dfdc8..3652df61 100755 --- a/lib/httplib2/__init__.py +++ b/lib/httplib2/__init__.py @@ -939,7 +939,7 @@ the same interface as FileCache.""" if response.has_key('location'): location = response['location'] (scheme, authority, path, query, fragment) = parse_uri(location) - if authority == None: + if authority is None: response['location'] = urlparse.urljoin(absolute_uri, location) if response.status == 301 and method in ["GET", "HEAD"]: response['-x-permanent-redirect-url'] = response['location'] diff --git a/lib/pytz/LICENSE.txt b/lib/pytz/LICENSE.txt new file mode 100644 index 00000000..5e12fcca --- /dev/null +++ b/lib/pytz/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2003-2009 Stuart Bishop + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/lib/pytz/__init__.py b/lib/pytz/__init__.py new file mode 100644 index 00000000..da80e710 --- /dev/null +++ b/lib/pytz/__init__.py @@ -0,0 +1,1511 @@ +''' +datetime.tzinfo timezone definitions generated from the +Olson timezone database: + + ftp://elsie.nci.nih.gov/pub/tz*.tar.gz + +See the datetime section of the Python Library Reference for information +on how to use these modules. +''' + +# The Olson database is updated several times a year. +OLSON_VERSION = '2014g' +VERSION = '2014.7' # Switching to pip compatible version numbering. +__version__ = VERSION + +OLSEN_VERSION = OLSON_VERSION # Old releases had this misspelling + +__all__ = [ + 'timezone', 'utc', 'country_timezones', 'country_names', + 'AmbiguousTimeError', 'InvalidTimeError', + 'NonExistentTimeError', 'UnknownTimeZoneError', + 'all_timezones', 'all_timezones_set', + 'common_timezones', 'common_timezones_set', + ] + +import sys, datetime, os.path, gettext + +try: + from pkg_resources import resource_stream +except ImportError: + resource_stream = None + +from pytz.exceptions import AmbiguousTimeError +from pytz.exceptions import InvalidTimeError +from pytz.exceptions import NonExistentTimeError +from pytz.exceptions import UnknownTimeZoneError +from pytz.lazy import LazyDict, LazyList, LazySet +from pytz.tzinfo import unpickler +from pytz.tzfile import build_tzinfo, _byte_string + + +try: + unicode + +except NameError: # Python 3.x + + # Python 3.x doesn't have unicode(), making writing code + # for Python 2.3 and Python 3.x a pain. + unicode = str + + def ascii(s): + r""" + >>> ascii('Hello') + 'Hello' + >>> ascii('\N{TRADE MARK SIGN}') #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + UnicodeEncodeError: ... + """ + s.encode('US-ASCII') # Raise an exception if not ASCII + return s # But return the original string - not a byte string. + +else: # Python 2.x + + def ascii(s): + r""" + >>> ascii('Hello') + 'Hello' + >>> ascii(u'Hello') + 'Hello' + >>> ascii(u'\N{TRADE MARK SIGN}') #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + UnicodeEncodeError: ... + """ + return s.encode('US-ASCII') + + +def open_resource(name): + """Open a resource from the zoneinfo subdir for reading. + + Uses the pkg_resources module if available and no standard file + found at the calculated location. + """ + name_parts = name.lstrip('/').split('/') + for part in name_parts: + if part == os.path.pardir or os.path.sep in part: + raise ValueError('Bad path segment: %r' % part) + filename = os.path.join(os.path.dirname(__file__), + 'zoneinfo', *name_parts) + if not os.path.exists(filename) and resource_stream is not None: + # http://bugs.launchpad.net/bugs/383171 - we avoid using this + # unless absolutely necessary to help when a broken version of + # pkg_resources is installed. + return resource_stream(__name__, 'zoneinfo/' + name) + return open(filename, 'rb') + + +def resource_exists(name): + """Return true if the given resource exists""" + try: + open_resource(name).close() + return True + except IOError: + return False + + +# Enable this when we get some translations? +# We want an i18n API that is useful to programs using Python's gettext +# module, as well as the Zope3 i18n package. Perhaps we should just provide +# the POT file and translations, and leave it up to callers to make use +# of them. +# +# t = gettext.translation( +# 'pytz', os.path.join(os.path.dirname(__file__), 'locales'), +# fallback=True +# ) +# def _(timezone_name): +# """Translate a timezone name using the current locale, returning Unicode""" +# return t.ugettext(timezone_name) + + +_tzinfo_cache = {} + +def timezone(zone): + r''' Return a datetime.tzinfo implementation for the given timezone + + >>> from datetime import datetime, timedelta + >>> utc = timezone('UTC') + >>> eastern = timezone('US/Eastern') + >>> eastern.zone + 'US/Eastern' + >>> timezone(unicode('US/Eastern')) is eastern + True + >>> utc_dt = datetime(2002, 10, 27, 6, 0, 0, tzinfo=utc) + >>> loc_dt = utc_dt.astimezone(eastern) + >>> fmt = '%Y-%m-%d %H:%M:%S %Z (%z)' + >>> loc_dt.strftime(fmt) + '2002-10-27 01:00:00 EST (-0500)' + >>> (loc_dt - timedelta(minutes=10)).strftime(fmt) + '2002-10-27 00:50:00 EST (-0500)' + >>> eastern.normalize(loc_dt - timedelta(minutes=10)).strftime(fmt) + '2002-10-27 01:50:00 EDT (-0400)' + >>> (loc_dt + timedelta(minutes=10)).strftime(fmt) + '2002-10-27 01:10:00 EST (-0500)' + + Raises UnknownTimeZoneError if passed an unknown zone. + + >>> try: + ... timezone('Asia/Shangri-La') + ... except UnknownTimeZoneError: + ... print('Unknown') + Unknown + + >>> try: + ... timezone(unicode('\N{TRADE MARK SIGN}')) + ... except UnknownTimeZoneError: + ... print('Unknown') + Unknown + + ''' + if zone.upper() == 'UTC': + return utc + + try: + zone = ascii(zone) + except UnicodeEncodeError: + # All valid timezones are ASCII + raise UnknownTimeZoneError(zone) + + zone = _unmunge_zone(zone) + if zone not in _tzinfo_cache: + if zone in all_timezones_set: + fp = open_resource(zone) + try: + _tzinfo_cache[zone] = build_tzinfo(zone, fp) + finally: + fp.close() + else: + raise UnknownTimeZoneError(zone) + + return _tzinfo_cache[zone] + + +def _unmunge_zone(zone): + """Undo the time zone name munging done by older versions of pytz.""" + return zone.replace('_plus_', '+').replace('_minus_', '-') + + +ZERO = datetime.timedelta(0) +HOUR = datetime.timedelta(hours=1) + + +class UTC(datetime.tzinfo): + """UTC + + Optimized UTC implementation. It unpickles using the single module global + instance defined beneath this class declaration. + """ + zone = "UTC" + + _utcoffset = ZERO + _dst = ZERO + _tzname = zone + + def fromutc(self, dt): + if dt.tzinfo is None: + return self.localize(dt) + return super(utc.__class__, self).fromutc(dt) + + def utcoffset(self, dt): + return ZERO + + def tzname(self, dt): + return "UTC" + + def dst(self, dt): + return ZERO + + def __reduce__(self): + return _UTC, () + + def localize(self, dt, is_dst=False): + '''Convert naive time to local time''' + if dt.tzinfo is not None: + raise ValueError('Not naive datetime (tzinfo is already set)') + return dt.replace(tzinfo=self) + + def normalize(self, dt, is_dst=False): + '''Correct the timezone information on the given datetime''' + if dt.tzinfo is self: + return dt + if dt.tzinfo is None: + raise ValueError('Naive time - no tzinfo set') + return dt.astimezone(self) + + def __repr__(self): + return "" + + def __str__(self): + return "UTC" + + +UTC = utc = UTC() # UTC is a singleton + + +def _UTC(): + """Factory function for utc unpickling. + + Makes sure that unpickling a utc instance always returns the same + module global. + + These examples belong in the UTC class above, but it is obscured; or in + the README.txt, but we are not depending on Python 2.4 so integrating + the README.txt examples with the unit tests is not trivial. + + >>> import datetime, pickle + >>> dt = datetime.datetime(2005, 3, 1, 14, 13, 21, tzinfo=utc) + >>> naive = dt.replace(tzinfo=None) + >>> p = pickle.dumps(dt, 1) + >>> naive_p = pickle.dumps(naive, 1) + >>> len(p) - len(naive_p) + 17 + >>> new = pickle.loads(p) + >>> new == dt + True + >>> new is dt + False + >>> new.tzinfo is dt.tzinfo + True + >>> utc is UTC is timezone('UTC') + True + >>> utc is timezone('GMT') + False + """ + return utc +_UTC.__safe_for_unpickling__ = True + + +def _p(*args): + """Factory function for unpickling pytz tzinfo instances. + + Just a wrapper around tzinfo.unpickler to save a few bytes in each pickle + by shortening the path. + """ + return unpickler(*args) +_p.__safe_for_unpickling__ = True + + + +class _CountryTimezoneDict(LazyDict): + """Map ISO 3166 country code to a list of timezone names commonly used + in that country. + + iso3166_code is the two letter code used to identify the country. + + >>> def print_list(list_of_strings): + ... 'We use a helper so doctests work under Python 2.3 -> 3.x' + ... for s in list_of_strings: + ... print(s) + + >>> print_list(country_timezones['nz']) + Pacific/Auckland + Pacific/Chatham + >>> print_list(country_timezones['ch']) + Europe/Zurich + >>> print_list(country_timezones['CH']) + Europe/Zurich + >>> print_list(country_timezones[unicode('ch')]) + Europe/Zurich + >>> print_list(country_timezones['XXX']) + Traceback (most recent call last): + ... + KeyError: 'XXX' + + Previously, this information was exposed as a function rather than a + dictionary. This is still supported:: + + >>> print_list(country_timezones('nz')) + Pacific/Auckland + Pacific/Chatham + """ + def __call__(self, iso3166_code): + """Backwards compatibility.""" + return self[iso3166_code] + + def _fill(self): + data = {} + zone_tab = open_resource('zone.tab') + try: + for line in zone_tab: + line = line.decode('US-ASCII') + if line.startswith('#'): + continue + code, coordinates, zone = line.split(None, 4)[:3] + if zone not in all_timezones_set: + continue + try: + data[code].append(zone) + except KeyError: + data[code] = [zone] + self.data = data + finally: + zone_tab.close() + +country_timezones = _CountryTimezoneDict() + + +class _CountryNameDict(LazyDict): + '''Dictionary proving ISO3166 code -> English name. + + >>> print(country_names['au']) + Australia + ''' + def _fill(self): + data = {} + zone_tab = open_resource('iso3166.tab') + try: + for line in zone_tab.readlines(): + line = line.decode('US-ASCII') + if line.startswith('#'): + continue + code, name = line.split(None, 1) + data[code] = name.strip() + self.data = data + finally: + zone_tab.close() + +country_names = _CountryNameDict() + + +# Time-zone info based solely on fixed offsets + +class _FixedOffset(datetime.tzinfo): + + zone = None # to match the standard pytz API + + def __init__(self, minutes): + if abs(minutes) >= 1440: + raise ValueError("absolute offset is too large", minutes) + self._minutes = minutes + self._offset = datetime.timedelta(minutes=minutes) + + def utcoffset(self, dt): + return self._offset + + def __reduce__(self): + return FixedOffset, (self._minutes, ) + + def dst(self, dt): + return ZERO + + def tzname(self, dt): + return None + + def __repr__(self): + return 'pytz.FixedOffset(%d)' % self._minutes + + def localize(self, dt, is_dst=False): + '''Convert naive time to local time''' + if dt.tzinfo is not None: + raise ValueError('Not naive datetime (tzinfo is already set)') + return dt.replace(tzinfo=self) + + def normalize(self, dt, is_dst=False): + '''Correct the timezone information on the given datetime''' + if dt.tzinfo is None: + raise ValueError('Naive time - no tzinfo set') + return dt.replace(tzinfo=self) + + +def FixedOffset(offset, _tzinfos = {}): + """return a fixed-offset timezone based off a number of minutes. + + >>> one = FixedOffset(-330) + >>> one + pytz.FixedOffset(-330) + >>> one.utcoffset(datetime.datetime.now()) + datetime.timedelta(-1, 66600) + >>> one.dst(datetime.datetime.now()) + datetime.timedelta(0) + + >>> two = FixedOffset(1380) + >>> two + pytz.FixedOffset(1380) + >>> two.utcoffset(datetime.datetime.now()) + datetime.timedelta(0, 82800) + >>> two.dst(datetime.datetime.now()) + datetime.timedelta(0) + + The datetime.timedelta must be between the range of -1 and 1 day, + non-inclusive. + + >>> FixedOffset(1440) + Traceback (most recent call last): + ... + ValueError: ('absolute offset is too large', 1440) + + >>> FixedOffset(-1440) + Traceback (most recent call last): + ... + ValueError: ('absolute offset is too large', -1440) + + An offset of 0 is special-cased to return UTC. + + >>> FixedOffset(0) is UTC + True + + There should always be only one instance of a FixedOffset per timedelta. + This should be true for multiple creation calls. + + >>> FixedOffset(-330) is one + True + >>> FixedOffset(1380) is two + True + + It should also be true for pickling. + + >>> import pickle + >>> pickle.loads(pickle.dumps(one)) is one + True + >>> pickle.loads(pickle.dumps(two)) is two + True + """ + if offset == 0: + return UTC + + info = _tzinfos.get(offset) + if info is None: + # We haven't seen this one before. we need to save it. + + # Use setdefault to avoid a race condition and make sure we have + # only one + info = _tzinfos.setdefault(offset, _FixedOffset(offset)) + + return info + +FixedOffset.__safe_for_unpickling__ = True + + +def _test(): + import doctest, os, sys + sys.path.insert(0, os.pardir) + import pytz + return doctest.testmod(pytz) + +if __name__ == '__main__': + _test() + +all_timezones = \ +['Africa/Abidjan', + 'Africa/Accra', + 'Africa/Addis_Ababa', + 'Africa/Algiers', + 'Africa/Asmara', + 'Africa/Asmera', + 'Africa/Bamako', + 'Africa/Bangui', + 'Africa/Banjul', + 'Africa/Bissau', + 'Africa/Blantyre', + 'Africa/Brazzaville', + 'Africa/Bujumbura', + 'Africa/Cairo', + 'Africa/Casablanca', + 'Africa/Ceuta', + 'Africa/Conakry', + 'Africa/Dakar', + 'Africa/Dar_es_Salaam', + 'Africa/Djibouti', + 'Africa/Douala', + 'Africa/El_Aaiun', + 'Africa/Freetown', + 'Africa/Gaborone', + 'Africa/Harare', + 'Africa/Johannesburg', + 'Africa/Juba', + 'Africa/Kampala', + 'Africa/Khartoum', + 'Africa/Kigali', + 'Africa/Kinshasa', + 'Africa/Lagos', + 'Africa/Libreville', + 'Africa/Lome', + 'Africa/Luanda', + 'Africa/Lubumbashi', + 'Africa/Lusaka', + 'Africa/Malabo', + 'Africa/Maputo', + 'Africa/Maseru', + 'Africa/Mbabane', + 'Africa/Mogadishu', + 'Africa/Monrovia', + 'Africa/Nairobi', + 'Africa/Ndjamena', + 'Africa/Niamey', + 'Africa/Nouakchott', + 'Africa/Ouagadougou', + 'Africa/Porto-Novo', + 'Africa/Sao_Tome', + 'Africa/Timbuktu', + 'Africa/Tripoli', + 'Africa/Tunis', + 'Africa/Windhoek', + 'America/Adak', + 'America/Anchorage', + 'America/Anguilla', + 'America/Antigua', + 'America/Araguaina', + 'America/Argentina/Buenos_Aires', + 'America/Argentina/Catamarca', + 'America/Argentina/ComodRivadavia', + 'America/Argentina/Cordoba', + 'America/Argentina/Jujuy', + 'America/Argentina/La_Rioja', + 'America/Argentina/Mendoza', + 'America/Argentina/Rio_Gallegos', + 'America/Argentina/Salta', + 'America/Argentina/San_Juan', + 'America/Argentina/San_Luis', + 'America/Argentina/Tucuman', + 'America/Argentina/Ushuaia', + 'America/Aruba', + 'America/Asuncion', + 'America/Atikokan', + 'America/Atka', + 'America/Bahia', + 'America/Bahia_Banderas', + 'America/Barbados', + 'America/Belem', + 'America/Belize', + 'America/Blanc-Sablon', + 'America/Boa_Vista', + 'America/Bogota', + 'America/Boise', + 'America/Buenos_Aires', + 'America/Cambridge_Bay', + 'America/Campo_Grande', + 'America/Cancun', + 'America/Caracas', + 'America/Catamarca', + 'America/Cayenne', + 'America/Cayman', + 'America/Chicago', + 'America/Chihuahua', + 'America/Coral_Harbour', + 'America/Cordoba', + 'America/Costa_Rica', + 'America/Creston', + 'America/Cuiaba', + 'America/Curacao', + 'America/Danmarkshavn', + 'America/Dawson', + 'America/Dawson_Creek', + 'America/Denver', + 'America/Detroit', + 'America/Dominica', + 'America/Edmonton', + 'America/Eirunepe', + 'America/El_Salvador', + 'America/Ensenada', + 'America/Fort_Wayne', + 'America/Fortaleza', + 'America/Glace_Bay', + 'America/Godthab', + 'America/Goose_Bay', + 'America/Grand_Turk', + 'America/Grenada', + 'America/Guadeloupe', + 'America/Guatemala', + 'America/Guayaquil', + 'America/Guyana', + 'America/Halifax', + 'America/Havana', + 'America/Hermosillo', + 'America/Indiana/Indianapolis', + 'America/Indiana/Knox', + 'America/Indiana/Marengo', + 'America/Indiana/Petersburg', + 'America/Indiana/Tell_City', + 'America/Indiana/Vevay', + 'America/Indiana/Vincennes', + 'America/Indiana/Winamac', + 'America/Indianapolis', + 'America/Inuvik', + 'America/Iqaluit', + 'America/Jamaica', + 'America/Jujuy', + 'America/Juneau', + 'America/Kentucky/Louisville', + 'America/Kentucky/Monticello', + 'America/Knox_IN', + 'America/Kralendijk', + 'America/La_Paz', + 'America/Lima', + 'America/Los_Angeles', + 'America/Louisville', + 'America/Lower_Princes', + 'America/Maceio', + 'America/Managua', + 'America/Manaus', + 'America/Marigot', + 'America/Martinique', + 'America/Matamoros', + 'America/Mazatlan', + 'America/Mendoza', + 'America/Menominee', + 'America/Merida', + 'America/Metlakatla', + 'America/Mexico_City', + 'America/Miquelon', + 'America/Moncton', + 'America/Monterrey', + 'America/Montevideo', + 'America/Montreal', + 'America/Montserrat', + 'America/Nassau', + 'America/New_York', + 'America/Nipigon', + 'America/Nome', + 'America/Noronha', + 'America/North_Dakota/Beulah', + 'America/North_Dakota/Center', + 'America/North_Dakota/New_Salem', + 'America/Ojinaga', + 'America/Panama', + 'America/Pangnirtung', + 'America/Paramaribo', + 'America/Phoenix', + 'America/Port-au-Prince', + 'America/Port_of_Spain', + 'America/Porto_Acre', + 'America/Porto_Velho', + 'America/Puerto_Rico', + 'America/Rainy_River', + 'America/Rankin_Inlet', + 'America/Recife', + 'America/Regina', + 'America/Resolute', + 'America/Rio_Branco', + 'America/Rosario', + 'America/Santa_Isabel', + 'America/Santarem', + 'America/Santiago', + 'America/Santo_Domingo', + 'America/Sao_Paulo', + 'America/Scoresbysund', + 'America/Shiprock', + 'America/Sitka', + 'America/St_Barthelemy', + 'America/St_Johns', + 'America/St_Kitts', + 'America/St_Lucia', + 'America/St_Thomas', + 'America/St_Vincent', + 'America/Swift_Current', + 'America/Tegucigalpa', + 'America/Thule', + 'America/Thunder_Bay', + 'America/Tijuana', + 'America/Toronto', + 'America/Tortola', + 'America/Vancouver', + 'America/Virgin', + 'America/Whitehorse', + 'America/Winnipeg', + 'America/Yakutat', + 'America/Yellowknife', + 'Antarctica/Casey', + 'Antarctica/Davis', + 'Antarctica/DumontDUrville', + 'Antarctica/Macquarie', + 'Antarctica/Mawson', + 'Antarctica/McMurdo', + 'Antarctica/Palmer', + 'Antarctica/Rothera', + 'Antarctica/South_Pole', + 'Antarctica/Syowa', + 'Antarctica/Troll', + 'Antarctica/Vostok', + 'Arctic/Longyearbyen', + 'Asia/Aden', + 'Asia/Almaty', + 'Asia/Amman', + 'Asia/Anadyr', + 'Asia/Aqtau', + 'Asia/Aqtobe', + 'Asia/Ashgabat', + 'Asia/Ashkhabad', + 'Asia/Baghdad', + 'Asia/Bahrain', + 'Asia/Baku', + 'Asia/Bangkok', + 'Asia/Beirut', + 'Asia/Bishkek', + 'Asia/Brunei', + 'Asia/Calcutta', + 'Asia/Chita', + 'Asia/Choibalsan', + 'Asia/Chongqing', + 'Asia/Chungking', + 'Asia/Colombo', + 'Asia/Dacca', + 'Asia/Damascus', + 'Asia/Dhaka', + 'Asia/Dili', + 'Asia/Dubai', + 'Asia/Dushanbe', + 'Asia/Gaza', + 'Asia/Harbin', + 'Asia/Hebron', + 'Asia/Ho_Chi_Minh', + 'Asia/Hong_Kong', + 'Asia/Hovd', + 'Asia/Irkutsk', + 'Asia/Istanbul', + 'Asia/Jakarta', + 'Asia/Jayapura', + 'Asia/Jerusalem', + 'Asia/Kabul', + 'Asia/Kamchatka', + 'Asia/Karachi', + 'Asia/Kashgar', + 'Asia/Kathmandu', + 'Asia/Katmandu', + 'Asia/Khandyga', + 'Asia/Kolkata', + 'Asia/Krasnoyarsk', + 'Asia/Kuala_Lumpur', + 'Asia/Kuching', + 'Asia/Kuwait', + 'Asia/Macao', + 'Asia/Macau', + 'Asia/Magadan', + 'Asia/Makassar', + 'Asia/Manila', + 'Asia/Muscat', + 'Asia/Nicosia', + 'Asia/Novokuznetsk', + 'Asia/Novosibirsk', + 'Asia/Omsk', + 'Asia/Oral', + 'Asia/Phnom_Penh', + 'Asia/Pontianak', + 'Asia/Pyongyang', + 'Asia/Qatar', + 'Asia/Qyzylorda', + 'Asia/Rangoon', + 'Asia/Riyadh', + 'Asia/Saigon', + 'Asia/Sakhalin', + 'Asia/Samarkand', + 'Asia/Seoul', + 'Asia/Shanghai', + 'Asia/Singapore', + 'Asia/Srednekolymsk', + 'Asia/Taipei', + 'Asia/Tashkent', + 'Asia/Tbilisi', + 'Asia/Tehran', + 'Asia/Tel_Aviv', + 'Asia/Thimbu', + 'Asia/Thimphu', + 'Asia/Tokyo', + 'Asia/Ujung_Pandang', + 'Asia/Ulaanbaatar', + 'Asia/Ulan_Bator', + 'Asia/Urumqi', + 'Asia/Ust-Nera', + 'Asia/Vientiane', + 'Asia/Vladivostok', + 'Asia/Yakutsk', + 'Asia/Yekaterinburg', + 'Asia/Yerevan', + 'Atlantic/Azores', + 'Atlantic/Bermuda', + 'Atlantic/Canary', + 'Atlantic/Cape_Verde', + 'Atlantic/Faeroe', + 'Atlantic/Faroe', + 'Atlantic/Jan_Mayen', + 'Atlantic/Madeira', + 'Atlantic/Reykjavik', + 'Atlantic/South_Georgia', + 'Atlantic/St_Helena', + 'Atlantic/Stanley', + 'Australia/ACT', + 'Australia/Adelaide', + 'Australia/Brisbane', + 'Australia/Broken_Hill', + 'Australia/Canberra', + 'Australia/Currie', + 'Australia/Darwin', + 'Australia/Eucla', + 'Australia/Hobart', + 'Australia/LHI', + 'Australia/Lindeman', + 'Australia/Lord_Howe', + 'Australia/Melbourne', + 'Australia/NSW', + 'Australia/North', + 'Australia/Perth', + 'Australia/Queensland', + 'Australia/South', + 'Australia/Sydney', + 'Australia/Tasmania', + 'Australia/Victoria', + 'Australia/West', + 'Australia/Yancowinna', + 'Brazil/Acre', + 'Brazil/DeNoronha', + 'Brazil/East', + 'Brazil/West', + 'CET', + 'CST6CDT', + 'Canada/Atlantic', + 'Canada/Central', + 'Canada/East-Saskatchewan', + 'Canada/Eastern', + 'Canada/Mountain', + 'Canada/Newfoundland', + 'Canada/Pacific', + 'Canada/Saskatchewan', + 'Canada/Yukon', + 'Chile/Continental', + 'Chile/EasterIsland', + 'Cuba', + 'EET', + 'EST', + 'EST5EDT', + 'Egypt', + 'Eire', + 'Etc/GMT', + 'Etc/GMT+0', + 'Etc/GMT+1', + 'Etc/GMT+10', + 'Etc/GMT+11', + 'Etc/GMT+12', + 'Etc/GMT+2', + 'Etc/GMT+3', + 'Etc/GMT+4', + 'Etc/GMT+5', + 'Etc/GMT+6', + 'Etc/GMT+7', + 'Etc/GMT+8', + 'Etc/GMT+9', + 'Etc/GMT-0', + 'Etc/GMT-1', + 'Etc/GMT-10', + 'Etc/GMT-11', + 'Etc/GMT-12', + 'Etc/GMT-13', + 'Etc/GMT-14', + 'Etc/GMT-2', + 'Etc/GMT-3', + 'Etc/GMT-4', + 'Etc/GMT-5', + 'Etc/GMT-6', + 'Etc/GMT-7', + 'Etc/GMT-8', + 'Etc/GMT-9', + 'Etc/GMT0', + 'Etc/Greenwich', + 'Etc/UCT', + 'Etc/UTC', + 'Etc/Universal', + 'Etc/Zulu', + 'Europe/Amsterdam', + 'Europe/Andorra', + 'Europe/Athens', + 'Europe/Belfast', + 'Europe/Belgrade', + 'Europe/Berlin', + 'Europe/Bratislava', + 'Europe/Brussels', + 'Europe/Bucharest', + 'Europe/Budapest', + 'Europe/Busingen', + 'Europe/Chisinau', + 'Europe/Copenhagen', + 'Europe/Dublin', + 'Europe/Gibraltar', + 'Europe/Guernsey', + 'Europe/Helsinki', + 'Europe/Isle_of_Man', + 'Europe/Istanbul', + 'Europe/Jersey', + 'Europe/Kaliningrad', + 'Europe/Kiev', + 'Europe/Lisbon', + 'Europe/Ljubljana', + 'Europe/London', + 'Europe/Luxembourg', + 'Europe/Madrid', + 'Europe/Malta', + 'Europe/Mariehamn', + 'Europe/Minsk', + 'Europe/Monaco', + 'Europe/Moscow', + 'Europe/Nicosia', + 'Europe/Oslo', + 'Europe/Paris', + 'Europe/Podgorica', + 'Europe/Prague', + 'Europe/Riga', + 'Europe/Rome', + 'Europe/Samara', + 'Europe/San_Marino', + 'Europe/Sarajevo', + 'Europe/Simferopol', + 'Europe/Skopje', + 'Europe/Sofia', + 'Europe/Stockholm', + 'Europe/Tallinn', + 'Europe/Tirane', + 'Europe/Tiraspol', + 'Europe/Uzhgorod', + 'Europe/Vaduz', + 'Europe/Vatican', + 'Europe/Vienna', + 'Europe/Vilnius', + 'Europe/Volgograd', + 'Europe/Warsaw', + 'Europe/Zagreb', + 'Europe/Zaporozhye', + 'Europe/Zurich', + 'GB', + 'GB-Eire', + 'GMT', + 'GMT+0', + 'GMT-0', + 'GMT0', + 'Greenwich', + 'HST', + 'Hongkong', + 'Iceland', + 'Indian/Antananarivo', + 'Indian/Chagos', + 'Indian/Christmas', + 'Indian/Cocos', + 'Indian/Comoro', + 'Indian/Kerguelen', + 'Indian/Mahe', + 'Indian/Maldives', + 'Indian/Mauritius', + 'Indian/Mayotte', + 'Indian/Reunion', + 'Iran', + 'Israel', + 'Jamaica', + 'Japan', + 'Kwajalein', + 'Libya', + 'MET', + 'MST', + 'MST7MDT', + 'Mexico/BajaNorte', + 'Mexico/BajaSur', + 'Mexico/General', + 'NZ', + 'NZ-CHAT', + 'Navajo', + 'PRC', + 'PST8PDT', + 'Pacific/Apia', + 'Pacific/Auckland', + 'Pacific/Chatham', + 'Pacific/Chuuk', + 'Pacific/Easter', + 'Pacific/Efate', + 'Pacific/Enderbury', + 'Pacific/Fakaofo', + 'Pacific/Fiji', + 'Pacific/Funafuti', + 'Pacific/Galapagos', + 'Pacific/Gambier', + 'Pacific/Guadalcanal', + 'Pacific/Guam', + 'Pacific/Honolulu', + 'Pacific/Johnston', + 'Pacific/Kiritimati', + 'Pacific/Kosrae', + 'Pacific/Kwajalein', + 'Pacific/Majuro', + 'Pacific/Marquesas', + 'Pacific/Midway', + 'Pacific/Nauru', + 'Pacific/Niue', + 'Pacific/Norfolk', + 'Pacific/Noumea', + 'Pacific/Pago_Pago', + 'Pacific/Palau', + 'Pacific/Pitcairn', + 'Pacific/Pohnpei', + 'Pacific/Ponape', + 'Pacific/Port_Moresby', + 'Pacific/Rarotonga', + 'Pacific/Saipan', + 'Pacific/Samoa', + 'Pacific/Tahiti', + 'Pacific/Tarawa', + 'Pacific/Tongatapu', + 'Pacific/Truk', + 'Pacific/Wake', + 'Pacific/Wallis', + 'Pacific/Yap', + 'Poland', + 'Portugal', + 'ROC', + 'ROK', + 'Singapore', + 'Turkey', + 'UCT', + 'US/Alaska', + 'US/Aleutian', + 'US/Arizona', + 'US/Central', + 'US/East-Indiana', + 'US/Eastern', + 'US/Hawaii', + 'US/Indiana-Starke', + 'US/Michigan', + 'US/Mountain', + 'US/Pacific', + 'US/Pacific-New', + 'US/Samoa', + 'UTC', + 'Universal', + 'W-SU', + 'WET', + 'Zulu'] +all_timezones = LazyList( + tz for tz in all_timezones if resource_exists(tz)) + +all_timezones_set = LazySet(all_timezones) +common_timezones = \ +['Africa/Abidjan', + 'Africa/Accra', + 'Africa/Addis_Ababa', + 'Africa/Algiers', + 'Africa/Asmara', + 'Africa/Bamako', + 'Africa/Bangui', + 'Africa/Banjul', + 'Africa/Bissau', + 'Africa/Blantyre', + 'Africa/Brazzaville', + 'Africa/Bujumbura', + 'Africa/Cairo', + 'Africa/Casablanca', + 'Africa/Ceuta', + 'Africa/Conakry', + 'Africa/Dakar', + 'Africa/Dar_es_Salaam', + 'Africa/Djibouti', + 'Africa/Douala', + 'Africa/El_Aaiun', + 'Africa/Freetown', + 'Africa/Gaborone', + 'Africa/Harare', + 'Africa/Johannesburg', + 'Africa/Juba', + 'Africa/Kampala', + 'Africa/Khartoum', + 'Africa/Kigali', + 'Africa/Kinshasa', + 'Africa/Lagos', + 'Africa/Libreville', + 'Africa/Lome', + 'Africa/Luanda', + 'Africa/Lubumbashi', + 'Africa/Lusaka', + 'Africa/Malabo', + 'Africa/Maputo', + 'Africa/Maseru', + 'Africa/Mbabane', + 'Africa/Mogadishu', + 'Africa/Monrovia', + 'Africa/Nairobi', + 'Africa/Ndjamena', + 'Africa/Niamey', + 'Africa/Nouakchott', + 'Africa/Ouagadougou', + 'Africa/Porto-Novo', + 'Africa/Sao_Tome', + 'Africa/Tripoli', + 'Africa/Tunis', + 'Africa/Windhoek', + 'America/Adak', + 'America/Anchorage', + 'America/Anguilla', + 'America/Antigua', + 'America/Araguaina', + 'America/Argentina/Buenos_Aires', + 'America/Argentina/Catamarca', + 'America/Argentina/Cordoba', + 'America/Argentina/Jujuy', + 'America/Argentina/La_Rioja', + 'America/Argentina/Mendoza', + 'America/Argentina/Rio_Gallegos', + 'America/Argentina/Salta', + 'America/Argentina/San_Juan', + 'America/Argentina/San_Luis', + 'America/Argentina/Tucuman', + 'America/Argentina/Ushuaia', + 'America/Aruba', + 'America/Asuncion', + 'America/Atikokan', + 'America/Bahia', + 'America/Bahia_Banderas', + 'America/Barbados', + 'America/Belem', + 'America/Belize', + 'America/Blanc-Sablon', + 'America/Boa_Vista', + 'America/Bogota', + 'America/Boise', + 'America/Cambridge_Bay', + 'America/Campo_Grande', + 'America/Cancun', + 'America/Caracas', + 'America/Cayenne', + 'America/Cayman', + 'America/Chicago', + 'America/Chihuahua', + 'America/Costa_Rica', + 'America/Creston', + 'America/Cuiaba', + 'America/Curacao', + 'America/Danmarkshavn', + 'America/Dawson', + 'America/Dawson_Creek', + 'America/Denver', + 'America/Detroit', + 'America/Dominica', + 'America/Edmonton', + 'America/Eirunepe', + 'America/El_Salvador', + 'America/Fortaleza', + 'America/Glace_Bay', + 'America/Godthab', + 'America/Goose_Bay', + 'America/Grand_Turk', + 'America/Grenada', + 'America/Guadeloupe', + 'America/Guatemala', + 'America/Guayaquil', + 'America/Guyana', + 'America/Halifax', + 'America/Havana', + 'America/Hermosillo', + 'America/Indiana/Indianapolis', + 'America/Indiana/Knox', + 'America/Indiana/Marengo', + 'America/Indiana/Petersburg', + 'America/Indiana/Tell_City', + 'America/Indiana/Vevay', + 'America/Indiana/Vincennes', + 'America/Indiana/Winamac', + 'America/Inuvik', + 'America/Iqaluit', + 'America/Jamaica', + 'America/Juneau', + 'America/Kentucky/Louisville', + 'America/Kentucky/Monticello', + 'America/Kralendijk', + 'America/La_Paz', + 'America/Lima', + 'America/Los_Angeles', + 'America/Lower_Princes', + 'America/Maceio', + 'America/Managua', + 'America/Manaus', + 'America/Marigot', + 'America/Martinique', + 'America/Matamoros', + 'America/Mazatlan', + 'America/Menominee', + 'America/Merida', + 'America/Metlakatla', + 'America/Mexico_City', + 'America/Miquelon', + 'America/Moncton', + 'America/Monterrey', + 'America/Montevideo', + 'America/Montreal', + 'America/Montserrat', + 'America/Nassau', + 'America/New_York', + 'America/Nipigon', + 'America/Nome', + 'America/Noronha', + 'America/North_Dakota/Beulah', + 'America/North_Dakota/Center', + 'America/North_Dakota/New_Salem', + 'America/Ojinaga', + 'America/Panama', + 'America/Pangnirtung', + 'America/Paramaribo', + 'America/Phoenix', + 'America/Port-au-Prince', + 'America/Port_of_Spain', + 'America/Porto_Velho', + 'America/Puerto_Rico', + 'America/Rainy_River', + 'America/Rankin_Inlet', + 'America/Recife', + 'America/Regina', + 'America/Resolute', + 'America/Rio_Branco', + 'America/Santa_Isabel', + 'America/Santarem', + 'America/Santiago', + 'America/Santo_Domingo', + 'America/Sao_Paulo', + 'America/Scoresbysund', + 'America/Sitka', + 'America/St_Barthelemy', + 'America/St_Johns', + 'America/St_Kitts', + 'America/St_Lucia', + 'America/St_Thomas', + 'America/St_Vincent', + 'America/Swift_Current', + 'America/Tegucigalpa', + 'America/Thule', + 'America/Thunder_Bay', + 'America/Tijuana', + 'America/Toronto', + 'America/Tortola', + 'America/Vancouver', + 'America/Whitehorse', + 'America/Winnipeg', + 'America/Yakutat', + 'America/Yellowknife', + 'Antarctica/Casey', + 'Antarctica/Davis', + 'Antarctica/DumontDUrville', + 'Antarctica/Macquarie', + 'Antarctica/Mawson', + 'Antarctica/McMurdo', + 'Antarctica/Palmer', + 'Antarctica/Rothera', + 'Antarctica/Syowa', + 'Antarctica/Troll', + 'Antarctica/Vostok', + 'Arctic/Longyearbyen', + 'Asia/Aden', + 'Asia/Almaty', + 'Asia/Amman', + 'Asia/Anadyr', + 'Asia/Aqtau', + 'Asia/Aqtobe', + 'Asia/Ashgabat', + 'Asia/Baghdad', + 'Asia/Bahrain', + 'Asia/Baku', + 'Asia/Bangkok', + 'Asia/Beirut', + 'Asia/Bishkek', + 'Asia/Brunei', + 'Asia/Chita', + 'Asia/Choibalsan', + 'Asia/Colombo', + 'Asia/Damascus', + 'Asia/Dhaka', + 'Asia/Dili', + 'Asia/Dubai', + 'Asia/Dushanbe', + 'Asia/Gaza', + 'Asia/Hebron', + 'Asia/Ho_Chi_Minh', + 'Asia/Hong_Kong', + 'Asia/Hovd', + 'Asia/Irkutsk', + 'Asia/Jakarta', + 'Asia/Jayapura', + 'Asia/Jerusalem', + 'Asia/Kabul', + 'Asia/Kamchatka', + 'Asia/Karachi', + 'Asia/Kathmandu', + 'Asia/Khandyga', + 'Asia/Kolkata', + 'Asia/Krasnoyarsk', + 'Asia/Kuala_Lumpur', + 'Asia/Kuching', + 'Asia/Kuwait', + 'Asia/Macau', + 'Asia/Magadan', + 'Asia/Makassar', + 'Asia/Manila', + 'Asia/Muscat', + 'Asia/Nicosia', + 'Asia/Novokuznetsk', + 'Asia/Novosibirsk', + 'Asia/Omsk', + 'Asia/Oral', + 'Asia/Phnom_Penh', + 'Asia/Pontianak', + 'Asia/Pyongyang', + 'Asia/Qatar', + 'Asia/Qyzylorda', + 'Asia/Rangoon', + 'Asia/Riyadh', + 'Asia/Sakhalin', + 'Asia/Samarkand', + 'Asia/Seoul', + 'Asia/Shanghai', + 'Asia/Singapore', + 'Asia/Srednekolymsk', + 'Asia/Taipei', + 'Asia/Tashkent', + 'Asia/Tbilisi', + 'Asia/Tehran', + 'Asia/Thimphu', + 'Asia/Tokyo', + 'Asia/Ulaanbaatar', + 'Asia/Urumqi', + 'Asia/Ust-Nera', + 'Asia/Vientiane', + 'Asia/Vladivostok', + 'Asia/Yakutsk', + 'Asia/Yekaterinburg', + 'Asia/Yerevan', + 'Atlantic/Azores', + 'Atlantic/Bermuda', + 'Atlantic/Canary', + 'Atlantic/Cape_Verde', + 'Atlantic/Faroe', + 'Atlantic/Madeira', + 'Atlantic/Reykjavik', + 'Atlantic/South_Georgia', + 'Atlantic/St_Helena', + 'Atlantic/Stanley', + 'Australia/Adelaide', + 'Australia/Brisbane', + 'Australia/Broken_Hill', + 'Australia/Currie', + 'Australia/Darwin', + 'Australia/Eucla', + 'Australia/Hobart', + 'Australia/Lindeman', + 'Australia/Lord_Howe', + 'Australia/Melbourne', + 'Australia/Perth', + 'Australia/Sydney', + 'Canada/Atlantic', + 'Canada/Central', + 'Canada/Eastern', + 'Canada/Mountain', + 'Canada/Newfoundland', + 'Canada/Pacific', + 'Europe/Amsterdam', + 'Europe/Andorra', + 'Europe/Athens', + 'Europe/Belgrade', + 'Europe/Berlin', + 'Europe/Bratislava', + 'Europe/Brussels', + 'Europe/Bucharest', + 'Europe/Budapest', + 'Europe/Busingen', + 'Europe/Chisinau', + 'Europe/Copenhagen', + 'Europe/Dublin', + 'Europe/Gibraltar', + 'Europe/Guernsey', + 'Europe/Helsinki', + 'Europe/Isle_of_Man', + 'Europe/Istanbul', + 'Europe/Jersey', + 'Europe/Kaliningrad', + 'Europe/Kiev', + 'Europe/Lisbon', + 'Europe/Ljubljana', + 'Europe/London', + 'Europe/Luxembourg', + 'Europe/Madrid', + 'Europe/Malta', + 'Europe/Mariehamn', + 'Europe/Minsk', + 'Europe/Monaco', + 'Europe/Moscow', + 'Europe/Oslo', + 'Europe/Paris', + 'Europe/Podgorica', + 'Europe/Prague', + 'Europe/Riga', + 'Europe/Rome', + 'Europe/Samara', + 'Europe/San_Marino', + 'Europe/Sarajevo', + 'Europe/Simferopol', + 'Europe/Skopje', + 'Europe/Sofia', + 'Europe/Stockholm', + 'Europe/Tallinn', + 'Europe/Tirane', + 'Europe/Uzhgorod', + 'Europe/Vaduz', + 'Europe/Vatican', + 'Europe/Vienna', + 'Europe/Vilnius', + 'Europe/Volgograd', + 'Europe/Warsaw', + 'Europe/Zagreb', + 'Europe/Zaporozhye', + 'Europe/Zurich', + 'GMT', + 'Indian/Antananarivo', + 'Indian/Chagos', + 'Indian/Christmas', + 'Indian/Cocos', + 'Indian/Comoro', + 'Indian/Kerguelen', + 'Indian/Mahe', + 'Indian/Maldives', + 'Indian/Mauritius', + 'Indian/Mayotte', + 'Indian/Reunion', + 'Pacific/Apia', + 'Pacific/Auckland', + 'Pacific/Chatham', + 'Pacific/Chuuk', + 'Pacific/Easter', + 'Pacific/Efate', + 'Pacific/Enderbury', + 'Pacific/Fakaofo', + 'Pacific/Fiji', + 'Pacific/Funafuti', + 'Pacific/Galapagos', + 'Pacific/Gambier', + 'Pacific/Guadalcanal', + 'Pacific/Guam', + 'Pacific/Honolulu', + 'Pacific/Johnston', + 'Pacific/Kiritimati', + 'Pacific/Kosrae', + 'Pacific/Kwajalein', + 'Pacific/Majuro', + 'Pacific/Marquesas', + 'Pacific/Midway', + 'Pacific/Nauru', + 'Pacific/Niue', + 'Pacific/Norfolk', + 'Pacific/Noumea', + 'Pacific/Pago_Pago', + 'Pacific/Palau', + 'Pacific/Pitcairn', + 'Pacific/Pohnpei', + 'Pacific/Port_Moresby', + 'Pacific/Rarotonga', + 'Pacific/Saipan', + 'Pacific/Tahiti', + 'Pacific/Tarawa', + 'Pacific/Tongatapu', + 'Pacific/Wake', + 'Pacific/Wallis', + 'US/Alaska', + 'US/Arizona', + 'US/Central', + 'US/Eastern', + 'US/Hawaii', + 'US/Mountain', + 'US/Pacific', + 'UTC'] +common_timezones = LazyList( + tz for tz in common_timezones if tz in all_timezones) + +common_timezones_set = LazySet(common_timezones) diff --git a/lib/pytz/exceptions.py b/lib/pytz/exceptions.py new file mode 100644 index 00000000..0376108e --- /dev/null +++ b/lib/pytz/exceptions.py @@ -0,0 +1,48 @@ +''' +Custom exceptions raised by pytz. +''' + +__all__ = [ + 'UnknownTimeZoneError', 'InvalidTimeError', 'AmbiguousTimeError', + 'NonExistentTimeError', + ] + + +class UnknownTimeZoneError(KeyError): + '''Exception raised when pytz is passed an unknown timezone. + + >>> isinstance(UnknownTimeZoneError(), LookupError) + True + + This class is actually a subclass of KeyError to provide backwards + compatibility with code relying on the undocumented behavior of earlier + pytz releases. + + >>> isinstance(UnknownTimeZoneError(), KeyError) + True + ''' + pass + + +class InvalidTimeError(Exception): + '''Base class for invalid time exceptions.''' + + +class AmbiguousTimeError(InvalidTimeError): + '''Exception raised when attempting to create an ambiguous wallclock time. + + At the end of a DST transition period, a particular wallclock time will + occur twice (once before the clocks are set back, once after). Both + possibilities may be correct, unless further information is supplied. + + See DstTzInfo.normalize() for more info + ''' + + +class NonExistentTimeError(InvalidTimeError): + '''Exception raised when attempting to create a wallclock time that + cannot exist. + + At the start of a DST transition period, the wallclock time jumps forward. + The instants jumped over never occur. + ''' diff --git a/lib/pytz/lazy.py b/lib/pytz/lazy.py new file mode 100644 index 00000000..f7fc597c --- /dev/null +++ b/lib/pytz/lazy.py @@ -0,0 +1,168 @@ +from threading import RLock +try: + from UserDict import DictMixin +except ImportError: + from collections import Mapping as DictMixin + + +# With lazy loading, we might end up with multiple threads triggering +# it at the same time. We need a lock. +_fill_lock = RLock() + + +class LazyDict(DictMixin): + """Dictionary populated on first use.""" + data = None + def __getitem__(self, key): + if self.data is None: + _fill_lock.acquire() + try: + if self.data is None: + self._fill() + finally: + _fill_lock.release() + return self.data[key.upper()] + + def __contains__(self, key): + if self.data is None: + _fill_lock.acquire() + try: + if self.data is None: + self._fill() + finally: + _fill_lock.release() + return key in self.data + + def __iter__(self): + if self.data is None: + _fill_lock.acquire() + try: + if self.data is None: + self._fill() + finally: + _fill_lock.release() + return iter(self.data) + + def __len__(self): + if self.data is None: + _fill_lock.acquire() + try: + if self.data is None: + self._fill() + finally: + _fill_lock.release() + return len(self.data) + + def keys(self): + if self.data is None: + _fill_lock.acquire() + try: + if self.data is None: + self._fill() + finally: + _fill_lock.release() + return self.data.keys() + + +class LazyList(list): + """List populated on first use.""" + + _props = [ + '__str__', '__repr__', '__unicode__', + '__hash__', '__sizeof__', '__cmp__', + '__lt__', '__le__', '__eq__', '__ne__', '__gt__', '__ge__', + 'append', 'count', 'index', 'extend', 'insert', 'pop', 'remove', + 'reverse', 'sort', '__add__', '__radd__', '__iadd__', '__mul__', + '__rmul__', '__imul__', '__contains__', '__len__', '__nonzero__', + '__getitem__', '__setitem__', '__delitem__', '__iter__', + '__reversed__', '__getslice__', '__setslice__', '__delslice__'] + + def __new__(cls, fill_iter=None): + + if fill_iter is None: + return list() + + # We need a new class as we will be dynamically messing with its + # methods. + class LazyList(list): + pass + + fill_iter = [fill_iter] + + def lazy(name): + def _lazy(self, *args, **kw): + _fill_lock.acquire() + try: + if len(fill_iter) > 0: + list.extend(self, fill_iter.pop()) + for method_name in cls._props: + delattr(LazyList, method_name) + finally: + _fill_lock.release() + return getattr(list, name)(self, *args, **kw) + return _lazy + + for name in cls._props: + setattr(LazyList, name, lazy(name)) + + new_list = LazyList() + return new_list + +# Not all versions of Python declare the same magic methods. +# Filter out properties that don't exist in this version of Python +# from the list. +LazyList._props = [prop for prop in LazyList._props if hasattr(list, prop)] + + +class LazySet(set): + """Set populated on first use.""" + + _props = ( + '__str__', '__repr__', '__unicode__', + '__hash__', '__sizeof__', '__cmp__', + '__lt__', '__le__', '__eq__', '__ne__', '__gt__', '__ge__', + '__contains__', '__len__', '__nonzero__', + '__getitem__', '__setitem__', '__delitem__', '__iter__', + '__sub__', '__and__', '__xor__', '__or__', + '__rsub__', '__rand__', '__rxor__', '__ror__', + '__isub__', '__iand__', '__ixor__', '__ior__', + 'add', 'clear', 'copy', 'difference', 'difference_update', + 'discard', 'intersection', 'intersection_update', 'isdisjoint', + 'issubset', 'issuperset', 'pop', 'remove', + 'symmetric_difference', 'symmetric_difference_update', + 'union', 'update') + + def __new__(cls, fill_iter=None): + + if fill_iter is None: + return set() + + class LazySet(set): + pass + + fill_iter = [fill_iter] + + def lazy(name): + def _lazy(self, *args, **kw): + _fill_lock.acquire() + try: + if len(fill_iter) > 0: + for i in fill_iter.pop(): + set.add(self, i) + for method_name in cls._props: + delattr(LazySet, method_name) + finally: + _fill_lock.release() + return getattr(set, name)(self, *args, **kw) + return _lazy + + for name in cls._props: + setattr(LazySet, name, lazy(name)) + + new_set = LazySet() + return new_set + +# Not all versions of Python declare the same magic methods. +# Filter out properties that don't exist in this version of Python +# from the list. +LazySet._props = [prop for prop in LazySet._props if hasattr(set, prop)] diff --git a/lib/pytz/reference.py b/lib/pytz/reference.py new file mode 100644 index 00000000..3dda13e7 --- /dev/null +++ b/lib/pytz/reference.py @@ -0,0 +1,127 @@ +''' +Reference tzinfo implementations from the Python docs. +Used for testing against as they are only correct for the years +1987 to 2006. Do not use these for real code. +''' + +from datetime import tzinfo, timedelta, datetime +from pytz import utc, UTC, HOUR, ZERO + +# A class building tzinfo objects for fixed-offset time zones. +# Note that FixedOffset(0, "UTC") is a different way to build a +# UTC tzinfo object. + +class FixedOffset(tzinfo): + """Fixed offset in minutes east from UTC.""" + + def __init__(self, offset, name): + self.__offset = timedelta(minutes = offset) + self.__name = name + + def utcoffset(self, dt): + return self.__offset + + def tzname(self, dt): + return self.__name + + def dst(self, dt): + return ZERO + +# A class capturing the platform's idea of local time. + +import time as _time + +STDOFFSET = timedelta(seconds = -_time.timezone) +if _time.daylight: + DSTOFFSET = timedelta(seconds = -_time.altzone) +else: + DSTOFFSET = STDOFFSET + +DSTDIFF = DSTOFFSET - STDOFFSET + +class LocalTimezone(tzinfo): + + def utcoffset(self, dt): + if self._isdst(dt): + return DSTOFFSET + else: + return STDOFFSET + + def dst(self, dt): + if self._isdst(dt): + return DSTDIFF + else: + return ZERO + + def tzname(self, dt): + return _time.tzname[self._isdst(dt)] + + def _isdst(self, dt): + tt = (dt.year, dt.month, dt.day, + dt.hour, dt.minute, dt.second, + dt.weekday(), 0, -1) + stamp = _time.mktime(tt) + tt = _time.localtime(stamp) + return tt.tm_isdst > 0 + +Local = LocalTimezone() + +# A complete implementation of current DST rules for major US time zones. + +def first_sunday_on_or_after(dt): + days_to_go = 6 - dt.weekday() + if days_to_go: + dt += timedelta(days_to_go) + return dt + +# In the US, DST starts at 2am (standard time) on the first Sunday in April. +DSTSTART = datetime(1, 4, 1, 2) +# and ends at 2am (DST time; 1am standard time) on the last Sunday of Oct. +# which is the first Sunday on or after Oct 25. +DSTEND = datetime(1, 10, 25, 1) + +class USTimeZone(tzinfo): + + def __init__(self, hours, reprname, stdname, dstname): + self.stdoffset = timedelta(hours=hours) + self.reprname = reprname + self.stdname = stdname + self.dstname = dstname + + def __repr__(self): + return self.reprname + + def tzname(self, dt): + if self.dst(dt): + return self.dstname + else: + return self.stdname + + def utcoffset(self, dt): + return self.stdoffset + self.dst(dt) + + def dst(self, dt): + if dt is None or dt.tzinfo is None: + # An exception may be sensible here, in one or both cases. + # It depends on how you want to treat them. The default + # fromutc() implementation (called by the default astimezone() + # implementation) passes a datetime with dt.tzinfo is self. + return ZERO + assert dt.tzinfo is self + + # Find first Sunday in April & the last in October. + start = first_sunday_on_or_after(DSTSTART.replace(year=dt.year)) + end = first_sunday_on_or_after(DSTEND.replace(year=dt.year)) + + # Can't compare naive to aware objects, so strip the timezone from + # dt first. + if start <= dt.replace(tzinfo=None) < end: + return HOUR + else: + return ZERO + +Eastern = USTimeZone(-5, "Eastern", "EST", "EDT") +Central = USTimeZone(-6, "Central", "CST", "CDT") +Mountain = USTimeZone(-7, "Mountain", "MST", "MDT") +Pacific = USTimeZone(-8, "Pacific", "PST", "PDT") + diff --git a/lib/pytz/tests/test_docs.py b/lib/pytz/tests/test_docs.py new file mode 100644 index 00000000..fb49ec15 --- /dev/null +++ b/lib/pytz/tests/test_docs.py @@ -0,0 +1,34 @@ +# -*- coding: ascii -*- + +from doctest import DocFileSuite +import unittest, os.path, sys + +THIS_DIR = os.path.dirname(__file__) + +README = os.path.join(THIS_DIR, os.pardir, os.pardir, 'README.txt') + + +class DocumentationTestCase(unittest.TestCase): + def test_readme_encoding(self): + '''Confirm the README.txt is pure ASCII.''' + f = open(README, 'rb') + try: + f.read().decode('US-ASCII') + finally: + f.close() + + +def test_suite(): + "For the Z3 test runner" + return unittest.TestSuite(( + DocumentationTestCase('test_readme_encoding'), + DocFileSuite(os.path.join(os.pardir, os.pardir, 'README.txt')))) + + +if __name__ == '__main__': + sys.path.insert(0, os.path.abspath(os.path.join( + THIS_DIR, os.pardir, os.pardir + ))) + unittest.main(defaultTest='test_suite') + + diff --git a/lib/pytz/tests/test_lazy.py b/lib/pytz/tests/test_lazy.py new file mode 100644 index 00000000..3a4afa63 --- /dev/null +++ b/lib/pytz/tests/test_lazy.py @@ -0,0 +1,313 @@ +from operator import * +import os.path +import sys +import unittest +import warnings + + +if __name__ == '__main__': + # Only munge path if invoked as a script. Testrunners should have setup + # the paths already + sys.path.insert(0, os.path.abspath(os.path.join(os.pardir, os.pardir))) + + +from pytz.lazy import LazyList, LazySet + + +class LazyListTestCase(unittest.TestCase): + initial_data = [3,2,1] + + def setUp(self): + self.base = [3, 2, 1] + self.lesser = [2, 1, 0] + self.greater = [4, 3, 2] + + self.lazy = LazyList(iter(list(self.base))) + + def test_unary_ops(self): + unary_ops = [str, repr, len, bool, not_] + try: + unary_ops.append(unicode) + except NameError: + pass # unicode no longer exists in Python 3. + + for op in unary_ops: + self.assertEqual( + op(self.lazy), + op(self.base), str(op)) + + def test_binary_ops(self): + binary_ops = [eq, ge, gt, le, lt, ne, add, concat] + try: + binary_ops.append(cmp) + except NameError: + pass # cmp no longer exists in Python 3. + + for op in binary_ops: + self.assertEqual( + op(self.lazy, self.lazy), + op(self.base, self.base), str(op)) + for other in [self.base, self.lesser, self.greater]: + self.assertEqual( + op(self.lazy, other), + op(self.base, other), '%s %s' % (op, other)) + self.assertEqual( + op(other, self.lazy), + op(other, self.base), '%s %s' % (op, other)) + + # Multiplication + self.assertEqual(self.lazy * 3, self.base * 3) + self.assertEqual(3 * self.lazy, 3 * self.base) + + # Contains + self.assertTrue(2 in self.lazy) + self.assertFalse(42 in self.lazy) + + def test_iadd(self): + self.lazy += [1] + self.base += [1] + self.assertEqual(self.lazy, self.base) + + def test_bool(self): + self.assertTrue(bool(self.lazy)) + self.assertFalse(bool(LazyList())) + self.assertFalse(bool(LazyList(iter([])))) + + def test_hash(self): + self.assertRaises(TypeError, hash, self.lazy) + + def test_isinstance(self): + self.assertTrue(isinstance(self.lazy, list)) + self.assertFalse(isinstance(self.lazy, tuple)) + + def test_callable(self): + try: + callable + except NameError: + return # No longer exists with Python 3. + self.assertFalse(callable(self.lazy)) + + def test_append(self): + self.base.append('extra') + self.lazy.append('extra') + self.assertEqual(self.lazy, self.base) + + def test_count(self): + self.assertEqual(self.lazy.count(2), 1) + + def test_index(self): + self.assertEqual(self.lazy.index(2), 1) + + def test_extend(self): + self.base.extend([6, 7]) + self.lazy.extend([6, 7]) + self.assertEqual(self.lazy, self.base) + + def test_insert(self): + self.base.insert(0, 'ping') + self.lazy.insert(0, 'ping') + self.assertEqual(self.lazy, self.base) + + def test_pop(self): + self.assertEqual(self.lazy.pop(), self.base.pop()) + self.assertEqual(self.lazy, self.base) + + def test_remove(self): + self.base.remove(2) + self.lazy.remove(2) + self.assertEqual(self.lazy, self.base) + + def test_reverse(self): + self.base.reverse() + self.lazy.reverse() + self.assertEqual(self.lazy, self.base) + + def test_reversed(self): + self.assertEqual(list(reversed(self.lazy)), list(reversed(self.base))) + + def test_sort(self): + self.base.sort() + self.assertNotEqual(self.lazy, self.base, 'Test data already sorted') + self.lazy.sort() + self.assertEqual(self.lazy, self.base) + + def test_sorted(self): + self.assertEqual(sorted(self.lazy), sorted(self.base)) + + def test_getitem(self): + for idx in range(-len(self.base), len(self.base)): + self.assertEqual(self.lazy[idx], self.base[idx]) + + def test_setitem(self): + for idx in range(-len(self.base), len(self.base)): + self.base[idx] = idx + 1000 + self.assertNotEqual(self.lazy, self.base) + self.lazy[idx] = idx + 1000 + self.assertEqual(self.lazy, self.base) + + def test_delitem(self): + del self.base[0] + self.assertNotEqual(self.lazy, self.base) + del self.lazy[0] + self.assertEqual(self.lazy, self.base) + + del self.base[-2] + self.assertNotEqual(self.lazy, self.base) + del self.lazy[-2] + self.assertEqual(self.lazy, self.base) + + def test_iter(self): + self.assertEqual(list(iter(self.lazy)), list(iter(self.base))) + + def test_getslice(self): + for i in range(-len(self.base), len(self.base)): + for j in range(-len(self.base), len(self.base)): + for step in [-1, 1]: + self.assertEqual(self.lazy[i:j:step], self.base[i:j:step]) + + def test_setslice(self): + for i in range(-len(self.base), len(self.base)): + for j in range(-len(self.base), len(self.base)): + for step in [-1, 1]: + replacement = range(0, len(self.base[i:j:step])) + self.base[i:j:step] = replacement + self.lazy[i:j:step] = replacement + self.assertEqual(self.lazy, self.base) + + def test_delslice(self): + del self.base[0:1] + del self.lazy[0:1] + self.assertEqual(self.lazy, self.base) + + del self.base[-1:1:-1] + del self.lazy[-1:1:-1] + self.assertEqual(self.lazy, self.base) + + +class LazySetTestCase(unittest.TestCase): + initial_data = set([3,2,1]) + + def setUp(self): + self.base = set([3, 2, 1]) + self.lazy = LazySet(iter(set(self.base))) + + def test_unary_ops(self): + # These ops just need to work. + unary_ops = [str, repr] + try: + unary_ops.append(unicode) + except NameError: + pass # unicode no longer exists in Python 3. + + for op in unary_ops: + op(self.lazy) # These ops just need to work. + + # These ops should return identical values as a real set. + unary_ops = [len, bool, not_] + + for op in unary_ops: + self.assertEqual( + op(self.lazy), + op(self.base), '%s(lazy) == %r' % (op, op(self.lazy))) + + def test_binary_ops(self): + binary_ops = [eq, ge, gt, le, lt, ne, sub, and_, or_, xor] + try: + binary_ops.append(cmp) + except NameError: + pass # cmp no longer exists in Python 3. + + for op in binary_ops: + self.assertEqual( + op(self.lazy, self.lazy), + op(self.base, self.base), str(op)) + self.assertEqual( + op(self.lazy, self.base), + op(self.base, self.base), str(op)) + self.assertEqual( + op(self.base, self.lazy), + op(self.base, self.base), str(op)) + + # Contains + self.assertTrue(2 in self.lazy) + self.assertFalse(42 in self.lazy) + + def test_iops(self): + try: + iops = [isub, iand, ior, ixor] + except NameError: + return # Don't exist in older Python versions. + for op in iops: + # Mutating operators, so make fresh copies. + lazy = LazySet(self.base) + base = self.base.copy() + op(lazy, set([1])) + op(base, set([1])) + self.assertEqual(lazy, base, str(op)) + + def test_bool(self): + self.assertTrue(bool(self.lazy)) + self.assertFalse(bool(LazySet())) + self.assertFalse(bool(LazySet(iter([])))) + + def test_hash(self): + self.assertRaises(TypeError, hash, self.lazy) + + def test_isinstance(self): + self.assertTrue(isinstance(self.lazy, set)) + + def test_callable(self): + try: + callable + except NameError: + return # No longer exists with Python 3. + self.assertFalse(callable(self.lazy)) + + def test_add(self): + self.base.add('extra') + self.lazy.add('extra') + self.assertEqual(self.lazy, self.base) + + def test_copy(self): + self.assertEqual(self.lazy.copy(), self.base) + + def test_method_ops(self): + ops = [ + 'difference', 'intersection', 'isdisjoint', + 'issubset', 'issuperset', 'symmetric_difference', 'union', + 'difference_update', 'intersection_update', + 'symmetric_difference_update', 'update'] + for op in ops: + if not hasattr(set, op): + continue # Not in this version of Python. + # Make a copy, as some of the ops are mutating. + lazy = LazySet(set(self.base)) + base = set(self.base) + self.assertEqual( + getattr(self.lazy, op)(set([1])), + getattr(self.base, op)(set([1])), op) + self.assertEqual(self.lazy, self.base, op) + + def test_discard(self): + self.base.discard(1) + self.assertNotEqual(self.lazy, self.base) + self.lazy.discard(1) + self.assertEqual(self.lazy, self.base) + + def test_pop(self): + self.assertEqual(self.lazy.pop(), self.base.pop()) + self.assertEqual(self.lazy, self.base) + + def test_remove(self): + self.base.remove(2) + self.lazy.remove(2) + self.assertEqual(self.lazy, self.base) + + def test_clear(self): + self.lazy.clear() + self.assertEqual(self.lazy, set()) + + +if __name__ == '__main__': + warnings.simplefilter("error") # Warnings should be fatal in tests. + unittest.main() diff --git a/lib/pytz/tests/test_tzinfo.py b/lib/pytz/tests/test_tzinfo.py new file mode 100644 index 00000000..5a929597 --- /dev/null +++ b/lib/pytz/tests/test_tzinfo.py @@ -0,0 +1,820 @@ +# -*- coding: ascii -*- + +import sys, os, os.path +import unittest, doctest +try: + import cPickle as pickle +except ImportError: + import pickle +from datetime import datetime, time, timedelta, tzinfo +import warnings + +if __name__ == '__main__': + # Only munge path if invoked as a script. Testrunners should have setup + # the paths already + sys.path.insert(0, os.path.abspath(os.path.join(os.pardir, os.pardir))) + +import pytz +from pytz import reference +from pytz.tzfile import _byte_string +from pytz.tzinfo import DstTzInfo, StaticTzInfo + +# I test for expected version to ensure the correct version of pytz is +# actually being tested. +EXPECTED_VERSION='2014.7' +EXPECTED_OLSON_VERSION='2014g' + +fmt = '%Y-%m-%d %H:%M:%S %Z%z' + +NOTIME = timedelta(0) + +# GMT is a tzinfo.StaticTzInfo--the class we primarily want to test--while +# UTC is reference implementation. They both have the same timezone meaning. +UTC = pytz.timezone('UTC') +GMT = pytz.timezone('GMT') +assert isinstance(GMT, StaticTzInfo), 'GMT is no longer a StaticTzInfo' + +def prettydt(dt): + """datetime as a string using a known format. + + We don't use strftime as it doesn't handle years earlier than 1900 + per http://bugs.python.org/issue1777412 + """ + if dt.utcoffset() >= timedelta(0): + offset = '+%s' % (dt.utcoffset(),) + else: + offset = '-%s' % (-1 * dt.utcoffset(),) + return '%04d-%02d-%02d %02d:%02d:%02d %s %s' % ( + dt.year, dt.month, dt.day, + dt.hour, dt.minute, dt.second, + dt.tzname(), offset) + + +try: + unicode +except NameError: + # Python 3.x doesn't have unicode(), making writing code + # for Python 2.3 and Python 3.x a pain. + unicode = str + + +class BasicTest(unittest.TestCase): + + def testVersion(self): + # Ensuring the correct version of pytz has been loaded + self.assertEqual(EXPECTED_VERSION, pytz.__version__, + 'Incorrect pytz version loaded. Import path is stuffed ' + 'or this test needs updating. (Wanted %s, got %s)' + % (EXPECTED_VERSION, pytz.__version__)) + + self.assertEqual(EXPECTED_OLSON_VERSION, pytz.OLSON_VERSION, + 'Incorrect pytz version loaded. Import path is stuffed ' + 'or this test needs updating. (Wanted %s, got %s)' + % (EXPECTED_OLSON_VERSION, pytz.OLSON_VERSION)) + + def testGMT(self): + now = datetime.now(tz=GMT) + self.assertTrue(now.utcoffset() == NOTIME) + self.assertTrue(now.dst() == NOTIME) + self.assertTrue(now.timetuple() == now.utctimetuple()) + self.assertTrue(now==now.replace(tzinfo=UTC)) + + def testReferenceUTC(self): + now = datetime.now(tz=UTC) + self.assertTrue(now.utcoffset() == NOTIME) + self.assertTrue(now.dst() == NOTIME) + self.assertTrue(now.timetuple() == now.utctimetuple()) + + def testUnknownOffsets(self): + # This tzinfo behavior is required to make + # datetime.time.{utcoffset, dst, tzname} work as documented. + + dst_tz = pytz.timezone('US/Eastern') + + # This information is not known when we don't have a date, + # so return None per API. + self.assertTrue(dst_tz.utcoffset(None) is None) + self.assertTrue(dst_tz.dst(None) is None) + # We don't know the abbreviation, but this is still a valid + # tzname per the Python documentation. + self.assertEqual(dst_tz.tzname(None), 'US/Eastern') + + def clearCache(self): + pytz._tzinfo_cache.clear() + + def testUnicodeTimezone(self): + # We need to ensure that cold lookups work for both Unicode + # and traditional strings, and that the desired singleton is + # returned. + self.clearCache() + eastern = pytz.timezone(unicode('US/Eastern')) + self.assertTrue(eastern is pytz.timezone('US/Eastern')) + + self.clearCache() + eastern = pytz.timezone('US/Eastern') + self.assertTrue(eastern is pytz.timezone(unicode('US/Eastern'))) + + +class PicklingTest(unittest.TestCase): + + def _roundtrip_tzinfo(self, tz): + p = pickle.dumps(tz) + unpickled_tz = pickle.loads(p) + self.assertTrue(tz is unpickled_tz, '%s did not roundtrip' % tz.zone) + + def _roundtrip_datetime(self, dt): + # Ensure that the tzinfo attached to a datetime instance + # is identical to the one returned. This is important for + # DST timezones, as some state is stored in the tzinfo. + tz = dt.tzinfo + p = pickle.dumps(dt) + unpickled_dt = pickle.loads(p) + unpickled_tz = unpickled_dt.tzinfo + self.assertTrue(tz is unpickled_tz, '%s did not roundtrip' % tz.zone) + + def testDst(self): + tz = pytz.timezone('Europe/Amsterdam') + dt = datetime(2004, 2, 1, 0, 0, 0) + + for localized_tz in tz._tzinfos.values(): + self._roundtrip_tzinfo(localized_tz) + self._roundtrip_datetime(dt.replace(tzinfo=localized_tz)) + + def testRoundtrip(self): + dt = datetime(2004, 2, 1, 0, 0, 0) + for zone in pytz.all_timezones: + tz = pytz.timezone(zone) + self._roundtrip_tzinfo(tz) + + def testDatabaseFixes(self): + # Hack the pickle to make it refer to a timezone abbreviation + # that does not match anything. The unpickler should be able + # to repair this case + tz = pytz.timezone('Australia/Melbourne') + p = pickle.dumps(tz) + tzname = tz._tzname + hacked_p = p.replace(_byte_string(tzname), _byte_string('???')) + self.assertNotEqual(p, hacked_p) + unpickled_tz = pickle.loads(hacked_p) + self.assertTrue(tz is unpickled_tz) + + # Simulate a database correction. In this case, the incorrect + # data will continue to be used. + p = pickle.dumps(tz) + new_utcoffset = tz._utcoffset.seconds + 42 + + # Python 3 introduced a new pickle protocol where numbers are stored in + # hexadecimal representation. Here we extract the pickle + # representation of the number for the current Python version. + old_pickle_pattern = pickle.dumps(tz._utcoffset.seconds)[3:-1] + new_pickle_pattern = pickle.dumps(new_utcoffset)[3:-1] + hacked_p = p.replace(old_pickle_pattern, new_pickle_pattern) + + self.assertNotEqual(p, hacked_p) + unpickled_tz = pickle.loads(hacked_p) + self.assertEqual(unpickled_tz._utcoffset.seconds, new_utcoffset) + self.assertTrue(tz is not unpickled_tz) + + def testOldPickles(self): + # Ensure that applications serializing pytz instances as pickles + # have no troubles upgrading to a new pytz release. These pickles + # where created with pytz2006j + east1 = pickle.loads(_byte_string( + "cpytz\n_p\np1\n(S'US/Eastern'\np2\nI-18000\n" + "I0\nS'EST'\np3\ntRp4\n." + )) + east2 = pytz.timezone('US/Eastern').localize( + datetime(2006, 1, 1)).tzinfo + self.assertTrue(east1 is east2) + + # Confirm changes in name munging between 2006j and 2007c cause + # no problems. + pap1 = pickle.loads(_byte_string( + "cpytz\n_p\np1\n(S'America/Port_minus_au_minus_Prince'" + "\np2\nI-17340\nI0\nS'PPMT'\np3\ntRp4\n.")) + pap2 = pytz.timezone('America/Port-au-Prince').localize( + datetime(1910, 1, 1)).tzinfo + self.assertTrue(pap1 is pap2) + + gmt1 = pickle.loads(_byte_string( + "cpytz\n_p\np1\n(S'Etc/GMT_plus_10'\np2\ntRp3\n.")) + gmt2 = pytz.timezone('Etc/GMT+10') + self.assertTrue(gmt1 is gmt2) + + +class USEasternDSTStartTestCase(unittest.TestCase): + tzinfo = pytz.timezone('US/Eastern') + + # 24 hours before DST changeover + transition_time = datetime(2002, 4, 7, 7, 0, 0, tzinfo=UTC) + + # Increase for 'flexible' DST transitions due to 1 minute granularity + # of Python's datetime library + instant = timedelta(seconds=1) + + # before transition + before = { + 'tzname': 'EST', + 'utcoffset': timedelta(hours = -5), + 'dst': timedelta(hours = 0), + } + + # after transition + after = { + 'tzname': 'EDT', + 'utcoffset': timedelta(hours = -4), + 'dst': timedelta(hours = 1), + } + + def _test_tzname(self, utc_dt, wanted): + tzname = wanted['tzname'] + dt = utc_dt.astimezone(self.tzinfo) + self.assertEqual(dt.tzname(), tzname, + 'Expected %s as tzname for %s. Got %s' % ( + tzname, str(utc_dt), dt.tzname() + ) + ) + + def _test_utcoffset(self, utc_dt, wanted): + utcoffset = wanted['utcoffset'] + dt = utc_dt.astimezone(self.tzinfo) + self.assertEqual( + dt.utcoffset(), wanted['utcoffset'], + 'Expected %s as utcoffset for %s. Got %s' % ( + utcoffset, utc_dt, dt.utcoffset() + ) + ) + + def _test_dst(self, utc_dt, wanted): + dst = wanted['dst'] + dt = utc_dt.astimezone(self.tzinfo) + self.assertEqual(dt.dst(),dst, + 'Expected %s as dst for %s. Got %s' % ( + dst, utc_dt, dt.dst() + ) + ) + + def test_arithmetic(self): + utc_dt = self.transition_time + + for days in range(-420, 720, 20): + delta = timedelta(days=days) + + # Make sure we can get back where we started + dt = utc_dt.astimezone(self.tzinfo) + dt2 = dt + delta + dt2 = dt2 - delta + self.assertEqual(dt, dt2) + + # Make sure arithmetic crossing DST boundaries ends + # up in the correct timezone after normalization + utc_plus_delta = (utc_dt + delta).astimezone(self.tzinfo) + local_plus_delta = self.tzinfo.normalize(dt + delta) + self.assertEqual( + prettydt(utc_plus_delta), + prettydt(local_plus_delta), + 'Incorrect result for delta==%d days. Wanted %r. Got %r'%( + days, + prettydt(utc_plus_delta), + prettydt(local_plus_delta), + ) + ) + + def _test_all(self, utc_dt, wanted): + self._test_utcoffset(utc_dt, wanted) + self._test_tzname(utc_dt, wanted) + self._test_dst(utc_dt, wanted) + + def testDayBefore(self): + self._test_all( + self.transition_time - timedelta(days=1), self.before + ) + + def testTwoHoursBefore(self): + self._test_all( + self.transition_time - timedelta(hours=2), self.before + ) + + def testHourBefore(self): + self._test_all( + self.transition_time - timedelta(hours=1), self.before + ) + + def testInstantBefore(self): + self._test_all( + self.transition_time - self.instant, self.before + ) + + def testTransition(self): + self._test_all( + self.transition_time, self.after + ) + + def testInstantAfter(self): + self._test_all( + self.transition_time + self.instant, self.after + ) + + def testHourAfter(self): + self._test_all( + self.transition_time + timedelta(hours=1), self.after + ) + + def testTwoHoursAfter(self): + self._test_all( + self.transition_time + timedelta(hours=1), self.after + ) + + def testDayAfter(self): + self._test_all( + self.transition_time + timedelta(days=1), self.after + ) + + +class USEasternDSTEndTestCase(USEasternDSTStartTestCase): + tzinfo = pytz.timezone('US/Eastern') + transition_time = datetime(2002, 10, 27, 6, 0, 0, tzinfo=UTC) + before = { + 'tzname': 'EDT', + 'utcoffset': timedelta(hours = -4), + 'dst': timedelta(hours = 1), + } + after = { + 'tzname': 'EST', + 'utcoffset': timedelta(hours = -5), + 'dst': timedelta(hours = 0), + } + + +class USEasternEPTStartTestCase(USEasternDSTStartTestCase): + transition_time = datetime(1945, 8, 14, 23, 0, 0, tzinfo=UTC) + before = { + 'tzname': 'EWT', + 'utcoffset': timedelta(hours = -4), + 'dst': timedelta(hours = 1), + } + after = { + 'tzname': 'EPT', + 'utcoffset': timedelta(hours = -4), + 'dst': timedelta(hours = 1), + } + + +class USEasternEPTEndTestCase(USEasternDSTStartTestCase): + transition_time = datetime(1945, 9, 30, 6, 0, 0, tzinfo=UTC) + before = { + 'tzname': 'EPT', + 'utcoffset': timedelta(hours = -4), + 'dst': timedelta(hours = 1), + } + after = { + 'tzname': 'EST', + 'utcoffset': timedelta(hours = -5), + 'dst': timedelta(hours = 0), + } + + +class WarsawWMTEndTestCase(USEasternDSTStartTestCase): + # In 1915, Warsaw changed from Warsaw to Central European time. + # This involved the clocks being set backwards, causing a end-of-DST + # like situation without DST being involved. + tzinfo = pytz.timezone('Europe/Warsaw') + transition_time = datetime(1915, 8, 4, 22, 36, 0, tzinfo=UTC) + before = { + 'tzname': 'WMT', + 'utcoffset': timedelta(hours=1, minutes=24), + 'dst': timedelta(0), + } + after = { + 'tzname': 'CET', + 'utcoffset': timedelta(hours=1), + 'dst': timedelta(0), + } + + +class VilniusWMTEndTestCase(USEasternDSTStartTestCase): + # At the end of 1916, Vilnius changed timezones putting its clock + # forward by 11 minutes 35 seconds. Neither timezone was in DST mode. + tzinfo = pytz.timezone('Europe/Vilnius') + instant = timedelta(seconds=31) + transition_time = datetime(1916, 12, 31, 22, 36, 00, tzinfo=UTC) + before = { + 'tzname': 'WMT', + 'utcoffset': timedelta(hours=1, minutes=24), + 'dst': timedelta(0), + } + after = { + 'tzname': 'KMT', + 'utcoffset': timedelta(hours=1, minutes=36), # Really 1:35:36 + 'dst': timedelta(0), + } + + +class VilniusCESTStartTestCase(USEasternDSTStartTestCase): + # In 1941, Vilnius changed from MSG to CEST, switching to summer + # time while simultaneously reducing its UTC offset by two hours, + # causing the clocks to go backwards for this summer time + # switchover. + tzinfo = pytz.timezone('Europe/Vilnius') + transition_time = datetime(1941, 6, 23, 21, 00, 00, tzinfo=UTC) + before = { + 'tzname': 'MSK', + 'utcoffset': timedelta(hours=3), + 'dst': timedelta(0), + } + after = { + 'tzname': 'CEST', + 'utcoffset': timedelta(hours=2), + 'dst': timedelta(hours=1), + } + + +class LondonHistoryStartTestCase(USEasternDSTStartTestCase): + # The first known timezone transition in London was in 1847 when + # clocks where synchronized to GMT. However, we currently only + # understand v1 format tzfile(5) files which does handle years + # this far in the past, so our earliest known transition is in + # 1916. + tzinfo = pytz.timezone('Europe/London') + # transition_time = datetime(1847, 12, 1, 1, 15, 00, tzinfo=UTC) + # before = { + # 'tzname': 'LMT', + # 'utcoffset': timedelta(minutes=-75), + # 'dst': timedelta(0), + # } + # after = { + # 'tzname': 'GMT', + # 'utcoffset': timedelta(0), + # 'dst': timedelta(0), + # } + transition_time = datetime(1916, 5, 21, 2, 00, 00, tzinfo=UTC) + before = { + 'tzname': 'GMT', + 'utcoffset': timedelta(0), + 'dst': timedelta(0), + } + after = { + 'tzname': 'BST', + 'utcoffset': timedelta(hours=1), + 'dst': timedelta(hours=1), + } + + +class LondonHistoryEndTestCase(USEasternDSTStartTestCase): + # Timezone switchovers are projected into the future, even + # though no official statements exist or could be believed even + # if they did exist. We currently only check the last known + # transition in 2037, as we are still using v1 format tzfile(5) + # files. + tzinfo = pytz.timezone('Europe/London') + # transition_time = datetime(2499, 10, 25, 1, 0, 0, tzinfo=UTC) + transition_time = datetime(2037, 10, 25, 1, 0, 0, tzinfo=UTC) + before = { + 'tzname': 'BST', + 'utcoffset': timedelta(hours=1), + 'dst': timedelta(hours=1), + } + after = { + 'tzname': 'GMT', + 'utcoffset': timedelta(0), + 'dst': timedelta(0), + } + + +class NoumeaHistoryStartTestCase(USEasternDSTStartTestCase): + # Noumea adopted a whole hour offset in 1912. Previously + # it was 11 hours, 5 minutes and 48 seconds off UTC. However, + # due to limitations of the Python datetime library, we need + # to round that to 11 hours 6 minutes. + tzinfo = pytz.timezone('Pacific/Noumea') + transition_time = datetime(1912, 1, 12, 12, 54, 12, tzinfo=UTC) + before = { + 'tzname': 'LMT', + 'utcoffset': timedelta(hours=11, minutes=6), + 'dst': timedelta(0), + } + after = { + 'tzname': 'NCT', + 'utcoffset': timedelta(hours=11), + 'dst': timedelta(0), + } + + +class NoumeaDSTEndTestCase(USEasternDSTStartTestCase): + # Noumea dropped DST in 1997. + tzinfo = pytz.timezone('Pacific/Noumea') + transition_time = datetime(1997, 3, 1, 15, 00, 00, tzinfo=UTC) + before = { + 'tzname': 'NCST', + 'utcoffset': timedelta(hours=12), + 'dst': timedelta(hours=1), + } + after = { + 'tzname': 'NCT', + 'utcoffset': timedelta(hours=11), + 'dst': timedelta(0), + } + + +class NoumeaNoMoreDSTTestCase(NoumeaDSTEndTestCase): + # Noumea dropped DST in 1997. Here we test that it stops occuring. + transition_time = ( + NoumeaDSTEndTestCase.transition_time + timedelta(days=365*10)) + before = NoumeaDSTEndTestCase.after + after = NoumeaDSTEndTestCase.after + + +class TahitiTestCase(USEasternDSTStartTestCase): + # Tahiti has had a single transition in its history. + tzinfo = pytz.timezone('Pacific/Tahiti') + transition_time = datetime(1912, 10, 1, 9, 58, 16, tzinfo=UTC) + before = { + 'tzname': 'LMT', + 'utcoffset': timedelta(hours=-9, minutes=-58), + 'dst': timedelta(0), + } + after = { + 'tzname': 'TAHT', + 'utcoffset': timedelta(hours=-10), + 'dst': timedelta(0), + } + + +class SamoaInternationalDateLineChange(USEasternDSTStartTestCase): + # At the end of 2011, Samoa will switch from being east of the + # international dateline to the west. There will be no Dec 30th + # 2011 and it will switch from UTC-10 to UTC+14. + tzinfo = pytz.timezone('Pacific/Apia') + transition_time = datetime(2011, 12, 30, 10, 0, 0, tzinfo=UTC) + before = { + 'tzname': 'SDT', + 'utcoffset': timedelta(hours=-10), + 'dst': timedelta(hours=1), + } + after = { + 'tzname': 'WSDT', + 'utcoffset': timedelta(hours=14), + 'dst': timedelta(hours=1), + } + + +class ReferenceUSEasternDSTStartTestCase(USEasternDSTStartTestCase): + tzinfo = reference.Eastern + def test_arithmetic(self): + # Reference implementation cannot handle this + pass + + +class ReferenceUSEasternDSTEndTestCase(USEasternDSTEndTestCase): + tzinfo = reference.Eastern + + def testHourBefore(self): + # Python's datetime library has a bug, where the hour before + # a daylight saving transition is one hour out. For example, + # at the end of US/Eastern daylight saving time, 01:00 EST + # occurs twice (once at 05:00 UTC and once at 06:00 UTC), + # whereas the first should actually be 01:00 EDT. + # Note that this bug is by design - by accepting this ambiguity + # for one hour one hour per year, an is_dst flag on datetime.time + # became unnecessary. + self._test_all( + self.transition_time - timedelta(hours=1), self.after + ) + + def testInstantBefore(self): + self._test_all( + self.transition_time - timedelta(seconds=1), self.after + ) + + def test_arithmetic(self): + # Reference implementation cannot handle this + pass + + +class LocalTestCase(unittest.TestCase): + def testLocalize(self): + loc_tz = pytz.timezone('Europe/Amsterdam') + + loc_time = loc_tz.localize(datetime(1930, 5, 10, 0, 0, 0)) + # Actually +00:19:32, but Python datetime rounds this + self.assertEqual(loc_time.strftime('%Z%z'), 'AMT+0020') + + loc_time = loc_tz.localize(datetime(1930, 5, 20, 0, 0, 0)) + # Actually +00:19:32, but Python datetime rounds this + self.assertEqual(loc_time.strftime('%Z%z'), 'NST+0120') + + loc_time = loc_tz.localize(datetime(1940, 5, 10, 0, 0, 0)) + self.assertEqual(loc_time.strftime('%Z%z'), 'NET+0020') + + loc_time = loc_tz.localize(datetime(1940, 5, 20, 0, 0, 0)) + self.assertEqual(loc_time.strftime('%Z%z'), 'CEST+0200') + + loc_time = loc_tz.localize(datetime(2004, 2, 1, 0, 0, 0)) + self.assertEqual(loc_time.strftime('%Z%z'), 'CET+0100') + + loc_time = loc_tz.localize(datetime(2004, 4, 1, 0, 0, 0)) + self.assertEqual(loc_time.strftime('%Z%z'), 'CEST+0200') + + tz = pytz.timezone('Europe/Amsterdam') + loc_time = loc_tz.localize(datetime(1943, 3, 29, 1, 59, 59)) + self.assertEqual(loc_time.strftime('%Z%z'), 'CET+0100') + + + # Switch to US + loc_tz = pytz.timezone('US/Eastern') + + # End of DST ambiguity check + loc_time = loc_tz.localize(datetime(1918, 10, 27, 1, 59, 59), is_dst=1) + self.assertEqual(loc_time.strftime('%Z%z'), 'EDT-0400') + + loc_time = loc_tz.localize(datetime(1918, 10, 27, 1, 59, 59), is_dst=0) + self.assertEqual(loc_time.strftime('%Z%z'), 'EST-0500') + + self.assertRaises(pytz.AmbiguousTimeError, + loc_tz.localize, datetime(1918, 10, 27, 1, 59, 59), is_dst=None + ) + + # Start of DST non-existent times + loc_time = loc_tz.localize(datetime(1918, 3, 31, 2, 0, 0), is_dst=0) + self.assertEqual(loc_time.strftime('%Z%z'), 'EST-0500') + + loc_time = loc_tz.localize(datetime(1918, 3, 31, 2, 0, 0), is_dst=1) + self.assertEqual(loc_time.strftime('%Z%z'), 'EDT-0400') + + self.assertRaises(pytz.NonExistentTimeError, + loc_tz.localize, datetime(1918, 3, 31, 2, 0, 0), is_dst=None + ) + + # Weird changes - war time and peace time both is_dst==True + + loc_time = loc_tz.localize(datetime(1942, 2, 9, 3, 0, 0)) + self.assertEqual(loc_time.strftime('%Z%z'), 'EWT-0400') + + loc_time = loc_tz.localize(datetime(1945, 8, 14, 19, 0, 0)) + self.assertEqual(loc_time.strftime('%Z%z'), 'EPT-0400') + + loc_time = loc_tz.localize(datetime(1945, 9, 30, 1, 0, 0), is_dst=1) + self.assertEqual(loc_time.strftime('%Z%z'), 'EPT-0400') + + loc_time = loc_tz.localize(datetime(1945, 9, 30, 1, 0, 0), is_dst=0) + self.assertEqual(loc_time.strftime('%Z%z'), 'EST-0500') + + def testNormalize(self): + tz = pytz.timezone('US/Eastern') + dt = datetime(2004, 4, 4, 7, 0, 0, tzinfo=UTC).astimezone(tz) + dt2 = dt - timedelta(minutes=10) + self.assertEqual( + dt2.strftime('%Y-%m-%d %H:%M:%S %Z%z'), + '2004-04-04 02:50:00 EDT-0400' + ) + + dt2 = tz.normalize(dt2) + self.assertEqual( + dt2.strftime('%Y-%m-%d %H:%M:%S %Z%z'), + '2004-04-04 01:50:00 EST-0500' + ) + + def testPartialMinuteOffsets(self): + # utcoffset in Amsterdam was not a whole minute until 1937 + # However, we fudge this by rounding them, as the Python + # datetime library + tz = pytz.timezone('Europe/Amsterdam') + utc_dt = datetime(1914, 1, 1, 13, 40, 28, tzinfo=UTC) # correct + utc_dt = utc_dt.replace(second=0) # But we need to fudge it + loc_dt = utc_dt.astimezone(tz) + self.assertEqual( + loc_dt.strftime('%Y-%m-%d %H:%M:%S %Z%z'), + '1914-01-01 14:00:00 AMT+0020' + ) + + # And get back... + utc_dt = loc_dt.astimezone(UTC) + self.assertEqual( + utc_dt.strftime('%Y-%m-%d %H:%M:%S %Z%z'), + '1914-01-01 13:40:00 UTC+0000' + ) + + def no_testCreateLocaltime(self): + # It would be nice if this worked, but it doesn't. + tz = pytz.timezone('Europe/Amsterdam') + dt = datetime(2004, 10, 31, 2, 0, 0, tzinfo=tz) + self.assertEqual( + dt.strftime(fmt), + '2004-10-31 02:00:00 CET+0100' + ) + + +class CommonTimezonesTestCase(unittest.TestCase): + def test_bratislava(self): + # Bratislava is the default timezone for Slovakia, but our + # heuristics where not adding it to common_timezones. Ideally, + # common_timezones should be populated from zone.tab at runtime, + # but I'm hesitant to pay the startup cost as loading the list + # on demand whilst remaining backwards compatible seems + # difficult. + self.assertTrue('Europe/Bratislava' in pytz.common_timezones) + self.assertTrue('Europe/Bratislava' in pytz.common_timezones_set) + + def test_us_eastern(self): + self.assertTrue('US/Eastern' in pytz.common_timezones) + self.assertTrue('US/Eastern' in pytz.common_timezones_set) + + def test_belfast(self): + # Belfast uses London time. + self.assertTrue('Europe/Belfast' in pytz.all_timezones_set) + self.assertFalse('Europe/Belfast' in pytz.common_timezones) + self.assertFalse('Europe/Belfast' in pytz.common_timezones_set) + + +class BaseTzInfoTestCase: + '''Ensure UTC, StaticTzInfo and DstTzInfo work consistently. + + These tests are run for each type of tzinfo. + ''' + tz = None # override + tz_class = None # override + + def test_expectedclass(self): + self.assertTrue(isinstance(self.tz, self.tz_class)) + + def test_fromutc(self): + # naive datetime. + dt1 = datetime(2011, 10, 31) + + # localized datetime, same timezone. + dt2 = self.tz.localize(dt1) + + # Both should give the same results. Note that the standard + # Python tzinfo.fromutc() only supports the second. + for dt in [dt1, dt2]: + loc_dt = self.tz.fromutc(dt) + loc_dt2 = pytz.utc.localize(dt1).astimezone(self.tz) + self.assertEqual(loc_dt, loc_dt2) + + # localized datetime, different timezone. + new_tz = pytz.timezone('Europe/Paris') + self.assertTrue(self.tz is not new_tz) + dt3 = new_tz.localize(dt1) + self.assertRaises(ValueError, self.tz.fromutc, dt3) + + def test_normalize(self): + other_tz = pytz.timezone('Europe/Paris') + self.assertTrue(self.tz is not other_tz) + + dt = datetime(2012, 3, 26, 12, 0) + other_dt = other_tz.localize(dt) + + local_dt = self.tz.normalize(other_dt) + + self.assertTrue(local_dt.tzinfo is not other_dt.tzinfo) + self.assertNotEqual( + local_dt.replace(tzinfo=None), other_dt.replace(tzinfo=None)) + + def test_astimezone(self): + other_tz = pytz.timezone('Europe/Paris') + self.assertTrue(self.tz is not other_tz) + + dt = datetime(2012, 3, 26, 12, 0) + other_dt = other_tz.localize(dt) + + local_dt = other_dt.astimezone(self.tz) + + self.assertTrue(local_dt.tzinfo is not other_dt.tzinfo) + self.assertNotEqual( + local_dt.replace(tzinfo=None), other_dt.replace(tzinfo=None)) + + +class OptimizedUTCTestCase(unittest.TestCase, BaseTzInfoTestCase): + tz = pytz.utc + tz_class = tz.__class__ + + +class LegacyUTCTestCase(unittest.TestCase, BaseTzInfoTestCase): + # Deprecated timezone, but useful for comparison tests. + tz = pytz.timezone('Etc/UTC') + tz_class = StaticTzInfo + + +class StaticTzInfoTestCase(unittest.TestCase, BaseTzInfoTestCase): + tz = pytz.timezone('GMT') + tz_class = StaticTzInfo + + +class DstTzInfoTestCase(unittest.TestCase, BaseTzInfoTestCase): + tz = pytz.timezone('Australia/Melbourne') + tz_class = DstTzInfo + + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(doctest.DocTestSuite('pytz')) + suite.addTest(doctest.DocTestSuite('pytz.tzinfo')) + import test_tzinfo + suite.addTest(unittest.defaultTestLoader.loadTestsFromModule(test_tzinfo)) + return suite + + +if __name__ == '__main__': + warnings.simplefilter("error") # Warnings should be fatal in tests. + unittest.main(defaultTest='test_suite') + diff --git a/lib/pytz/tzfile.py b/lib/pytz/tzfile.py new file mode 100644 index 00000000..9c007c80 --- /dev/null +++ b/lib/pytz/tzfile.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +''' +$Id: tzfile.py,v 1.8 2004/06/03 00:15:24 zenzen Exp $ +''' + +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO +from datetime import datetime, timedelta +from struct import unpack, calcsize + +from pytz.tzinfo import StaticTzInfo, DstTzInfo, memorized_ttinfo +from pytz.tzinfo import memorized_datetime, memorized_timedelta + +def _byte_string(s): + """Cast a string or byte string to an ASCII byte string.""" + return s.encode('US-ASCII') + +_NULL = _byte_string('\0') + +def _std_string(s): + """Cast a string or byte string to an ASCII string.""" + return str(s.decode('US-ASCII')) + +def build_tzinfo(zone, fp): + head_fmt = '>4s c 15x 6l' + head_size = calcsize(head_fmt) + (magic, format, ttisgmtcnt, ttisstdcnt,leapcnt, timecnt, + typecnt, charcnt) = unpack(head_fmt, fp.read(head_size)) + + # Make sure it is a tzfile(5) file + assert magic == _byte_string('TZif'), 'Got magic %s' % repr(magic) + + # Read out the transition times, localtime indices and ttinfo structures. + data_fmt = '>%(timecnt)dl %(timecnt)dB %(ttinfo)s %(charcnt)ds' % dict( + timecnt=timecnt, ttinfo='lBB'*typecnt, charcnt=charcnt) + data_size = calcsize(data_fmt) + data = unpack(data_fmt, fp.read(data_size)) + + # make sure we unpacked the right number of values + assert len(data) == 2 * timecnt + 3 * typecnt + 1 + transitions = [memorized_datetime(trans) + for trans in data[:timecnt]] + lindexes = list(data[timecnt:2 * timecnt]) + ttinfo_raw = data[2 * timecnt:-1] + tznames_raw = data[-1] + del data + + # Process ttinfo into separate structs + ttinfo = [] + tznames = {} + i = 0 + while i < len(ttinfo_raw): + # have we looked up this timezone name yet? + tzname_offset = ttinfo_raw[i+2] + if tzname_offset not in tznames: + nul = tznames_raw.find(_NULL, tzname_offset) + if nul < 0: + nul = len(tznames_raw) + tznames[tzname_offset] = _std_string( + tznames_raw[tzname_offset:nul]) + ttinfo.append((ttinfo_raw[i], + bool(ttinfo_raw[i+1]), + tznames[tzname_offset])) + i += 3 + + # Now build the timezone object + if len(transitions) == 0: + ttinfo[0][0], ttinfo[0][2] + cls = type(zone, (StaticTzInfo,), dict( + zone=zone, + _utcoffset=memorized_timedelta(ttinfo[0][0]), + _tzname=ttinfo[0][2])) + else: + # Early dates use the first standard time ttinfo + i = 0 + while ttinfo[i][1]: + i += 1 + if ttinfo[i] == ttinfo[lindexes[0]]: + transitions[0] = datetime.min + else: + transitions.insert(0, datetime.min) + lindexes.insert(0, i) + + # calculate transition info + transition_info = [] + for i in range(len(transitions)): + inf = ttinfo[lindexes[i]] + utcoffset = inf[0] + if not inf[1]: + dst = 0 + else: + for j in range(i-1, -1, -1): + prev_inf = ttinfo[lindexes[j]] + if not prev_inf[1]: + break + dst = inf[0] - prev_inf[0] # dst offset + + # Bad dst? Look further. DST > 24 hours happens when + # a timzone has moved across the international dateline. + if dst <= 0 or dst > 3600*3: + for j in range(i+1, len(transitions)): + stdinf = ttinfo[lindexes[j]] + if not stdinf[1]: + dst = inf[0] - stdinf[0] + if dst > 0: + break # Found a useful std time. + + tzname = inf[2] + + # Round utcoffset and dst to the nearest minute or the + # datetime library will complain. Conversions to these timezones + # might be up to plus or minus 30 seconds out, but it is + # the best we can do. + utcoffset = int((utcoffset + 30) // 60) * 60 + dst = int((dst + 30) // 60) * 60 + transition_info.append(memorized_ttinfo(utcoffset, dst, tzname)) + + cls = type(zone, (DstTzInfo,), dict( + zone=zone, + _utc_transition_times=transitions, + _transition_info=transition_info)) + + return cls() + +if __name__ == '__main__': + import os.path + from pprint import pprint + base = os.path.join(os.path.dirname(__file__), 'zoneinfo') + tz = build_tzinfo('Australia/Melbourne', + open(os.path.join(base,'Australia','Melbourne'), 'rb')) + tz = build_tzinfo('US/Eastern', + open(os.path.join(base,'US','Eastern'), 'rb')) + pprint(tz._utc_transition_times) + #print tz.asPython(4) + #print tz.transitions_mapping diff --git a/lib/pytz/tzinfo.py b/lib/pytz/tzinfo.py new file mode 100644 index 00000000..d53e9ff1 --- /dev/null +++ b/lib/pytz/tzinfo.py @@ -0,0 +1,563 @@ +'''Base classes and helpers for building zone specific tzinfo classes''' + +from datetime import datetime, timedelta, tzinfo +from bisect import bisect_right +try: + set +except NameError: + from sets import Set as set + +import pytz +from pytz.exceptions import AmbiguousTimeError, NonExistentTimeError + +__all__ = [] + +_timedelta_cache = {} +def memorized_timedelta(seconds): + '''Create only one instance of each distinct timedelta''' + try: + return _timedelta_cache[seconds] + except KeyError: + delta = timedelta(seconds=seconds) + _timedelta_cache[seconds] = delta + return delta + +_epoch = datetime.utcfromtimestamp(0) +_datetime_cache = {0: _epoch} +def memorized_datetime(seconds): + '''Create only one instance of each distinct datetime''' + try: + return _datetime_cache[seconds] + except KeyError: + # NB. We can't just do datetime.utcfromtimestamp(seconds) as this + # fails with negative values under Windows (Bug #90096) + dt = _epoch + timedelta(seconds=seconds) + _datetime_cache[seconds] = dt + return dt + +_ttinfo_cache = {} +def memorized_ttinfo(*args): + '''Create only one instance of each distinct tuple''' + try: + return _ttinfo_cache[args] + except KeyError: + ttinfo = ( + memorized_timedelta(args[0]), + memorized_timedelta(args[1]), + args[2] + ) + _ttinfo_cache[args] = ttinfo + return ttinfo + +_notime = memorized_timedelta(0) + +def _to_seconds(td): + '''Convert a timedelta to seconds''' + return td.seconds + td.days * 24 * 60 * 60 + + +class BaseTzInfo(tzinfo): + # Overridden in subclass + _utcoffset = None + _tzname = None + zone = None + + def __str__(self): + return self.zone + + +class StaticTzInfo(BaseTzInfo): + '''A timezone that has a constant offset from UTC + + These timezones are rare, as most locations have changed their + offset at some point in their history + ''' + def fromutc(self, dt): + '''See datetime.tzinfo.fromutc''' + if dt.tzinfo is not None and dt.tzinfo is not self: + raise ValueError('fromutc: dt.tzinfo is not self') + return (dt + self._utcoffset).replace(tzinfo=self) + + def utcoffset(self, dt, is_dst=None): + '''See datetime.tzinfo.utcoffset + + is_dst is ignored for StaticTzInfo, and exists only to + retain compatibility with DstTzInfo. + ''' + return self._utcoffset + + def dst(self, dt, is_dst=None): + '''See datetime.tzinfo.dst + + is_dst is ignored for StaticTzInfo, and exists only to + retain compatibility with DstTzInfo. + ''' + return _notime + + def tzname(self, dt, is_dst=None): + '''See datetime.tzinfo.tzname + + is_dst is ignored for StaticTzInfo, and exists only to + retain compatibility with DstTzInfo. + ''' + return self._tzname + + def localize(self, dt, is_dst=False): + '''Convert naive time to local time''' + if dt.tzinfo is not None: + raise ValueError('Not naive datetime (tzinfo is already set)') + return dt.replace(tzinfo=self) + + def normalize(self, dt, is_dst=False): + '''Correct the timezone information on the given datetime. + + This is normally a no-op, as StaticTzInfo timezones never have + ambiguous cases to correct: + + >>> from pytz import timezone + >>> gmt = timezone('GMT') + >>> isinstance(gmt, StaticTzInfo) + True + >>> dt = datetime(2011, 5, 8, 1, 2, 3, tzinfo=gmt) + >>> gmt.normalize(dt) is dt + True + + The supported method of converting between timezones is to use + datetime.astimezone(). Currently normalize() also works: + + >>> la = timezone('America/Los_Angeles') + >>> dt = la.localize(datetime(2011, 5, 7, 1, 2, 3)) + >>> fmt = '%Y-%m-%d %H:%M:%S %Z (%z)' + >>> gmt.normalize(dt).strftime(fmt) + '2011-05-07 08:02:03 GMT (+0000)' + ''' + if dt.tzinfo is self: + return dt + if dt.tzinfo is None: + raise ValueError('Naive time - no tzinfo set') + return dt.astimezone(self) + + def __repr__(self): + return '' % (self.zone,) + + def __reduce__(self): + # Special pickle to zone remains a singleton and to cope with + # database changes. + return pytz._p, (self.zone,) + + +class DstTzInfo(BaseTzInfo): + '''A timezone that has a variable offset from UTC + + The offset might change if daylight saving time comes into effect, + or at a point in history when the region decides to change their + timezone definition. + ''' + # Overridden in subclass + _utc_transition_times = None # Sorted list of DST transition times in UTC + _transition_info = None # [(utcoffset, dstoffset, tzname)] corresponding + # to _utc_transition_times entries + zone = None + + # Set in __init__ + _tzinfos = None + _dst = None # DST offset + + def __init__(self, _inf=None, _tzinfos=None): + if _inf: + self._tzinfos = _tzinfos + self._utcoffset, self._dst, self._tzname = _inf + else: + _tzinfos = {} + self._tzinfos = _tzinfos + self._utcoffset, self._dst, self._tzname = self._transition_info[0] + _tzinfos[self._transition_info[0]] = self + for inf in self._transition_info[1:]: + if inf not in _tzinfos: + _tzinfos[inf] = self.__class__(inf, _tzinfos) + + def fromutc(self, dt): + '''See datetime.tzinfo.fromutc''' + if (dt.tzinfo is not None + and getattr(dt.tzinfo, '_tzinfos', None) is not self._tzinfos): + raise ValueError('fromutc: dt.tzinfo is not self') + dt = dt.replace(tzinfo=None) + idx = max(0, bisect_right(self._utc_transition_times, dt) - 1) + inf = self._transition_info[idx] + return (dt + inf[0]).replace(tzinfo=self._tzinfos[inf]) + + def normalize(self, dt): + '''Correct the timezone information on the given datetime + + If date arithmetic crosses DST boundaries, the tzinfo + is not magically adjusted. This method normalizes the + tzinfo to the correct one. + + To test, first we need to do some setup + + >>> from pytz import timezone + >>> utc = timezone('UTC') + >>> eastern = timezone('US/Eastern') + >>> fmt = '%Y-%m-%d %H:%M:%S %Z (%z)' + + We next create a datetime right on an end-of-DST transition point, + the instant when the wallclocks are wound back one hour. + + >>> utc_dt = datetime(2002, 10, 27, 6, 0, 0, tzinfo=utc) + >>> loc_dt = utc_dt.astimezone(eastern) + >>> loc_dt.strftime(fmt) + '2002-10-27 01:00:00 EST (-0500)' + + Now, if we subtract a few minutes from it, note that the timezone + information has not changed. + + >>> before = loc_dt - timedelta(minutes=10) + >>> before.strftime(fmt) + '2002-10-27 00:50:00 EST (-0500)' + + But we can fix that by calling the normalize method + + >>> before = eastern.normalize(before) + >>> before.strftime(fmt) + '2002-10-27 01:50:00 EDT (-0400)' + + The supported method of converting between timezones is to use + datetime.astimezone(). Currently, normalize() also works: + + >>> th = timezone('Asia/Bangkok') + >>> am = timezone('Europe/Amsterdam') + >>> dt = th.localize(datetime(2011, 5, 7, 1, 2, 3)) + >>> fmt = '%Y-%m-%d %H:%M:%S %Z (%z)' + >>> am.normalize(dt).strftime(fmt) + '2011-05-06 20:02:03 CEST (+0200)' + ''' + if dt.tzinfo is None: + raise ValueError('Naive time - no tzinfo set') + + # Convert dt in localtime to UTC + offset = dt.tzinfo._utcoffset + dt = dt.replace(tzinfo=None) + dt = dt - offset + # convert it back, and return it + return self.fromutc(dt) + + def localize(self, dt, is_dst=False): + '''Convert naive time to local time. + + This method should be used to construct localtimes, rather + than passing a tzinfo argument to a datetime constructor. + + is_dst is used to determine the correct timezone in the ambigous + period at the end of daylight saving time. + + >>> from pytz import timezone + >>> fmt = '%Y-%m-%d %H:%M:%S %Z (%z)' + >>> amdam = timezone('Europe/Amsterdam') + >>> dt = datetime(2004, 10, 31, 2, 0, 0) + >>> loc_dt1 = amdam.localize(dt, is_dst=True) + >>> loc_dt2 = amdam.localize(dt, is_dst=False) + >>> loc_dt1.strftime(fmt) + '2004-10-31 02:00:00 CEST (+0200)' + >>> loc_dt2.strftime(fmt) + '2004-10-31 02:00:00 CET (+0100)' + >>> str(loc_dt2 - loc_dt1) + '1:00:00' + + Use is_dst=None to raise an AmbiguousTimeError for ambiguous + times at the end of daylight saving time + + >>> try: + ... loc_dt1 = amdam.localize(dt, is_dst=None) + ... except AmbiguousTimeError: + ... print('Ambiguous') + Ambiguous + + is_dst defaults to False + + >>> amdam.localize(dt) == amdam.localize(dt, False) + True + + is_dst is also used to determine the correct timezone in the + wallclock times jumped over at the start of daylight saving time. + + >>> pacific = timezone('US/Pacific') + >>> dt = datetime(2008, 3, 9, 2, 0, 0) + >>> ploc_dt1 = pacific.localize(dt, is_dst=True) + >>> ploc_dt2 = pacific.localize(dt, is_dst=False) + >>> ploc_dt1.strftime(fmt) + '2008-03-09 02:00:00 PDT (-0700)' + >>> ploc_dt2.strftime(fmt) + '2008-03-09 02:00:00 PST (-0800)' + >>> str(ploc_dt2 - ploc_dt1) + '1:00:00' + + Use is_dst=None to raise a NonExistentTimeError for these skipped + times. + + >>> try: + ... loc_dt1 = pacific.localize(dt, is_dst=None) + ... except NonExistentTimeError: + ... print('Non-existent') + Non-existent + ''' + if dt.tzinfo is not None: + raise ValueError('Not naive datetime (tzinfo is already set)') + + # Find the two best possibilities. + possible_loc_dt = set() + for delta in [timedelta(days=-1), timedelta(days=1)]: + loc_dt = dt + delta + idx = max(0, bisect_right( + self._utc_transition_times, loc_dt) - 1) + inf = self._transition_info[idx] + tzinfo = self._tzinfos[inf] + loc_dt = tzinfo.normalize(dt.replace(tzinfo=tzinfo)) + if loc_dt.replace(tzinfo=None) == dt: + possible_loc_dt.add(loc_dt) + + if len(possible_loc_dt) == 1: + return possible_loc_dt.pop() + + # If there are no possibly correct timezones, we are attempting + # to convert a time that never happened - the time period jumped + # during the start-of-DST transition period. + if len(possible_loc_dt) == 0: + # If we refuse to guess, raise an exception. + if is_dst is None: + raise NonExistentTimeError(dt) + + # If we are forcing the pre-DST side of the DST transition, we + # obtain the correct timezone by winding the clock forward a few + # hours. + elif is_dst: + return self.localize( + dt + timedelta(hours=6), is_dst=True) - timedelta(hours=6) + + # If we are forcing the post-DST side of the DST transition, we + # obtain the correct timezone by winding the clock back. + else: + return self.localize( + dt - timedelta(hours=6), is_dst=False) + timedelta(hours=6) + + + # If we get this far, we have multiple possible timezones - this + # is an ambiguous case occuring during the end-of-DST transition. + + # If told to be strict, raise an exception since we have an + # ambiguous case + if is_dst is None: + raise AmbiguousTimeError(dt) + + # Filter out the possiblilities that don't match the requested + # is_dst + filtered_possible_loc_dt = [ + p for p in possible_loc_dt + if bool(p.tzinfo._dst) == is_dst + ] + + # Hopefully we only have one possibility left. Return it. + if len(filtered_possible_loc_dt) == 1: + return filtered_possible_loc_dt[0] + + if len(filtered_possible_loc_dt) == 0: + filtered_possible_loc_dt = list(possible_loc_dt) + + # If we get this far, we have in a wierd timezone transition + # where the clocks have been wound back but is_dst is the same + # in both (eg. Europe/Warsaw 1915 when they switched to CET). + # At this point, we just have to guess unless we allow more + # hints to be passed in (such as the UTC offset or abbreviation), + # but that is just getting silly. + # + # Choose the earliest (by UTC) applicable timezone. + sorting_keys = {} + for local_dt in filtered_possible_loc_dt: + key = local_dt.replace(tzinfo=None) - local_dt.tzinfo._utcoffset + sorting_keys[key] = local_dt + first_key = sorted(sorting_keys)[0] + return sorting_keys[first_key] + + def utcoffset(self, dt, is_dst=None): + '''See datetime.tzinfo.utcoffset + + The is_dst parameter may be used to remove ambiguity during DST + transitions. + + >>> from pytz import timezone + >>> tz = timezone('America/St_Johns') + >>> ambiguous = datetime(2009, 10, 31, 23, 30) + + >>> tz.utcoffset(ambiguous, is_dst=False) + datetime.timedelta(-1, 73800) + + >>> tz.utcoffset(ambiguous, is_dst=True) + datetime.timedelta(-1, 77400) + + >>> try: + ... tz.utcoffset(ambiguous) + ... except AmbiguousTimeError: + ... print('Ambiguous') + Ambiguous + + ''' + if dt is None: + return None + elif dt.tzinfo is not self: + dt = self.localize(dt, is_dst) + return dt.tzinfo._utcoffset + else: + return self._utcoffset + + def dst(self, dt, is_dst=None): + '''See datetime.tzinfo.dst + + The is_dst parameter may be used to remove ambiguity during DST + transitions. + + >>> from pytz import timezone + >>> tz = timezone('America/St_Johns') + + >>> normal = datetime(2009, 9, 1) + + >>> tz.dst(normal) + datetime.timedelta(0, 3600) + >>> tz.dst(normal, is_dst=False) + datetime.timedelta(0, 3600) + >>> tz.dst(normal, is_dst=True) + datetime.timedelta(0, 3600) + + >>> ambiguous = datetime(2009, 10, 31, 23, 30) + + >>> tz.dst(ambiguous, is_dst=False) + datetime.timedelta(0) + >>> tz.dst(ambiguous, is_dst=True) + datetime.timedelta(0, 3600) + >>> try: + ... tz.dst(ambiguous) + ... except AmbiguousTimeError: + ... print('Ambiguous') + Ambiguous + + ''' + if dt is None: + return None + elif dt.tzinfo is not self: + dt = self.localize(dt, is_dst) + return dt.tzinfo._dst + else: + return self._dst + + def tzname(self, dt, is_dst=None): + '''See datetime.tzinfo.tzname + + The is_dst parameter may be used to remove ambiguity during DST + transitions. + + >>> from pytz import timezone + >>> tz = timezone('America/St_Johns') + + >>> normal = datetime(2009, 9, 1) + + >>> tz.tzname(normal) + 'NDT' + >>> tz.tzname(normal, is_dst=False) + 'NDT' + >>> tz.tzname(normal, is_dst=True) + 'NDT' + + >>> ambiguous = datetime(2009, 10, 31, 23, 30) + + >>> tz.tzname(ambiguous, is_dst=False) + 'NST' + >>> tz.tzname(ambiguous, is_dst=True) + 'NDT' + >>> try: + ... tz.tzname(ambiguous) + ... except AmbiguousTimeError: + ... print('Ambiguous') + Ambiguous + ''' + if dt is None: + return self.zone + elif dt.tzinfo is not self: + dt = self.localize(dt, is_dst) + return dt.tzinfo._tzname + else: + return self._tzname + + def __repr__(self): + if self._dst: + dst = 'DST' + else: + dst = 'STD' + if self._utcoffset > _notime: + return '' % ( + self.zone, self._tzname, self._utcoffset, dst + ) + else: + return '' % ( + self.zone, self._tzname, self._utcoffset, dst + ) + + def __reduce__(self): + # Special pickle to zone remains a singleton and to cope with + # database changes. + return pytz._p, ( + self.zone, + _to_seconds(self._utcoffset), + _to_seconds(self._dst), + self._tzname + ) + + + +def unpickler(zone, utcoffset=None, dstoffset=None, tzname=None): + """Factory function for unpickling pytz tzinfo instances. + + This is shared for both StaticTzInfo and DstTzInfo instances, because + database changes could cause a zones implementation to switch between + these two base classes and we can't break pickles on a pytz version + upgrade. + """ + # Raises a KeyError if zone no longer exists, which should never happen + # and would be a bug. + tz = pytz.timezone(zone) + + # A StaticTzInfo - just return it + if utcoffset is None: + return tz + + # This pickle was created from a DstTzInfo. We need to + # determine which of the list of tzinfo instances for this zone + # to use in order to restore the state of any datetime instances using + # it correctly. + utcoffset = memorized_timedelta(utcoffset) + dstoffset = memorized_timedelta(dstoffset) + try: + return tz._tzinfos[(utcoffset, dstoffset, tzname)] + except KeyError: + # The particular state requested in this timezone no longer exists. + # This indicates a corrupt pickle, or the timezone database has been + # corrected violently enough to make this particular + # (utcoffset,dstoffset) no longer exist in the zone, or the + # abbreviation has been changed. + pass + + # See if we can find an entry differing only by tzname. Abbreviations + # get changed from the initial guess by the database maintainers to + # match reality when this information is discovered. + for localized_tz in tz._tzinfos.values(): + if (localized_tz._utcoffset == utcoffset + and localized_tz._dst == dstoffset): + return localized_tz + + # This (utcoffset, dstoffset) information has been removed from the + # zone. Add it back. This might occur when the database maintainers have + # corrected incorrect information. datetime instances using this + # incorrect information will continue to do so, exactly as they were + # before being pickled. This is purely an overly paranoid safety net - I + # doubt this will ever been needed in real life. + inf = (utcoffset, dstoffset, tzname) + tz._tzinfos[inf] = tz.__class__(inf, tz._tzinfos) + return tz._tzinfos[inf] + diff --git a/lib/pytz/zoneinfo/Africa/Abidjan b/lib/pytz/zoneinfo/Africa/Abidjan new file mode 100644 index 00000000..6fd1af32 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Abidjan differ diff --git a/lib/pytz/zoneinfo/Africa/Accra b/lib/pytz/zoneinfo/Africa/Accra new file mode 100644 index 00000000..6ff8fb6b Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Accra differ diff --git a/lib/pytz/zoneinfo/Africa/Addis_Ababa b/lib/pytz/zoneinfo/Africa/Addis_Ababa new file mode 100644 index 00000000..4dfa06ab Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Addis_Ababa differ diff --git a/lib/pytz/zoneinfo/Africa/Algiers b/lib/pytz/zoneinfo/Africa/Algiers new file mode 100644 index 00000000..2a25f3ac Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Algiers differ diff --git a/lib/pytz/zoneinfo/Africa/Asmara b/lib/pytz/zoneinfo/Africa/Asmara new file mode 100644 index 00000000..0bc80c44 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Asmara differ diff --git a/lib/pytz/zoneinfo/Africa/Asmera b/lib/pytz/zoneinfo/Africa/Asmera new file mode 100644 index 00000000..0bc80c44 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Asmera differ diff --git a/lib/pytz/zoneinfo/Africa/Bamako b/lib/pytz/zoneinfo/Africa/Bamako new file mode 100644 index 00000000..6fd1af32 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Bamako differ diff --git a/lib/pytz/zoneinfo/Africa/Bangui b/lib/pytz/zoneinfo/Africa/Bangui new file mode 100644 index 00000000..b1c97cc5 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Bangui differ diff --git a/lib/pytz/zoneinfo/Africa/Banjul b/lib/pytz/zoneinfo/Africa/Banjul new file mode 100644 index 00000000..6fd1af32 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Banjul differ diff --git a/lib/pytz/zoneinfo/Africa/Bissau b/lib/pytz/zoneinfo/Africa/Bissau new file mode 100644 index 00000000..0696667c Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Bissau differ diff --git a/lib/pytz/zoneinfo/Africa/Blantyre b/lib/pytz/zoneinfo/Africa/Blantyre new file mode 100644 index 00000000..aebba5d9 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Blantyre differ diff --git a/lib/pytz/zoneinfo/Africa/Brazzaville b/lib/pytz/zoneinfo/Africa/Brazzaville new file mode 100644 index 00000000..b1c97cc5 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Brazzaville differ diff --git a/lib/pytz/zoneinfo/Africa/Bujumbura b/lib/pytz/zoneinfo/Africa/Bujumbura new file mode 100644 index 00000000..fff46c52 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Bujumbura differ diff --git a/lib/pytz/zoneinfo/Africa/Cairo b/lib/pytz/zoneinfo/Africa/Cairo new file mode 100644 index 00000000..0eeed113 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Cairo differ diff --git a/lib/pytz/zoneinfo/Africa/Casablanca b/lib/pytz/zoneinfo/Africa/Casablanca new file mode 100644 index 00000000..c001c375 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Casablanca differ diff --git a/lib/pytz/zoneinfo/Africa/Ceuta b/lib/pytz/zoneinfo/Africa/Ceuta new file mode 100644 index 00000000..6227e2bb Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Ceuta differ diff --git a/lib/pytz/zoneinfo/Africa/Conakry b/lib/pytz/zoneinfo/Africa/Conakry new file mode 100644 index 00000000..6fd1af32 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Conakry differ diff --git a/lib/pytz/zoneinfo/Africa/Dakar b/lib/pytz/zoneinfo/Africa/Dakar new file mode 100644 index 00000000..6fd1af32 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Dakar differ diff --git a/lib/pytz/zoneinfo/Africa/Dar_es_Salaam b/lib/pytz/zoneinfo/Africa/Dar_es_Salaam new file mode 100644 index 00000000..2ddddc5f Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Dar_es_Salaam differ diff --git a/lib/pytz/zoneinfo/Africa/Djibouti b/lib/pytz/zoneinfo/Africa/Djibouti new file mode 100644 index 00000000..559aabc1 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Djibouti differ diff --git a/lib/pytz/zoneinfo/Africa/Douala b/lib/pytz/zoneinfo/Africa/Douala new file mode 100644 index 00000000..b1c97cc5 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Douala differ diff --git a/lib/pytz/zoneinfo/Africa/El_Aaiun b/lib/pytz/zoneinfo/Africa/El_Aaiun new file mode 100644 index 00000000..805d39e4 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/El_Aaiun differ diff --git a/lib/pytz/zoneinfo/Africa/Freetown b/lib/pytz/zoneinfo/Africa/Freetown new file mode 100644 index 00000000..6fd1af32 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Freetown differ diff --git a/lib/pytz/zoneinfo/Africa/Gaborone b/lib/pytz/zoneinfo/Africa/Gaborone new file mode 100644 index 00000000..424534c4 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Gaborone differ diff --git a/lib/pytz/zoneinfo/Africa/Harare b/lib/pytz/zoneinfo/Africa/Harare new file mode 100644 index 00000000..0e53de0a Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Harare differ diff --git a/lib/pytz/zoneinfo/Africa/Johannesburg b/lib/pytz/zoneinfo/Africa/Johannesburg new file mode 100644 index 00000000..ddf3652e Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Johannesburg differ diff --git a/lib/pytz/zoneinfo/Africa/Juba b/lib/pytz/zoneinfo/Africa/Juba new file mode 100644 index 00000000..36291882 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Juba differ diff --git a/lib/pytz/zoneinfo/Africa/Kampala b/lib/pytz/zoneinfo/Africa/Kampala new file mode 100644 index 00000000..c6b5720e Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Kampala differ diff --git a/lib/pytz/zoneinfo/Africa/Khartoum b/lib/pytz/zoneinfo/Africa/Khartoum new file mode 100644 index 00000000..36291882 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Khartoum differ diff --git a/lib/pytz/zoneinfo/Africa/Kigali b/lib/pytz/zoneinfo/Africa/Kigali new file mode 100644 index 00000000..b99c2094 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Kigali differ diff --git a/lib/pytz/zoneinfo/Africa/Kinshasa b/lib/pytz/zoneinfo/Africa/Kinshasa new file mode 100644 index 00000000..b1c97cc5 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Kinshasa differ diff --git a/lib/pytz/zoneinfo/Africa/Lagos b/lib/pytz/zoneinfo/Africa/Lagos new file mode 100644 index 00000000..b1c97cc5 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Lagos differ diff --git a/lib/pytz/zoneinfo/Africa/Libreville b/lib/pytz/zoneinfo/Africa/Libreville new file mode 100644 index 00000000..b1c97cc5 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Libreville differ diff --git a/lib/pytz/zoneinfo/Africa/Lome b/lib/pytz/zoneinfo/Africa/Lome new file mode 100644 index 00000000..6fd1af32 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Lome differ diff --git a/lib/pytz/zoneinfo/Africa/Luanda b/lib/pytz/zoneinfo/Africa/Luanda new file mode 100644 index 00000000..b1c97cc5 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Luanda differ diff --git a/lib/pytz/zoneinfo/Africa/Lubumbashi b/lib/pytz/zoneinfo/Africa/Lubumbashi new file mode 100644 index 00000000..05aad3c8 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Lubumbashi differ diff --git a/lib/pytz/zoneinfo/Africa/Lusaka b/lib/pytz/zoneinfo/Africa/Lusaka new file mode 100644 index 00000000..612a8a07 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Lusaka differ diff --git a/lib/pytz/zoneinfo/Africa/Malabo b/lib/pytz/zoneinfo/Africa/Malabo new file mode 100644 index 00000000..b1c97cc5 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Malabo differ diff --git a/lib/pytz/zoneinfo/Africa/Maputo b/lib/pytz/zoneinfo/Africa/Maputo new file mode 100644 index 00000000..5b871dba Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Maputo differ diff --git a/lib/pytz/zoneinfo/Africa/Maseru b/lib/pytz/zoneinfo/Africa/Maseru new file mode 100644 index 00000000..7fb3b0a7 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Maseru differ diff --git a/lib/pytz/zoneinfo/Africa/Mbabane b/lib/pytz/zoneinfo/Africa/Mbabane new file mode 100644 index 00000000..8f0d40f2 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Mbabane differ diff --git a/lib/pytz/zoneinfo/Africa/Mogadishu b/lib/pytz/zoneinfo/Africa/Mogadishu new file mode 100644 index 00000000..3c278ab2 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Mogadishu differ diff --git a/lib/pytz/zoneinfo/Africa/Monrovia b/lib/pytz/zoneinfo/Africa/Monrovia new file mode 100644 index 00000000..0f2294ea Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Monrovia differ diff --git a/lib/pytz/zoneinfo/Africa/Nairobi b/lib/pytz/zoneinfo/Africa/Nairobi new file mode 100644 index 00000000..750d3dc1 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Nairobi differ diff --git a/lib/pytz/zoneinfo/Africa/Ndjamena b/lib/pytz/zoneinfo/Africa/Ndjamena new file mode 100644 index 00000000..bbfe19d6 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Ndjamena differ diff --git a/lib/pytz/zoneinfo/Africa/Niamey b/lib/pytz/zoneinfo/Africa/Niamey new file mode 100644 index 00000000..b1c97cc5 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Niamey differ diff --git a/lib/pytz/zoneinfo/Africa/Nouakchott b/lib/pytz/zoneinfo/Africa/Nouakchott new file mode 100644 index 00000000..6fd1af32 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Nouakchott differ diff --git a/lib/pytz/zoneinfo/Africa/Ouagadougou b/lib/pytz/zoneinfo/Africa/Ouagadougou new file mode 100644 index 00000000..6fd1af32 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Ouagadougou differ diff --git a/lib/pytz/zoneinfo/Africa/Porto-Novo b/lib/pytz/zoneinfo/Africa/Porto-Novo new file mode 100644 index 00000000..b1c97cc5 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Porto-Novo differ diff --git a/lib/pytz/zoneinfo/Africa/Sao_Tome b/lib/pytz/zoneinfo/Africa/Sao_Tome new file mode 100644 index 00000000..6fd1af32 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Sao_Tome differ diff --git a/lib/pytz/zoneinfo/Africa/Timbuktu b/lib/pytz/zoneinfo/Africa/Timbuktu new file mode 100644 index 00000000..6fd1af32 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Timbuktu differ diff --git a/lib/pytz/zoneinfo/Africa/Tripoli b/lib/pytz/zoneinfo/Africa/Tripoli new file mode 100644 index 00000000..b32e2202 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Tripoli differ diff --git a/lib/pytz/zoneinfo/Africa/Tunis b/lib/pytz/zoneinfo/Africa/Tunis new file mode 100644 index 00000000..4bd3885a Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Tunis differ diff --git a/lib/pytz/zoneinfo/Africa/Windhoek b/lib/pytz/zoneinfo/Africa/Windhoek new file mode 100644 index 00000000..33bdfdf2 Binary files /dev/null and b/lib/pytz/zoneinfo/Africa/Windhoek differ diff --git a/lib/pytz/zoneinfo/America/Adak b/lib/pytz/zoneinfo/America/Adak new file mode 100644 index 00000000..b0a5dd60 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Adak differ diff --git a/lib/pytz/zoneinfo/America/Anchorage b/lib/pytz/zoneinfo/America/Anchorage new file mode 100644 index 00000000..a4627cac Binary files /dev/null and b/lib/pytz/zoneinfo/America/Anchorage differ diff --git a/lib/pytz/zoneinfo/America/Anguilla b/lib/pytz/zoneinfo/America/Anguilla new file mode 100644 index 00000000..447efbe2 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Anguilla differ diff --git a/lib/pytz/zoneinfo/America/Antigua b/lib/pytz/zoneinfo/America/Antigua new file mode 100644 index 00000000..66ab1474 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Antigua differ diff --git a/lib/pytz/zoneinfo/America/Araguaina b/lib/pytz/zoneinfo/America/Araguaina new file mode 100644 index 00000000..507ea469 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Araguaina differ diff --git a/lib/pytz/zoneinfo/America/Argentina/Buenos_Aires b/lib/pytz/zoneinfo/America/Argentina/Buenos_Aires new file mode 100644 index 00000000..a1fae8c8 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Argentina/Buenos_Aires differ diff --git a/lib/pytz/zoneinfo/America/Argentina/Catamarca b/lib/pytz/zoneinfo/America/Argentina/Catamarca new file mode 100644 index 00000000..7cbc9f4b Binary files /dev/null and b/lib/pytz/zoneinfo/America/Argentina/Catamarca differ diff --git a/lib/pytz/zoneinfo/America/Argentina/ComodRivadavia b/lib/pytz/zoneinfo/America/Argentina/ComodRivadavia new file mode 100644 index 00000000..7cbc9f4b Binary files /dev/null and b/lib/pytz/zoneinfo/America/Argentina/ComodRivadavia differ diff --git a/lib/pytz/zoneinfo/America/Argentina/Cordoba b/lib/pytz/zoneinfo/America/Argentina/Cordoba new file mode 100644 index 00000000..cd97a24b Binary files /dev/null and b/lib/pytz/zoneinfo/America/Argentina/Cordoba differ diff --git a/lib/pytz/zoneinfo/America/Argentina/Jujuy b/lib/pytz/zoneinfo/America/Argentina/Jujuy new file mode 100644 index 00000000..7be3eeb6 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Argentina/Jujuy differ diff --git a/lib/pytz/zoneinfo/America/Argentina/La_Rioja b/lib/pytz/zoneinfo/America/Argentina/La_Rioja new file mode 100644 index 00000000..1296ed44 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Argentina/La_Rioja differ diff --git a/lib/pytz/zoneinfo/America/Argentina/Mendoza b/lib/pytz/zoneinfo/America/Argentina/Mendoza new file mode 100644 index 00000000..f9eb526c Binary files /dev/null and b/lib/pytz/zoneinfo/America/Argentina/Mendoza differ diff --git a/lib/pytz/zoneinfo/America/Argentina/Rio_Gallegos b/lib/pytz/zoneinfo/America/Argentina/Rio_Gallegos new file mode 100644 index 00000000..8fd203d1 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Argentina/Rio_Gallegos differ diff --git a/lib/pytz/zoneinfo/America/Argentina/Salta b/lib/pytz/zoneinfo/America/Argentina/Salta new file mode 100644 index 00000000..5778059f Binary files /dev/null and b/lib/pytz/zoneinfo/America/Argentina/Salta differ diff --git a/lib/pytz/zoneinfo/America/Argentina/San_Juan b/lib/pytz/zoneinfo/America/Argentina/San_Juan new file mode 100644 index 00000000..8670279e Binary files /dev/null and b/lib/pytz/zoneinfo/America/Argentina/San_Juan differ diff --git a/lib/pytz/zoneinfo/America/Argentina/San_Luis b/lib/pytz/zoneinfo/America/Argentina/San_Luis new file mode 100644 index 00000000..51eb1d84 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Argentina/San_Luis differ diff --git a/lib/pytz/zoneinfo/America/Argentina/Tucuman b/lib/pytz/zoneinfo/America/Argentina/Tucuman new file mode 100644 index 00000000..694093e7 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Argentina/Tucuman differ diff --git a/lib/pytz/zoneinfo/America/Argentina/Ushuaia b/lib/pytz/zoneinfo/America/Argentina/Ushuaia new file mode 100644 index 00000000..dc42621d Binary files /dev/null and b/lib/pytz/zoneinfo/America/Argentina/Ushuaia differ diff --git a/lib/pytz/zoneinfo/America/Aruba b/lib/pytz/zoneinfo/America/Aruba new file mode 100644 index 00000000..05e77ab4 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Aruba differ diff --git a/lib/pytz/zoneinfo/America/Asuncion b/lib/pytz/zoneinfo/America/Asuncion new file mode 100644 index 00000000..79541fdd Binary files /dev/null and b/lib/pytz/zoneinfo/America/Asuncion differ diff --git a/lib/pytz/zoneinfo/America/Atikokan b/lib/pytz/zoneinfo/America/Atikokan new file mode 100644 index 00000000..5708b55a Binary files /dev/null and b/lib/pytz/zoneinfo/America/Atikokan differ diff --git a/lib/pytz/zoneinfo/America/Atka b/lib/pytz/zoneinfo/America/Atka new file mode 100644 index 00000000..b0a5dd60 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Atka differ diff --git a/lib/pytz/zoneinfo/America/Bahia b/lib/pytz/zoneinfo/America/Bahia new file mode 100644 index 00000000..3b599585 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Bahia differ diff --git a/lib/pytz/zoneinfo/America/Bahia_Banderas b/lib/pytz/zoneinfo/America/Bahia_Banderas new file mode 100644 index 00000000..21e2b719 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Bahia_Banderas differ diff --git a/lib/pytz/zoneinfo/America/Barbados b/lib/pytz/zoneinfo/America/Barbados new file mode 100644 index 00000000..63399360 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Barbados differ diff --git a/lib/pytz/zoneinfo/America/Belem b/lib/pytz/zoneinfo/America/Belem new file mode 100644 index 00000000..d85c0f72 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Belem differ diff --git a/lib/pytz/zoneinfo/America/Belize b/lib/pytz/zoneinfo/America/Belize new file mode 100644 index 00000000..eada52e7 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Belize differ diff --git a/lib/pytz/zoneinfo/America/Blanc-Sablon b/lib/pytz/zoneinfo/America/Blanc-Sablon new file mode 100644 index 00000000..abcde7d9 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Blanc-Sablon differ diff --git a/lib/pytz/zoneinfo/America/Boa_Vista b/lib/pytz/zoneinfo/America/Boa_Vista new file mode 100644 index 00000000..2466a25f Binary files /dev/null and b/lib/pytz/zoneinfo/America/Boa_Vista differ diff --git a/lib/pytz/zoneinfo/America/Bogota b/lib/pytz/zoneinfo/America/Bogota new file mode 100644 index 00000000..7a5a445a Binary files /dev/null and b/lib/pytz/zoneinfo/America/Bogota differ diff --git a/lib/pytz/zoneinfo/America/Boise b/lib/pytz/zoneinfo/America/Boise new file mode 100644 index 00000000..ada6d64b Binary files /dev/null and b/lib/pytz/zoneinfo/America/Boise differ diff --git a/lib/pytz/zoneinfo/America/Buenos_Aires b/lib/pytz/zoneinfo/America/Buenos_Aires new file mode 100644 index 00000000..a1fae8c8 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Buenos_Aires differ diff --git a/lib/pytz/zoneinfo/America/Cambridge_Bay b/lib/pytz/zoneinfo/America/Cambridge_Bay new file mode 100644 index 00000000..58e21baa Binary files /dev/null and b/lib/pytz/zoneinfo/America/Cambridge_Bay differ diff --git a/lib/pytz/zoneinfo/America/Campo_Grande b/lib/pytz/zoneinfo/America/Campo_Grande new file mode 100644 index 00000000..d810ae56 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Campo_Grande differ diff --git a/lib/pytz/zoneinfo/America/Cancun b/lib/pytz/zoneinfo/America/Cancun new file mode 100644 index 00000000..a99eedd7 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Cancun differ diff --git a/lib/pytz/zoneinfo/America/Caracas b/lib/pytz/zoneinfo/America/Caracas new file mode 100644 index 00000000..15b9a52c Binary files /dev/null and b/lib/pytz/zoneinfo/America/Caracas differ diff --git a/lib/pytz/zoneinfo/America/Catamarca b/lib/pytz/zoneinfo/America/Catamarca new file mode 100644 index 00000000..7cbc9f4b Binary files /dev/null and b/lib/pytz/zoneinfo/America/Catamarca differ diff --git a/lib/pytz/zoneinfo/America/Cayenne b/lib/pytz/zoneinfo/America/Cayenne new file mode 100644 index 00000000..bffe9b02 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Cayenne differ diff --git a/lib/pytz/zoneinfo/America/Cayman b/lib/pytz/zoneinfo/America/Cayman new file mode 100644 index 00000000..0eb14b75 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Cayman differ diff --git a/lib/pytz/zoneinfo/America/Chicago b/lib/pytz/zoneinfo/America/Chicago new file mode 100644 index 00000000..3dd8f0fa Binary files /dev/null and b/lib/pytz/zoneinfo/America/Chicago differ diff --git a/lib/pytz/zoneinfo/America/Chihuahua b/lib/pytz/zoneinfo/America/Chihuahua new file mode 100644 index 00000000..e3adbdbf Binary files /dev/null and b/lib/pytz/zoneinfo/America/Chihuahua differ diff --git a/lib/pytz/zoneinfo/America/Coral_Harbour b/lib/pytz/zoneinfo/America/Coral_Harbour new file mode 100644 index 00000000..5708b55a Binary files /dev/null and b/lib/pytz/zoneinfo/America/Coral_Harbour differ diff --git a/lib/pytz/zoneinfo/America/Cordoba b/lib/pytz/zoneinfo/America/Cordoba new file mode 100644 index 00000000..cd97a24b Binary files /dev/null and b/lib/pytz/zoneinfo/America/Cordoba differ diff --git a/lib/pytz/zoneinfo/America/Costa_Rica b/lib/pytz/zoneinfo/America/Costa_Rica new file mode 100644 index 00000000..c247133e Binary files /dev/null and b/lib/pytz/zoneinfo/America/Costa_Rica differ diff --git a/lib/pytz/zoneinfo/America/Creston b/lib/pytz/zoneinfo/America/Creston new file mode 100644 index 00000000..798f627a Binary files /dev/null and b/lib/pytz/zoneinfo/America/Creston differ diff --git a/lib/pytz/zoneinfo/America/Cuiaba b/lib/pytz/zoneinfo/America/Cuiaba new file mode 100644 index 00000000..e3aec8cc Binary files /dev/null and b/lib/pytz/zoneinfo/America/Cuiaba differ diff --git a/lib/pytz/zoneinfo/America/Curacao b/lib/pytz/zoneinfo/America/Curacao new file mode 100644 index 00000000..05e77ab4 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Curacao differ diff --git a/lib/pytz/zoneinfo/America/Danmarkshavn b/lib/pytz/zoneinfo/America/Danmarkshavn new file mode 100644 index 00000000..a8b58ada Binary files /dev/null and b/lib/pytz/zoneinfo/America/Danmarkshavn differ diff --git a/lib/pytz/zoneinfo/America/Dawson b/lib/pytz/zoneinfo/America/Dawson new file mode 100644 index 00000000..61c96889 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Dawson differ diff --git a/lib/pytz/zoneinfo/America/Dawson_Creek b/lib/pytz/zoneinfo/America/Dawson_Creek new file mode 100644 index 00000000..78f90763 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Dawson_Creek differ diff --git a/lib/pytz/zoneinfo/America/Denver b/lib/pytz/zoneinfo/America/Denver new file mode 100644 index 00000000..7fc66917 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Denver differ diff --git a/lib/pytz/zoneinfo/America/Detroit b/lib/pytz/zoneinfo/America/Detroit new file mode 100644 index 00000000..a123b331 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Detroit differ diff --git a/lib/pytz/zoneinfo/America/Dominica b/lib/pytz/zoneinfo/America/Dominica new file mode 100644 index 00000000..447efbe2 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Dominica differ diff --git a/lib/pytz/zoneinfo/America/Edmonton b/lib/pytz/zoneinfo/America/Edmonton new file mode 100644 index 00000000..d02fbcd4 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Edmonton differ diff --git a/lib/pytz/zoneinfo/America/Eirunepe b/lib/pytz/zoneinfo/America/Eirunepe new file mode 100644 index 00000000..3359731e Binary files /dev/null and b/lib/pytz/zoneinfo/America/Eirunepe differ diff --git a/lib/pytz/zoneinfo/America/El_Salvador b/lib/pytz/zoneinfo/America/El_Salvador new file mode 100644 index 00000000..9b8bc7a8 Binary files /dev/null and b/lib/pytz/zoneinfo/America/El_Salvador differ diff --git a/lib/pytz/zoneinfo/America/Ensenada b/lib/pytz/zoneinfo/America/Ensenada new file mode 100644 index 00000000..13874753 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Ensenada differ diff --git a/lib/pytz/zoneinfo/America/Fort_Wayne b/lib/pytz/zoneinfo/America/Fort_Wayne new file mode 100644 index 00000000..4a92c065 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Fort_Wayne differ diff --git a/lib/pytz/zoneinfo/America/Fortaleza b/lib/pytz/zoneinfo/America/Fortaleza new file mode 100644 index 00000000..c0bcf4dc Binary files /dev/null and b/lib/pytz/zoneinfo/America/Fortaleza differ diff --git a/lib/pytz/zoneinfo/America/Glace_Bay b/lib/pytz/zoneinfo/America/Glace_Bay new file mode 100644 index 00000000..f58522b6 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Glace_Bay differ diff --git a/lib/pytz/zoneinfo/America/Godthab b/lib/pytz/zoneinfo/America/Godthab new file mode 100644 index 00000000..111d9a81 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Godthab differ diff --git a/lib/pytz/zoneinfo/America/Goose_Bay b/lib/pytz/zoneinfo/America/Goose_Bay new file mode 100644 index 00000000..b4b945e8 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Goose_Bay differ diff --git a/lib/pytz/zoneinfo/America/Grand_Turk b/lib/pytz/zoneinfo/America/Grand_Turk new file mode 100644 index 00000000..6cadf969 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Grand_Turk differ diff --git a/lib/pytz/zoneinfo/America/Grenada b/lib/pytz/zoneinfo/America/Grenada new file mode 100644 index 00000000..447efbe2 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Grenada differ diff --git a/lib/pytz/zoneinfo/America/Guadeloupe b/lib/pytz/zoneinfo/America/Guadeloupe new file mode 100644 index 00000000..447efbe2 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Guadeloupe differ diff --git a/lib/pytz/zoneinfo/America/Guatemala b/lib/pytz/zoneinfo/America/Guatemala new file mode 100644 index 00000000..abf943be Binary files /dev/null and b/lib/pytz/zoneinfo/America/Guatemala differ diff --git a/lib/pytz/zoneinfo/America/Guayaquil b/lib/pytz/zoneinfo/America/Guayaquil new file mode 100644 index 00000000..08289046 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Guayaquil differ diff --git a/lib/pytz/zoneinfo/America/Guyana b/lib/pytz/zoneinfo/America/Guyana new file mode 100644 index 00000000..036dbe06 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Guyana differ diff --git a/lib/pytz/zoneinfo/America/Halifax b/lib/pytz/zoneinfo/America/Halifax new file mode 100644 index 00000000..f86ece4c Binary files /dev/null and b/lib/pytz/zoneinfo/America/Halifax differ diff --git a/lib/pytz/zoneinfo/America/Havana b/lib/pytz/zoneinfo/America/Havana new file mode 100644 index 00000000..1a58fcdc Binary files /dev/null and b/lib/pytz/zoneinfo/America/Havana differ diff --git a/lib/pytz/zoneinfo/America/Hermosillo b/lib/pytz/zoneinfo/America/Hermosillo new file mode 100644 index 00000000..ec435c23 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Hermosillo differ diff --git a/lib/pytz/zoneinfo/America/Indiana/Indianapolis b/lib/pytz/zoneinfo/America/Indiana/Indianapolis new file mode 100644 index 00000000..4a92c065 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Indiana/Indianapolis differ diff --git a/lib/pytz/zoneinfo/America/Indiana/Knox b/lib/pytz/zoneinfo/America/Indiana/Knox new file mode 100644 index 00000000..cc785da9 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Indiana/Knox differ diff --git a/lib/pytz/zoneinfo/America/Indiana/Marengo b/lib/pytz/zoneinfo/America/Indiana/Marengo new file mode 100644 index 00000000..a23d7b75 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Indiana/Marengo differ diff --git a/lib/pytz/zoneinfo/America/Indiana/Petersburg b/lib/pytz/zoneinfo/America/Indiana/Petersburg new file mode 100644 index 00000000..f16cb304 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Indiana/Petersburg differ diff --git a/lib/pytz/zoneinfo/America/Indiana/Tell_City b/lib/pytz/zoneinfo/America/Indiana/Tell_City new file mode 100644 index 00000000..0250bf90 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Indiana/Tell_City differ diff --git a/lib/pytz/zoneinfo/America/Indiana/Vevay b/lib/pytz/zoneinfo/America/Indiana/Vevay new file mode 100644 index 00000000..e934de61 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Indiana/Vevay differ diff --git a/lib/pytz/zoneinfo/America/Indiana/Vincennes b/lib/pytz/zoneinfo/America/Indiana/Vincennes new file mode 100644 index 00000000..adbdbeee Binary files /dev/null and b/lib/pytz/zoneinfo/America/Indiana/Vincennes differ diff --git a/lib/pytz/zoneinfo/America/Indiana/Winamac b/lib/pytz/zoneinfo/America/Indiana/Winamac new file mode 100644 index 00000000..b34f7b27 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Indiana/Winamac differ diff --git a/lib/pytz/zoneinfo/America/Indianapolis b/lib/pytz/zoneinfo/America/Indianapolis new file mode 100644 index 00000000..4a92c065 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Indianapolis differ diff --git a/lib/pytz/zoneinfo/America/Inuvik b/lib/pytz/zoneinfo/America/Inuvik new file mode 100644 index 00000000..077fad4f Binary files /dev/null and b/lib/pytz/zoneinfo/America/Inuvik differ diff --git a/lib/pytz/zoneinfo/America/Iqaluit b/lib/pytz/zoneinfo/America/Iqaluit new file mode 100644 index 00000000..e67b71fe Binary files /dev/null and b/lib/pytz/zoneinfo/America/Iqaluit differ diff --git a/lib/pytz/zoneinfo/America/Jamaica b/lib/pytz/zoneinfo/America/Jamaica new file mode 100644 index 00000000..24ea5dc0 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Jamaica differ diff --git a/lib/pytz/zoneinfo/America/Jujuy b/lib/pytz/zoneinfo/America/Jujuy new file mode 100644 index 00000000..7be3eeb6 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Jujuy differ diff --git a/lib/pytz/zoneinfo/America/Juneau b/lib/pytz/zoneinfo/America/Juneau new file mode 100644 index 00000000..ade50a8e Binary files /dev/null and b/lib/pytz/zoneinfo/America/Juneau differ diff --git a/lib/pytz/zoneinfo/America/Kentucky/Louisville b/lib/pytz/zoneinfo/America/Kentucky/Louisville new file mode 100644 index 00000000..fdf2e88b Binary files /dev/null and b/lib/pytz/zoneinfo/America/Kentucky/Louisville differ diff --git a/lib/pytz/zoneinfo/America/Kentucky/Monticello b/lib/pytz/zoneinfo/America/Kentucky/Monticello new file mode 100644 index 00000000..60991aa3 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Kentucky/Monticello differ diff --git a/lib/pytz/zoneinfo/America/Knox_IN b/lib/pytz/zoneinfo/America/Knox_IN new file mode 100644 index 00000000..cc785da9 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Knox_IN differ diff --git a/lib/pytz/zoneinfo/America/Kralendijk b/lib/pytz/zoneinfo/America/Kralendijk new file mode 100644 index 00000000..05e77ab4 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Kralendijk differ diff --git a/lib/pytz/zoneinfo/America/La_Paz b/lib/pytz/zoneinfo/America/La_Paz new file mode 100644 index 00000000..cedf0b3a Binary files /dev/null and b/lib/pytz/zoneinfo/America/La_Paz differ diff --git a/lib/pytz/zoneinfo/America/Lima b/lib/pytz/zoneinfo/America/Lima new file mode 100644 index 00000000..789fa5c2 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Lima differ diff --git a/lib/pytz/zoneinfo/America/Los_Angeles b/lib/pytz/zoneinfo/America/Los_Angeles new file mode 100644 index 00000000..1fa9149f Binary files /dev/null and b/lib/pytz/zoneinfo/America/Los_Angeles differ diff --git a/lib/pytz/zoneinfo/America/Louisville b/lib/pytz/zoneinfo/America/Louisville new file mode 100644 index 00000000..fdf2e88b Binary files /dev/null and b/lib/pytz/zoneinfo/America/Louisville differ diff --git a/lib/pytz/zoneinfo/America/Lower_Princes b/lib/pytz/zoneinfo/America/Lower_Princes new file mode 100644 index 00000000..05e77ab4 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Lower_Princes differ diff --git a/lib/pytz/zoneinfo/America/Maceio b/lib/pytz/zoneinfo/America/Maceio new file mode 100644 index 00000000..de749909 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Maceio differ diff --git a/lib/pytz/zoneinfo/America/Managua b/lib/pytz/zoneinfo/America/Managua new file mode 100644 index 00000000..c543ffd4 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Managua differ diff --git a/lib/pytz/zoneinfo/America/Manaus b/lib/pytz/zoneinfo/America/Manaus new file mode 100644 index 00000000..e0222f18 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Manaus differ diff --git a/lib/pytz/zoneinfo/America/Marigot b/lib/pytz/zoneinfo/America/Marigot new file mode 100644 index 00000000..447efbe2 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Marigot differ diff --git a/lib/pytz/zoneinfo/America/Martinique b/lib/pytz/zoneinfo/America/Martinique new file mode 100644 index 00000000..f9e2399c Binary files /dev/null and b/lib/pytz/zoneinfo/America/Martinique differ diff --git a/lib/pytz/zoneinfo/America/Matamoros b/lib/pytz/zoneinfo/America/Matamoros new file mode 100644 index 00000000..5671d258 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Matamoros differ diff --git a/lib/pytz/zoneinfo/America/Mazatlan b/lib/pytz/zoneinfo/America/Mazatlan new file mode 100644 index 00000000..afa94c2a Binary files /dev/null and b/lib/pytz/zoneinfo/America/Mazatlan differ diff --git a/lib/pytz/zoneinfo/America/Mendoza b/lib/pytz/zoneinfo/America/Mendoza new file mode 100644 index 00000000..f9eb526c Binary files /dev/null and b/lib/pytz/zoneinfo/America/Mendoza differ diff --git a/lib/pytz/zoneinfo/America/Menominee b/lib/pytz/zoneinfo/America/Menominee new file mode 100644 index 00000000..55d6e326 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Menominee differ diff --git a/lib/pytz/zoneinfo/America/Merida b/lib/pytz/zoneinfo/America/Merida new file mode 100644 index 00000000..ecc1856e Binary files /dev/null and b/lib/pytz/zoneinfo/America/Merida differ diff --git a/lib/pytz/zoneinfo/America/Metlakatla b/lib/pytz/zoneinfo/America/Metlakatla new file mode 100644 index 00000000..e66cc341 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Metlakatla differ diff --git a/lib/pytz/zoneinfo/America/Mexico_City b/lib/pytz/zoneinfo/America/Mexico_City new file mode 100644 index 00000000..f11e3d2d Binary files /dev/null and b/lib/pytz/zoneinfo/America/Mexico_City differ diff --git a/lib/pytz/zoneinfo/America/Miquelon b/lib/pytz/zoneinfo/America/Miquelon new file mode 100644 index 00000000..36f66961 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Miquelon differ diff --git a/lib/pytz/zoneinfo/America/Moncton b/lib/pytz/zoneinfo/America/Moncton new file mode 100644 index 00000000..51cb1ba3 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Moncton differ diff --git a/lib/pytz/zoneinfo/America/Monterrey b/lib/pytz/zoneinfo/America/Monterrey new file mode 100644 index 00000000..dcac92ba Binary files /dev/null and b/lib/pytz/zoneinfo/America/Monterrey differ diff --git a/lib/pytz/zoneinfo/America/Montevideo b/lib/pytz/zoneinfo/America/Montevideo new file mode 100644 index 00000000..ab3d6807 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Montevideo differ diff --git a/lib/pytz/zoneinfo/America/Montreal b/lib/pytz/zoneinfo/America/Montreal new file mode 100644 index 00000000..89b9f493 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Montreal differ diff --git a/lib/pytz/zoneinfo/America/Montserrat b/lib/pytz/zoneinfo/America/Montserrat new file mode 100644 index 00000000..447efbe2 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Montserrat differ diff --git a/lib/pytz/zoneinfo/America/Nassau b/lib/pytz/zoneinfo/America/Nassau new file mode 100644 index 00000000..e5d0289b Binary files /dev/null and b/lib/pytz/zoneinfo/America/Nassau differ diff --git a/lib/pytz/zoneinfo/America/New_York b/lib/pytz/zoneinfo/America/New_York new file mode 100644 index 00000000..7553fee3 Binary files /dev/null and b/lib/pytz/zoneinfo/America/New_York differ diff --git a/lib/pytz/zoneinfo/America/Nipigon b/lib/pytz/zoneinfo/America/Nipigon new file mode 100644 index 00000000..f8a0292b Binary files /dev/null and b/lib/pytz/zoneinfo/America/Nipigon differ diff --git a/lib/pytz/zoneinfo/America/Nome b/lib/pytz/zoneinfo/America/Nome new file mode 100644 index 00000000..d370ab14 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Nome differ diff --git a/lib/pytz/zoneinfo/America/Noronha b/lib/pytz/zoneinfo/America/Noronha new file mode 100644 index 00000000..774b14e6 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Noronha differ diff --git a/lib/pytz/zoneinfo/America/North_Dakota/Beulah b/lib/pytz/zoneinfo/America/North_Dakota/Beulah new file mode 100644 index 00000000..8174c882 Binary files /dev/null and b/lib/pytz/zoneinfo/America/North_Dakota/Beulah differ diff --git a/lib/pytz/zoneinfo/America/North_Dakota/Center b/lib/pytz/zoneinfo/America/North_Dakota/Center new file mode 100644 index 00000000..8035b24f Binary files /dev/null and b/lib/pytz/zoneinfo/America/North_Dakota/Center differ diff --git a/lib/pytz/zoneinfo/America/North_Dakota/New_Salem b/lib/pytz/zoneinfo/America/North_Dakota/New_Salem new file mode 100644 index 00000000..5b630ee6 Binary files /dev/null and b/lib/pytz/zoneinfo/America/North_Dakota/New_Salem differ diff --git a/lib/pytz/zoneinfo/America/Ojinaga b/lib/pytz/zoneinfo/America/Ojinaga new file mode 100644 index 00000000..190c5c86 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Ojinaga differ diff --git a/lib/pytz/zoneinfo/America/Panama b/lib/pytz/zoneinfo/America/Panama new file mode 100644 index 00000000..5c1c0637 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Panama differ diff --git a/lib/pytz/zoneinfo/America/Pangnirtung b/lib/pytz/zoneinfo/America/Pangnirtung new file mode 100644 index 00000000..994da430 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Pangnirtung differ diff --git a/lib/pytz/zoneinfo/America/Paramaribo b/lib/pytz/zoneinfo/America/Paramaribo new file mode 100644 index 00000000..2f05b236 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Paramaribo differ diff --git a/lib/pytz/zoneinfo/America/Phoenix b/lib/pytz/zoneinfo/America/Phoenix new file mode 100644 index 00000000..adf28236 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Phoenix differ diff --git a/lib/pytz/zoneinfo/America/Port-au-Prince b/lib/pytz/zoneinfo/America/Port-au-Prince new file mode 100644 index 00000000..dd8895c0 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Port-au-Prince differ diff --git a/lib/pytz/zoneinfo/America/Port_of_Spain b/lib/pytz/zoneinfo/America/Port_of_Spain new file mode 100644 index 00000000..447efbe2 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Port_of_Spain differ diff --git a/lib/pytz/zoneinfo/America/Porto_Acre b/lib/pytz/zoneinfo/America/Porto_Acre new file mode 100644 index 00000000..788d0e9c Binary files /dev/null and b/lib/pytz/zoneinfo/America/Porto_Acre differ diff --git a/lib/pytz/zoneinfo/America/Porto_Velho b/lib/pytz/zoneinfo/America/Porto_Velho new file mode 100644 index 00000000..aa8cf315 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Porto_Velho differ diff --git a/lib/pytz/zoneinfo/America/Puerto_Rico b/lib/pytz/zoneinfo/America/Puerto_Rico new file mode 100644 index 00000000..d4525a68 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Puerto_Rico differ diff --git a/lib/pytz/zoneinfo/America/Rainy_River b/lib/pytz/zoneinfo/America/Rainy_River new file mode 100644 index 00000000..70dcd2d8 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Rainy_River differ diff --git a/lib/pytz/zoneinfo/America/Rankin_Inlet b/lib/pytz/zoneinfo/America/Rankin_Inlet new file mode 100644 index 00000000..cc15d831 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Rankin_Inlet differ diff --git a/lib/pytz/zoneinfo/America/Recife b/lib/pytz/zoneinfo/America/Recife new file mode 100644 index 00000000..f0ad7b98 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Recife differ diff --git a/lib/pytz/zoneinfo/America/Regina b/lib/pytz/zoneinfo/America/Regina new file mode 100644 index 00000000..5fe8d6b6 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Regina differ diff --git a/lib/pytz/zoneinfo/America/Resolute b/lib/pytz/zoneinfo/America/Resolute new file mode 100644 index 00000000..53079413 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Resolute differ diff --git a/lib/pytz/zoneinfo/America/Rio_Branco b/lib/pytz/zoneinfo/America/Rio_Branco new file mode 100644 index 00000000..788d0e9c Binary files /dev/null and b/lib/pytz/zoneinfo/America/Rio_Branco differ diff --git a/lib/pytz/zoneinfo/America/Rosario b/lib/pytz/zoneinfo/America/Rosario new file mode 100644 index 00000000..cd97a24b Binary files /dev/null and b/lib/pytz/zoneinfo/America/Rosario differ diff --git a/lib/pytz/zoneinfo/America/Santa_Isabel b/lib/pytz/zoneinfo/America/Santa_Isabel new file mode 100644 index 00000000..e1c4d161 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Santa_Isabel differ diff --git a/lib/pytz/zoneinfo/America/Santarem b/lib/pytz/zoneinfo/America/Santarem new file mode 100644 index 00000000..bb469d39 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Santarem differ diff --git a/lib/pytz/zoneinfo/America/Santiago b/lib/pytz/zoneinfo/America/Santiago new file mode 100644 index 00000000..92cf5597 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Santiago differ diff --git a/lib/pytz/zoneinfo/America/Santo_Domingo b/lib/pytz/zoneinfo/America/Santo_Domingo new file mode 100644 index 00000000..77eab315 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Santo_Domingo differ diff --git a/lib/pytz/zoneinfo/America/Sao_Paulo b/lib/pytz/zoneinfo/America/Sao_Paulo new file mode 100644 index 00000000..552ce7c2 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Sao_Paulo differ diff --git a/lib/pytz/zoneinfo/America/Scoresbysund b/lib/pytz/zoneinfo/America/Scoresbysund new file mode 100644 index 00000000..85676ca3 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Scoresbysund differ diff --git a/lib/pytz/zoneinfo/America/Shiprock b/lib/pytz/zoneinfo/America/Shiprock new file mode 100644 index 00000000..7fc66917 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Shiprock differ diff --git a/lib/pytz/zoneinfo/America/Sitka b/lib/pytz/zoneinfo/America/Sitka new file mode 100644 index 00000000..48fc6aff Binary files /dev/null and b/lib/pytz/zoneinfo/America/Sitka differ diff --git a/lib/pytz/zoneinfo/America/St_Barthelemy b/lib/pytz/zoneinfo/America/St_Barthelemy new file mode 100644 index 00000000..447efbe2 Binary files /dev/null and b/lib/pytz/zoneinfo/America/St_Barthelemy differ diff --git a/lib/pytz/zoneinfo/America/St_Johns b/lib/pytz/zoneinfo/America/St_Johns new file mode 100644 index 00000000..a1d14854 Binary files /dev/null and b/lib/pytz/zoneinfo/America/St_Johns differ diff --git a/lib/pytz/zoneinfo/America/St_Kitts b/lib/pytz/zoneinfo/America/St_Kitts new file mode 100644 index 00000000..447efbe2 Binary files /dev/null and b/lib/pytz/zoneinfo/America/St_Kitts differ diff --git a/lib/pytz/zoneinfo/America/St_Lucia b/lib/pytz/zoneinfo/America/St_Lucia new file mode 100644 index 00000000..447efbe2 Binary files /dev/null and b/lib/pytz/zoneinfo/America/St_Lucia differ diff --git a/lib/pytz/zoneinfo/America/St_Thomas b/lib/pytz/zoneinfo/America/St_Thomas new file mode 100644 index 00000000..447efbe2 Binary files /dev/null and b/lib/pytz/zoneinfo/America/St_Thomas differ diff --git a/lib/pytz/zoneinfo/America/St_Vincent b/lib/pytz/zoneinfo/America/St_Vincent new file mode 100644 index 00000000..447efbe2 Binary files /dev/null and b/lib/pytz/zoneinfo/America/St_Vincent differ diff --git a/lib/pytz/zoneinfo/America/Swift_Current b/lib/pytz/zoneinfo/America/Swift_Current new file mode 100644 index 00000000..4db1300a Binary files /dev/null and b/lib/pytz/zoneinfo/America/Swift_Current differ diff --git a/lib/pytz/zoneinfo/America/Tegucigalpa b/lib/pytz/zoneinfo/America/Tegucigalpa new file mode 100644 index 00000000..7aea8f99 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Tegucigalpa differ diff --git a/lib/pytz/zoneinfo/America/Thule b/lib/pytz/zoneinfo/America/Thule new file mode 100644 index 00000000..deefcc8d Binary files /dev/null and b/lib/pytz/zoneinfo/America/Thule differ diff --git a/lib/pytz/zoneinfo/America/Thunder_Bay b/lib/pytz/zoneinfo/America/Thunder_Bay new file mode 100644 index 00000000..aa1d4860 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Thunder_Bay differ diff --git a/lib/pytz/zoneinfo/America/Tijuana b/lib/pytz/zoneinfo/America/Tijuana new file mode 100644 index 00000000..13874753 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Tijuana differ diff --git a/lib/pytz/zoneinfo/America/Toronto b/lib/pytz/zoneinfo/America/Toronto new file mode 100644 index 00000000..7b4682a3 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Toronto differ diff --git a/lib/pytz/zoneinfo/America/Tortola b/lib/pytz/zoneinfo/America/Tortola new file mode 100644 index 00000000..447efbe2 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Tortola differ diff --git a/lib/pytz/zoneinfo/America/Vancouver b/lib/pytz/zoneinfo/America/Vancouver new file mode 100644 index 00000000..9b5d9241 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Vancouver differ diff --git a/lib/pytz/zoneinfo/America/Virgin b/lib/pytz/zoneinfo/America/Virgin new file mode 100644 index 00000000..447efbe2 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Virgin differ diff --git a/lib/pytz/zoneinfo/America/Whitehorse b/lib/pytz/zoneinfo/America/Whitehorse new file mode 100644 index 00000000..8604c5c5 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Whitehorse differ diff --git a/lib/pytz/zoneinfo/America/Winnipeg b/lib/pytz/zoneinfo/America/Winnipeg new file mode 100644 index 00000000..2ffe3d8d Binary files /dev/null and b/lib/pytz/zoneinfo/America/Winnipeg differ diff --git a/lib/pytz/zoneinfo/America/Yakutat b/lib/pytz/zoneinfo/America/Yakutat new file mode 100644 index 00000000..f3d73990 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Yakutat differ diff --git a/lib/pytz/zoneinfo/America/Yellowknife b/lib/pytz/zoneinfo/America/Yellowknife new file mode 100644 index 00000000..21cba6e4 Binary files /dev/null and b/lib/pytz/zoneinfo/America/Yellowknife differ diff --git a/lib/pytz/zoneinfo/Antarctica/Casey b/lib/pytz/zoneinfo/Antarctica/Casey new file mode 100644 index 00000000..c2a99056 Binary files /dev/null and b/lib/pytz/zoneinfo/Antarctica/Casey differ diff --git a/lib/pytz/zoneinfo/Antarctica/Davis b/lib/pytz/zoneinfo/Antarctica/Davis new file mode 100644 index 00000000..7321c67f Binary files /dev/null and b/lib/pytz/zoneinfo/Antarctica/Davis differ diff --git a/lib/pytz/zoneinfo/Antarctica/DumontDUrville b/lib/pytz/zoneinfo/Antarctica/DumontDUrville new file mode 100644 index 00000000..c406b8d5 Binary files /dev/null and b/lib/pytz/zoneinfo/Antarctica/DumontDUrville differ diff --git a/lib/pytz/zoneinfo/Antarctica/Macquarie b/lib/pytz/zoneinfo/Antarctica/Macquarie new file mode 100644 index 00000000..fc7b96fe Binary files /dev/null and b/lib/pytz/zoneinfo/Antarctica/Macquarie differ diff --git a/lib/pytz/zoneinfo/Antarctica/Mawson b/lib/pytz/zoneinfo/Antarctica/Mawson new file mode 100644 index 00000000..6c5b0fa1 Binary files /dev/null and b/lib/pytz/zoneinfo/Antarctica/Mawson differ diff --git a/lib/pytz/zoneinfo/Antarctica/McMurdo b/lib/pytz/zoneinfo/Antarctica/McMurdo new file mode 100644 index 00000000..a5f5b6d5 Binary files /dev/null and b/lib/pytz/zoneinfo/Antarctica/McMurdo differ diff --git a/lib/pytz/zoneinfo/Antarctica/Palmer b/lib/pytz/zoneinfo/Antarctica/Palmer new file mode 100644 index 00000000..9e9cdd0b Binary files /dev/null and b/lib/pytz/zoneinfo/Antarctica/Palmer differ diff --git a/lib/pytz/zoneinfo/Antarctica/Rothera b/lib/pytz/zoneinfo/Antarctica/Rothera new file mode 100644 index 00000000..28f82baa Binary files /dev/null and b/lib/pytz/zoneinfo/Antarctica/Rothera differ diff --git a/lib/pytz/zoneinfo/Antarctica/South_Pole b/lib/pytz/zoneinfo/Antarctica/South_Pole new file mode 100644 index 00000000..a5f5b6d5 Binary files /dev/null and b/lib/pytz/zoneinfo/Antarctica/South_Pole differ diff --git a/lib/pytz/zoneinfo/Antarctica/Syowa b/lib/pytz/zoneinfo/Antarctica/Syowa new file mode 100644 index 00000000..b837b071 Binary files /dev/null and b/lib/pytz/zoneinfo/Antarctica/Syowa differ diff --git a/lib/pytz/zoneinfo/Antarctica/Troll b/lib/pytz/zoneinfo/Antarctica/Troll new file mode 100644 index 00000000..d973a122 Binary files /dev/null and b/lib/pytz/zoneinfo/Antarctica/Troll differ diff --git a/lib/pytz/zoneinfo/Antarctica/Vostok b/lib/pytz/zoneinfo/Antarctica/Vostok new file mode 100644 index 00000000..cbec909e Binary files /dev/null and b/lib/pytz/zoneinfo/Antarctica/Vostok differ diff --git a/lib/pytz/zoneinfo/Arctic/Longyearbyen b/lib/pytz/zoneinfo/Arctic/Longyearbyen new file mode 100644 index 00000000..239c0174 Binary files /dev/null and b/lib/pytz/zoneinfo/Arctic/Longyearbyen differ diff --git a/lib/pytz/zoneinfo/Asia/Aden b/lib/pytz/zoneinfo/Asia/Aden new file mode 100644 index 00000000..505e1d22 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Aden differ diff --git a/lib/pytz/zoneinfo/Asia/Almaty b/lib/pytz/zoneinfo/Asia/Almaty new file mode 100644 index 00000000..75a007de Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Almaty differ diff --git a/lib/pytz/zoneinfo/Asia/Amman b/lib/pytz/zoneinfo/Asia/Amman new file mode 100644 index 00000000..c3f0994a Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Amman differ diff --git a/lib/pytz/zoneinfo/Asia/Anadyr b/lib/pytz/zoneinfo/Asia/Anadyr new file mode 100644 index 00000000..766594bc Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Anadyr differ diff --git a/lib/pytz/zoneinfo/Asia/Aqtau b/lib/pytz/zoneinfo/Asia/Aqtau new file mode 100644 index 00000000..811ed2f9 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Aqtau differ diff --git a/lib/pytz/zoneinfo/Asia/Aqtobe b/lib/pytz/zoneinfo/Asia/Aqtobe new file mode 100644 index 00000000..ff3b96b3 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Aqtobe differ diff --git a/lib/pytz/zoneinfo/Asia/Ashgabat b/lib/pytz/zoneinfo/Asia/Ashgabat new file mode 100644 index 00000000..f79fe046 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Ashgabat differ diff --git a/lib/pytz/zoneinfo/Asia/Ashkhabad b/lib/pytz/zoneinfo/Asia/Ashkhabad new file mode 100644 index 00000000..f79fe046 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Ashkhabad differ diff --git a/lib/pytz/zoneinfo/Asia/Baghdad b/lib/pytz/zoneinfo/Asia/Baghdad new file mode 100644 index 00000000..f0a96ec3 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Baghdad differ diff --git a/lib/pytz/zoneinfo/Asia/Bahrain b/lib/pytz/zoneinfo/Asia/Bahrain new file mode 100644 index 00000000..cda04a15 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Bahrain differ diff --git a/lib/pytz/zoneinfo/Asia/Baku b/lib/pytz/zoneinfo/Asia/Baku new file mode 100644 index 00000000..f78e7645 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Baku differ diff --git a/lib/pytz/zoneinfo/Asia/Bangkok b/lib/pytz/zoneinfo/Asia/Bangkok new file mode 100644 index 00000000..e8e76276 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Bangkok differ diff --git a/lib/pytz/zoneinfo/Asia/Beirut b/lib/pytz/zoneinfo/Asia/Beirut new file mode 100644 index 00000000..72f08963 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Beirut differ diff --git a/lib/pytz/zoneinfo/Asia/Bishkek b/lib/pytz/zoneinfo/Asia/Bishkek new file mode 100644 index 00000000..eee8278a Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Bishkek differ diff --git a/lib/pytz/zoneinfo/Asia/Brunei b/lib/pytz/zoneinfo/Asia/Brunei new file mode 100644 index 00000000..1ac3115a Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Brunei differ diff --git a/lib/pytz/zoneinfo/Asia/Calcutta b/lib/pytz/zoneinfo/Asia/Calcutta new file mode 100644 index 00000000..3c0d5abc Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Calcutta differ diff --git a/lib/pytz/zoneinfo/Asia/Chita b/lib/pytz/zoneinfo/Asia/Chita new file mode 100644 index 00000000..c0906547 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Chita differ diff --git a/lib/pytz/zoneinfo/Asia/Choibalsan b/lib/pytz/zoneinfo/Asia/Choibalsan new file mode 100644 index 00000000..f0990926 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Choibalsan differ diff --git a/lib/pytz/zoneinfo/Asia/Chongqing b/lib/pytz/zoneinfo/Asia/Chongqing new file mode 100644 index 00000000..dbd132f2 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Chongqing differ diff --git a/lib/pytz/zoneinfo/Asia/Chungking b/lib/pytz/zoneinfo/Asia/Chungking new file mode 100644 index 00000000..dbd132f2 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Chungking differ diff --git a/lib/pytz/zoneinfo/Asia/Colombo b/lib/pytz/zoneinfo/Asia/Colombo new file mode 100644 index 00000000..d10439af Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Colombo differ diff --git a/lib/pytz/zoneinfo/Asia/Dacca b/lib/pytz/zoneinfo/Asia/Dacca new file mode 100644 index 00000000..b6b326b2 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Dacca differ diff --git a/lib/pytz/zoneinfo/Asia/Damascus b/lib/pytz/zoneinfo/Asia/Damascus new file mode 100644 index 00000000..ac457646 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Damascus differ diff --git a/lib/pytz/zoneinfo/Asia/Dhaka b/lib/pytz/zoneinfo/Asia/Dhaka new file mode 100644 index 00000000..b6b326b2 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Dhaka differ diff --git a/lib/pytz/zoneinfo/Asia/Dili b/lib/pytz/zoneinfo/Asia/Dili new file mode 100644 index 00000000..8124fb70 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Dili differ diff --git a/lib/pytz/zoneinfo/Asia/Dubai b/lib/pytz/zoneinfo/Asia/Dubai new file mode 100644 index 00000000..415e443c Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Dubai differ diff --git a/lib/pytz/zoneinfo/Asia/Dushanbe b/lib/pytz/zoneinfo/Asia/Dushanbe new file mode 100644 index 00000000..3b1e978b Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Dushanbe differ diff --git a/lib/pytz/zoneinfo/Asia/Gaza b/lib/pytz/zoneinfo/Asia/Gaza new file mode 100644 index 00000000..bd683e83 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Gaza differ diff --git a/lib/pytz/zoneinfo/Asia/Harbin b/lib/pytz/zoneinfo/Asia/Harbin new file mode 100644 index 00000000..dbd132f2 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Harbin differ diff --git a/lib/pytz/zoneinfo/Asia/Hebron b/lib/pytz/zoneinfo/Asia/Hebron new file mode 100644 index 00000000..0bc7674b Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Hebron differ diff --git a/lib/pytz/zoneinfo/Asia/Ho_Chi_Minh b/lib/pytz/zoneinfo/Asia/Ho_Chi_Minh new file mode 100644 index 00000000..86fff6b9 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Ho_Chi_Minh differ diff --git a/lib/pytz/zoneinfo/Asia/Hong_Kong b/lib/pytz/zoneinfo/Asia/Hong_Kong new file mode 100644 index 00000000..dc9058e4 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Hong_Kong differ diff --git a/lib/pytz/zoneinfo/Asia/Hovd b/lib/pytz/zoneinfo/Asia/Hovd new file mode 100644 index 00000000..71c3cad4 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Hovd differ diff --git a/lib/pytz/zoneinfo/Asia/Irkutsk b/lib/pytz/zoneinfo/Asia/Irkutsk new file mode 100644 index 00000000..1e94a479 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Irkutsk differ diff --git a/lib/pytz/zoneinfo/Asia/Istanbul b/lib/pytz/zoneinfo/Asia/Istanbul new file mode 100644 index 00000000..d89aa3a8 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Istanbul differ diff --git a/lib/pytz/zoneinfo/Asia/Jakarta b/lib/pytz/zoneinfo/Asia/Jakarta new file mode 100644 index 00000000..3130bff5 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Jakarta differ diff --git a/lib/pytz/zoneinfo/Asia/Jayapura b/lib/pytz/zoneinfo/Asia/Jayapura new file mode 100644 index 00000000..a9d12177 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Jayapura differ diff --git a/lib/pytz/zoneinfo/Asia/Jerusalem b/lib/pytz/zoneinfo/Asia/Jerusalem new file mode 100644 index 00000000..df511993 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Jerusalem differ diff --git a/lib/pytz/zoneinfo/Asia/Kabul b/lib/pytz/zoneinfo/Asia/Kabul new file mode 100644 index 00000000..266cc7e9 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Kabul differ diff --git a/lib/pytz/zoneinfo/Asia/Kamchatka b/lib/pytz/zoneinfo/Asia/Kamchatka new file mode 100644 index 00000000..a0541cfa Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Kamchatka differ diff --git a/lib/pytz/zoneinfo/Asia/Karachi b/lib/pytz/zoneinfo/Asia/Karachi new file mode 100644 index 00000000..6a6de1b2 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Karachi differ diff --git a/lib/pytz/zoneinfo/Asia/Kashgar b/lib/pytz/zoneinfo/Asia/Kashgar new file mode 100644 index 00000000..964a5c24 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Kashgar differ diff --git a/lib/pytz/zoneinfo/Asia/Kathmandu b/lib/pytz/zoneinfo/Asia/Kathmandu new file mode 100644 index 00000000..28247098 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Kathmandu differ diff --git a/lib/pytz/zoneinfo/Asia/Katmandu b/lib/pytz/zoneinfo/Asia/Katmandu new file mode 100644 index 00000000..28247098 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Katmandu differ diff --git a/lib/pytz/zoneinfo/Asia/Khandyga b/lib/pytz/zoneinfo/Asia/Khandyga new file mode 100644 index 00000000..26becb32 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Khandyga differ diff --git a/lib/pytz/zoneinfo/Asia/Kolkata b/lib/pytz/zoneinfo/Asia/Kolkata new file mode 100644 index 00000000..3c0d5abc Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Kolkata differ diff --git a/lib/pytz/zoneinfo/Asia/Krasnoyarsk b/lib/pytz/zoneinfo/Asia/Krasnoyarsk new file mode 100644 index 00000000..31078090 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Krasnoyarsk differ diff --git a/lib/pytz/zoneinfo/Asia/Kuala_Lumpur b/lib/pytz/zoneinfo/Asia/Kuala_Lumpur new file mode 100644 index 00000000..35b987d2 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Kuala_Lumpur differ diff --git a/lib/pytz/zoneinfo/Asia/Kuching b/lib/pytz/zoneinfo/Asia/Kuching new file mode 100644 index 00000000..4f891db7 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Kuching differ diff --git a/lib/pytz/zoneinfo/Asia/Kuwait b/lib/pytz/zoneinfo/Asia/Kuwait new file mode 100644 index 00000000..5623811d Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Kuwait differ diff --git a/lib/pytz/zoneinfo/Asia/Macao b/lib/pytz/zoneinfo/Asia/Macao new file mode 100644 index 00000000..b8f9c369 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Macao differ diff --git a/lib/pytz/zoneinfo/Asia/Macau b/lib/pytz/zoneinfo/Asia/Macau new file mode 100644 index 00000000..b8f9c369 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Macau differ diff --git a/lib/pytz/zoneinfo/Asia/Magadan b/lib/pytz/zoneinfo/Asia/Magadan new file mode 100644 index 00000000..e09c4dc2 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Magadan differ diff --git a/lib/pytz/zoneinfo/Asia/Makassar b/lib/pytz/zoneinfo/Asia/Makassar new file mode 100644 index 00000000..0d689236 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Makassar differ diff --git a/lib/pytz/zoneinfo/Asia/Manila b/lib/pytz/zoneinfo/Asia/Manila new file mode 100644 index 00000000..ac0f3a63 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Manila differ diff --git a/lib/pytz/zoneinfo/Asia/Muscat b/lib/pytz/zoneinfo/Asia/Muscat new file mode 100644 index 00000000..53a22190 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Muscat differ diff --git a/lib/pytz/zoneinfo/Asia/Nicosia b/lib/pytz/zoneinfo/Asia/Nicosia new file mode 100644 index 00000000..3e663b21 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Nicosia differ diff --git a/lib/pytz/zoneinfo/Asia/Novokuznetsk b/lib/pytz/zoneinfo/Asia/Novokuznetsk new file mode 100644 index 00000000..c5cadc0e Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Novokuznetsk differ diff --git a/lib/pytz/zoneinfo/Asia/Novosibirsk b/lib/pytz/zoneinfo/Asia/Novosibirsk new file mode 100644 index 00000000..ed6d7dc5 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Novosibirsk differ diff --git a/lib/pytz/zoneinfo/Asia/Omsk b/lib/pytz/zoneinfo/Asia/Omsk new file mode 100644 index 00000000..760c3910 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Omsk differ diff --git a/lib/pytz/zoneinfo/Asia/Oral b/lib/pytz/zoneinfo/Asia/Oral new file mode 100644 index 00000000..1467cafc Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Oral differ diff --git a/lib/pytz/zoneinfo/Asia/Phnom_Penh b/lib/pytz/zoneinfo/Asia/Phnom_Penh new file mode 100644 index 00000000..37c9e15f Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Phnom_Penh differ diff --git a/lib/pytz/zoneinfo/Asia/Pontianak b/lib/pytz/zoneinfo/Asia/Pontianak new file mode 100644 index 00000000..dcd70140 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Pontianak differ diff --git a/lib/pytz/zoneinfo/Asia/Pyongyang b/lib/pytz/zoneinfo/Asia/Pyongyang new file mode 100644 index 00000000..a743fbba Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Pyongyang differ diff --git a/lib/pytz/zoneinfo/Asia/Qatar b/lib/pytz/zoneinfo/Asia/Qatar new file mode 100644 index 00000000..3e203739 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Qatar differ diff --git a/lib/pytz/zoneinfo/Asia/Qyzylorda b/lib/pytz/zoneinfo/Asia/Qyzylorda new file mode 100644 index 00000000..ce535161 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Qyzylorda differ diff --git a/lib/pytz/zoneinfo/Asia/Rangoon b/lib/pytz/zoneinfo/Asia/Rangoon new file mode 100644 index 00000000..934ca7ef Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Rangoon differ diff --git a/lib/pytz/zoneinfo/Asia/Riyadh b/lib/pytz/zoneinfo/Asia/Riyadh new file mode 100644 index 00000000..c35e42a1 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Riyadh differ diff --git a/lib/pytz/zoneinfo/Asia/Saigon b/lib/pytz/zoneinfo/Asia/Saigon new file mode 100644 index 00000000..86fff6b9 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Saigon differ diff --git a/lib/pytz/zoneinfo/Asia/Sakhalin b/lib/pytz/zoneinfo/Asia/Sakhalin new file mode 100644 index 00000000..ec62afc5 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Sakhalin differ diff --git a/lib/pytz/zoneinfo/Asia/Samarkand b/lib/pytz/zoneinfo/Asia/Samarkand new file mode 100644 index 00000000..65fb5b03 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Samarkand differ diff --git a/lib/pytz/zoneinfo/Asia/Seoul b/lib/pytz/zoneinfo/Asia/Seoul new file mode 100644 index 00000000..6931d782 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Seoul differ diff --git a/lib/pytz/zoneinfo/Asia/Shanghai b/lib/pytz/zoneinfo/Asia/Shanghai new file mode 100644 index 00000000..dbd132f2 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Shanghai differ diff --git a/lib/pytz/zoneinfo/Asia/Singapore b/lib/pytz/zoneinfo/Asia/Singapore new file mode 100644 index 00000000..9dd49cb7 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Singapore differ diff --git a/lib/pytz/zoneinfo/Asia/Srednekolymsk b/lib/pytz/zoneinfo/Asia/Srednekolymsk new file mode 100644 index 00000000..0929f66d Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Srednekolymsk differ diff --git a/lib/pytz/zoneinfo/Asia/Taipei b/lib/pytz/zoneinfo/Asia/Taipei new file mode 100644 index 00000000..4810a0b6 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Taipei differ diff --git a/lib/pytz/zoneinfo/Asia/Tashkent b/lib/pytz/zoneinfo/Asia/Tashkent new file mode 100644 index 00000000..1f59faa5 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Tashkent differ diff --git a/lib/pytz/zoneinfo/Asia/Tbilisi b/lib/pytz/zoneinfo/Asia/Tbilisi new file mode 100644 index 00000000..0d7081e9 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Tbilisi differ diff --git a/lib/pytz/zoneinfo/Asia/Tehran b/lib/pytz/zoneinfo/Asia/Tehran new file mode 100644 index 00000000..87107811 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Tehran differ diff --git a/lib/pytz/zoneinfo/Asia/Tel_Aviv b/lib/pytz/zoneinfo/Asia/Tel_Aviv new file mode 100644 index 00000000..df511993 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Tel_Aviv differ diff --git a/lib/pytz/zoneinfo/Asia/Thimbu b/lib/pytz/zoneinfo/Asia/Thimbu new file mode 100644 index 00000000..0bd94cb4 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Thimbu differ diff --git a/lib/pytz/zoneinfo/Asia/Thimphu b/lib/pytz/zoneinfo/Asia/Thimphu new file mode 100644 index 00000000..0bd94cb4 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Thimphu differ diff --git a/lib/pytz/zoneinfo/Asia/Tokyo b/lib/pytz/zoneinfo/Asia/Tokyo new file mode 100644 index 00000000..02441403 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Tokyo differ diff --git a/lib/pytz/zoneinfo/Asia/Ujung_Pandang b/lib/pytz/zoneinfo/Asia/Ujung_Pandang new file mode 100644 index 00000000..0d689236 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Ujung_Pandang differ diff --git a/lib/pytz/zoneinfo/Asia/Ulaanbaatar b/lib/pytz/zoneinfo/Asia/Ulaanbaatar new file mode 100644 index 00000000..61505e95 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Ulaanbaatar differ diff --git a/lib/pytz/zoneinfo/Asia/Ulan_Bator b/lib/pytz/zoneinfo/Asia/Ulan_Bator new file mode 100644 index 00000000..61505e95 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Ulan_Bator differ diff --git a/lib/pytz/zoneinfo/Asia/Urumqi b/lib/pytz/zoneinfo/Asia/Urumqi new file mode 100644 index 00000000..964a5c24 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Urumqi differ diff --git a/lib/pytz/zoneinfo/Asia/Ust-Nera b/lib/pytz/zoneinfo/Asia/Ust-Nera new file mode 100644 index 00000000..0efacd6b Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Ust-Nera differ diff --git a/lib/pytz/zoneinfo/Asia/Vientiane b/lib/pytz/zoneinfo/Asia/Vientiane new file mode 100644 index 00000000..67e90e0c Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Vientiane differ diff --git a/lib/pytz/zoneinfo/Asia/Vladivostok b/lib/pytz/zoneinfo/Asia/Vladivostok new file mode 100644 index 00000000..156c8e6f Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Vladivostok differ diff --git a/lib/pytz/zoneinfo/Asia/Yakutsk b/lib/pytz/zoneinfo/Asia/Yakutsk new file mode 100644 index 00000000..58ff25ea Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Yakutsk differ diff --git a/lib/pytz/zoneinfo/Asia/Yekaterinburg b/lib/pytz/zoneinfo/Asia/Yekaterinburg new file mode 100644 index 00000000..a1baafae Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Yekaterinburg differ diff --git a/lib/pytz/zoneinfo/Asia/Yerevan b/lib/pytz/zoneinfo/Asia/Yerevan new file mode 100644 index 00000000..fa62c249 Binary files /dev/null and b/lib/pytz/zoneinfo/Asia/Yerevan differ diff --git a/lib/pytz/zoneinfo/Atlantic/Azores b/lib/pytz/zoneinfo/Atlantic/Azores new file mode 100644 index 00000000..1f532532 Binary files /dev/null and b/lib/pytz/zoneinfo/Atlantic/Azores differ diff --git a/lib/pytz/zoneinfo/Atlantic/Bermuda b/lib/pytz/zoneinfo/Atlantic/Bermuda new file mode 100644 index 00000000..548d979b Binary files /dev/null and b/lib/pytz/zoneinfo/Atlantic/Bermuda differ diff --git a/lib/pytz/zoneinfo/Atlantic/Canary b/lib/pytz/zoneinfo/Atlantic/Canary new file mode 100644 index 00000000..007dcf49 Binary files /dev/null and b/lib/pytz/zoneinfo/Atlantic/Canary differ diff --git a/lib/pytz/zoneinfo/Atlantic/Cape_Verde b/lib/pytz/zoneinfo/Atlantic/Cape_Verde new file mode 100644 index 00000000..18b676ce Binary files /dev/null and b/lib/pytz/zoneinfo/Atlantic/Cape_Verde differ diff --git a/lib/pytz/zoneinfo/Atlantic/Faeroe b/lib/pytz/zoneinfo/Atlantic/Faeroe new file mode 100644 index 00000000..c4865186 Binary files /dev/null and b/lib/pytz/zoneinfo/Atlantic/Faeroe differ diff --git a/lib/pytz/zoneinfo/Atlantic/Faroe b/lib/pytz/zoneinfo/Atlantic/Faroe new file mode 100644 index 00000000..c4865186 Binary files /dev/null and b/lib/pytz/zoneinfo/Atlantic/Faroe differ diff --git a/lib/pytz/zoneinfo/Atlantic/Jan_Mayen b/lib/pytz/zoneinfo/Atlantic/Jan_Mayen new file mode 100644 index 00000000..239c0174 Binary files /dev/null and b/lib/pytz/zoneinfo/Atlantic/Jan_Mayen differ diff --git a/lib/pytz/zoneinfo/Atlantic/Madeira b/lib/pytz/zoneinfo/Atlantic/Madeira new file mode 100644 index 00000000..3687fd66 Binary files /dev/null and b/lib/pytz/zoneinfo/Atlantic/Madeira differ diff --git a/lib/pytz/zoneinfo/Atlantic/Reykjavik b/lib/pytz/zoneinfo/Atlantic/Reykjavik new file mode 100644 index 00000000..35ba7a15 Binary files /dev/null and b/lib/pytz/zoneinfo/Atlantic/Reykjavik differ diff --git a/lib/pytz/zoneinfo/Atlantic/South_Georgia b/lib/pytz/zoneinfo/Atlantic/South_Georgia new file mode 100644 index 00000000..b1191c9f Binary files /dev/null and b/lib/pytz/zoneinfo/Atlantic/South_Georgia differ diff --git a/lib/pytz/zoneinfo/Atlantic/St_Helena b/lib/pytz/zoneinfo/Atlantic/St_Helena new file mode 100644 index 00000000..6fd1af32 Binary files /dev/null and b/lib/pytz/zoneinfo/Atlantic/St_Helena differ diff --git a/lib/pytz/zoneinfo/Atlantic/Stanley b/lib/pytz/zoneinfo/Atlantic/Stanley new file mode 100644 index 00000000..aec7a5d3 Binary files /dev/null and b/lib/pytz/zoneinfo/Atlantic/Stanley differ diff --git a/lib/pytz/zoneinfo/Australia/ACT b/lib/pytz/zoneinfo/Australia/ACT new file mode 100644 index 00000000..aaed12ca Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/ACT differ diff --git a/lib/pytz/zoneinfo/Australia/Adelaide b/lib/pytz/zoneinfo/Australia/Adelaide new file mode 100644 index 00000000..4f331a87 Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/Adelaide differ diff --git a/lib/pytz/zoneinfo/Australia/Brisbane b/lib/pytz/zoneinfo/Australia/Brisbane new file mode 100644 index 00000000..a327d83b Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/Brisbane differ diff --git a/lib/pytz/zoneinfo/Australia/Broken_Hill b/lib/pytz/zoneinfo/Australia/Broken_Hill new file mode 100644 index 00000000..768b1678 Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/Broken_Hill differ diff --git a/lib/pytz/zoneinfo/Australia/Canberra b/lib/pytz/zoneinfo/Australia/Canberra new file mode 100644 index 00000000..aaed12ca Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/Canberra differ diff --git a/lib/pytz/zoneinfo/Australia/Currie b/lib/pytz/zoneinfo/Australia/Currie new file mode 100644 index 00000000..a3f6f29a Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/Currie differ diff --git a/lib/pytz/zoneinfo/Australia/Darwin b/lib/pytz/zoneinfo/Australia/Darwin new file mode 100644 index 00000000..c6ae9a7b Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/Darwin differ diff --git a/lib/pytz/zoneinfo/Australia/Eucla b/lib/pytz/zoneinfo/Australia/Eucla new file mode 100644 index 00000000..baba07a3 Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/Eucla differ diff --git a/lib/pytz/zoneinfo/Australia/Hobart b/lib/pytz/zoneinfo/Australia/Hobart new file mode 100644 index 00000000..07784ce5 Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/Hobart differ diff --git a/lib/pytz/zoneinfo/Australia/LHI b/lib/pytz/zoneinfo/Australia/LHI new file mode 100644 index 00000000..a653e516 Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/LHI differ diff --git a/lib/pytz/zoneinfo/Australia/Lindeman b/lib/pytz/zoneinfo/Australia/Lindeman new file mode 100644 index 00000000..71ca143f Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/Lindeman differ diff --git a/lib/pytz/zoneinfo/Australia/Lord_Howe b/lib/pytz/zoneinfo/Australia/Lord_Howe new file mode 100644 index 00000000..a653e516 Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/Lord_Howe differ diff --git a/lib/pytz/zoneinfo/Australia/Melbourne b/lib/pytz/zoneinfo/Australia/Melbourne new file mode 100644 index 00000000..ec8dfe03 Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/Melbourne differ diff --git a/lib/pytz/zoneinfo/Australia/NSW b/lib/pytz/zoneinfo/Australia/NSW new file mode 100644 index 00000000..aaed12ca Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/NSW differ diff --git a/lib/pytz/zoneinfo/Australia/North b/lib/pytz/zoneinfo/Australia/North new file mode 100644 index 00000000..c6ae9a7b Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/North differ diff --git a/lib/pytz/zoneinfo/Australia/Perth b/lib/pytz/zoneinfo/Australia/Perth new file mode 100644 index 00000000..85c26d50 Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/Perth differ diff --git a/lib/pytz/zoneinfo/Australia/Queensland b/lib/pytz/zoneinfo/Australia/Queensland new file mode 100644 index 00000000..a327d83b Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/Queensland differ diff --git a/lib/pytz/zoneinfo/Australia/South b/lib/pytz/zoneinfo/Australia/South new file mode 100644 index 00000000..4f331a87 Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/South differ diff --git a/lib/pytz/zoneinfo/Australia/Sydney b/lib/pytz/zoneinfo/Australia/Sydney new file mode 100644 index 00000000..aaed12ca Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/Sydney differ diff --git a/lib/pytz/zoneinfo/Australia/Tasmania b/lib/pytz/zoneinfo/Australia/Tasmania new file mode 100644 index 00000000..07784ce5 Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/Tasmania differ diff --git a/lib/pytz/zoneinfo/Australia/Victoria b/lib/pytz/zoneinfo/Australia/Victoria new file mode 100644 index 00000000..ec8dfe03 Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/Victoria differ diff --git a/lib/pytz/zoneinfo/Australia/West b/lib/pytz/zoneinfo/Australia/West new file mode 100644 index 00000000..85c26d50 Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/West differ diff --git a/lib/pytz/zoneinfo/Australia/Yancowinna b/lib/pytz/zoneinfo/Australia/Yancowinna new file mode 100644 index 00000000..768b1678 Binary files /dev/null and b/lib/pytz/zoneinfo/Australia/Yancowinna differ diff --git a/lib/pytz/zoneinfo/Brazil/Acre b/lib/pytz/zoneinfo/Brazil/Acre new file mode 100644 index 00000000..788d0e9c Binary files /dev/null and b/lib/pytz/zoneinfo/Brazil/Acre differ diff --git a/lib/pytz/zoneinfo/Brazil/DeNoronha b/lib/pytz/zoneinfo/Brazil/DeNoronha new file mode 100644 index 00000000..774b14e6 Binary files /dev/null and b/lib/pytz/zoneinfo/Brazil/DeNoronha differ diff --git a/lib/pytz/zoneinfo/Brazil/East b/lib/pytz/zoneinfo/Brazil/East new file mode 100644 index 00000000..552ce7c2 Binary files /dev/null and b/lib/pytz/zoneinfo/Brazil/East differ diff --git a/lib/pytz/zoneinfo/Brazil/West b/lib/pytz/zoneinfo/Brazil/West new file mode 100644 index 00000000..e0222f18 Binary files /dev/null and b/lib/pytz/zoneinfo/Brazil/West differ diff --git a/lib/pytz/zoneinfo/CET b/lib/pytz/zoneinfo/CET new file mode 100644 index 00000000..4c4f8ef9 Binary files /dev/null and b/lib/pytz/zoneinfo/CET differ diff --git a/lib/pytz/zoneinfo/CST6CDT b/lib/pytz/zoneinfo/CST6CDT new file mode 100644 index 00000000..5c8a1d9a Binary files /dev/null and b/lib/pytz/zoneinfo/CST6CDT differ diff --git a/lib/pytz/zoneinfo/Canada/Atlantic b/lib/pytz/zoneinfo/Canada/Atlantic new file mode 100644 index 00000000..f86ece4c Binary files /dev/null and b/lib/pytz/zoneinfo/Canada/Atlantic differ diff --git a/lib/pytz/zoneinfo/Canada/Central b/lib/pytz/zoneinfo/Canada/Central new file mode 100644 index 00000000..2ffe3d8d Binary files /dev/null and b/lib/pytz/zoneinfo/Canada/Central differ diff --git a/lib/pytz/zoneinfo/Canada/East-Saskatchewan b/lib/pytz/zoneinfo/Canada/East-Saskatchewan new file mode 100644 index 00000000..5fe8d6b6 Binary files /dev/null and b/lib/pytz/zoneinfo/Canada/East-Saskatchewan differ diff --git a/lib/pytz/zoneinfo/Canada/Eastern b/lib/pytz/zoneinfo/Canada/Eastern new file mode 100644 index 00000000..7b4682a3 Binary files /dev/null and b/lib/pytz/zoneinfo/Canada/Eastern differ diff --git a/lib/pytz/zoneinfo/Canada/Mountain b/lib/pytz/zoneinfo/Canada/Mountain new file mode 100644 index 00000000..d02fbcd4 Binary files /dev/null and b/lib/pytz/zoneinfo/Canada/Mountain differ diff --git a/lib/pytz/zoneinfo/Canada/Newfoundland b/lib/pytz/zoneinfo/Canada/Newfoundland new file mode 100644 index 00000000..a1d14854 Binary files /dev/null and b/lib/pytz/zoneinfo/Canada/Newfoundland differ diff --git a/lib/pytz/zoneinfo/Canada/Pacific b/lib/pytz/zoneinfo/Canada/Pacific new file mode 100644 index 00000000..9b5d9241 Binary files /dev/null and b/lib/pytz/zoneinfo/Canada/Pacific differ diff --git a/lib/pytz/zoneinfo/Canada/Saskatchewan b/lib/pytz/zoneinfo/Canada/Saskatchewan new file mode 100644 index 00000000..5fe8d6b6 Binary files /dev/null and b/lib/pytz/zoneinfo/Canada/Saskatchewan differ diff --git a/lib/pytz/zoneinfo/Canada/Yukon b/lib/pytz/zoneinfo/Canada/Yukon new file mode 100644 index 00000000..8604c5c5 Binary files /dev/null and b/lib/pytz/zoneinfo/Canada/Yukon differ diff --git a/lib/pytz/zoneinfo/Chile/Continental b/lib/pytz/zoneinfo/Chile/Continental new file mode 100644 index 00000000..92cf5597 Binary files /dev/null and b/lib/pytz/zoneinfo/Chile/Continental differ diff --git a/lib/pytz/zoneinfo/Chile/EasterIsland b/lib/pytz/zoneinfo/Chile/EasterIsland new file mode 100644 index 00000000..8c8a6c7d Binary files /dev/null and b/lib/pytz/zoneinfo/Chile/EasterIsland differ diff --git a/lib/pytz/zoneinfo/Cuba b/lib/pytz/zoneinfo/Cuba new file mode 100644 index 00000000..1a58fcdc Binary files /dev/null and b/lib/pytz/zoneinfo/Cuba differ diff --git a/lib/pytz/zoneinfo/EET b/lib/pytz/zoneinfo/EET new file mode 100644 index 00000000..beb273a2 Binary files /dev/null and b/lib/pytz/zoneinfo/EET differ diff --git a/lib/pytz/zoneinfo/EST b/lib/pytz/zoneinfo/EST new file mode 100644 index 00000000..ae346633 Binary files /dev/null and b/lib/pytz/zoneinfo/EST differ diff --git a/lib/pytz/zoneinfo/EST5EDT b/lib/pytz/zoneinfo/EST5EDT new file mode 100644 index 00000000..54541fc2 Binary files /dev/null and b/lib/pytz/zoneinfo/EST5EDT differ diff --git a/lib/pytz/zoneinfo/Egypt b/lib/pytz/zoneinfo/Egypt new file mode 100644 index 00000000..0eeed113 Binary files /dev/null and b/lib/pytz/zoneinfo/Egypt differ diff --git a/lib/pytz/zoneinfo/Eire b/lib/pytz/zoneinfo/Eire new file mode 100644 index 00000000..a7cffbbb Binary files /dev/null and b/lib/pytz/zoneinfo/Eire differ diff --git a/lib/pytz/zoneinfo/Etc/GMT b/lib/pytz/zoneinfo/Etc/GMT new file mode 100644 index 00000000..c05e45fd Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT differ diff --git a/lib/pytz/zoneinfo/Etc/GMT+0 b/lib/pytz/zoneinfo/Etc/GMT+0 new file mode 100644 index 00000000..c05e45fd Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT+0 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT+1 b/lib/pytz/zoneinfo/Etc/GMT+1 new file mode 100644 index 00000000..2f40cc76 Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT+1 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT+10 b/lib/pytz/zoneinfo/Etc/GMT+10 new file mode 100644 index 00000000..2087965e Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT+10 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT+11 b/lib/pytz/zoneinfo/Etc/GMT+11 new file mode 100644 index 00000000..af4a6b34 Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT+11 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT+12 b/lib/pytz/zoneinfo/Etc/GMT+12 new file mode 100644 index 00000000..f0955345 Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT+12 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT+2 b/lib/pytz/zoneinfo/Etc/GMT+2 new file mode 100644 index 00000000..85a1fc1d Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT+2 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT+3 b/lib/pytz/zoneinfo/Etc/GMT+3 new file mode 100644 index 00000000..a24f5870 Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT+3 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT+4 b/lib/pytz/zoneinfo/Etc/GMT+4 new file mode 100644 index 00000000..ab745174 Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT+4 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT+5 b/lib/pytz/zoneinfo/Etc/GMT+5 new file mode 100644 index 00000000..01f1d775 Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT+5 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT+6 b/lib/pytz/zoneinfo/Etc/GMT+6 new file mode 100644 index 00000000..3ced48bb Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT+6 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT+7 b/lib/pytz/zoneinfo/Etc/GMT+7 new file mode 100644 index 00000000..5f58127e Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT+7 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT+8 b/lib/pytz/zoneinfo/Etc/GMT+8 new file mode 100644 index 00000000..be23d966 Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT+8 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT+9 b/lib/pytz/zoneinfo/Etc/GMT+9 new file mode 100644 index 00000000..d00c50c5 Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT+9 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT-0 b/lib/pytz/zoneinfo/Etc/GMT-0 new file mode 100644 index 00000000..c05e45fd Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT-0 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT-1 b/lib/pytz/zoneinfo/Etc/GMT-1 new file mode 100644 index 00000000..088a76ed Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT-1 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT-10 b/lib/pytz/zoneinfo/Etc/GMT-10 new file mode 100644 index 00000000..a4da44f5 Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT-10 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT-11 b/lib/pytz/zoneinfo/Etc/GMT-11 new file mode 100644 index 00000000..e0112a9c Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT-11 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT-12 b/lib/pytz/zoneinfo/Etc/GMT-12 new file mode 100644 index 00000000..c1e08b77 Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT-12 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT-13 b/lib/pytz/zoneinfo/Etc/GMT-13 new file mode 100644 index 00000000..1ab05199 Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT-13 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT-14 b/lib/pytz/zoneinfo/Etc/GMT-14 new file mode 100644 index 00000000..afaf3fa9 Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT-14 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT-2 b/lib/pytz/zoneinfo/Etc/GMT-2 new file mode 100644 index 00000000..6289cad8 Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT-2 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT-3 b/lib/pytz/zoneinfo/Etc/GMT-3 new file mode 100644 index 00000000..27434cdb Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT-3 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT-4 b/lib/pytz/zoneinfo/Etc/GMT-4 new file mode 100644 index 00000000..2fc69663 Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT-4 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT-5 b/lib/pytz/zoneinfo/Etc/GMT-5 new file mode 100644 index 00000000..8508e723 Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT-5 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT-6 b/lib/pytz/zoneinfo/Etc/GMT-6 new file mode 100644 index 00000000..5b9678ea Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT-6 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT-7 b/lib/pytz/zoneinfo/Etc/GMT-7 new file mode 100644 index 00000000..ccf4c394 Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT-7 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT-8 b/lib/pytz/zoneinfo/Etc/GMT-8 new file mode 100644 index 00000000..db4cfa6a Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT-8 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT-9 b/lib/pytz/zoneinfo/Etc/GMT-9 new file mode 100644 index 00000000..56ea1174 Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT-9 differ diff --git a/lib/pytz/zoneinfo/Etc/GMT0 b/lib/pytz/zoneinfo/Etc/GMT0 new file mode 100644 index 00000000..c05e45fd Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/GMT0 differ diff --git a/lib/pytz/zoneinfo/Etc/Greenwich b/lib/pytz/zoneinfo/Etc/Greenwich new file mode 100644 index 00000000..c05e45fd Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/Greenwich differ diff --git a/lib/pytz/zoneinfo/Etc/UCT b/lib/pytz/zoneinfo/Etc/UCT new file mode 100644 index 00000000..40147b9e Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/UCT differ diff --git a/lib/pytz/zoneinfo/Etc/UTC b/lib/pytz/zoneinfo/Etc/UTC new file mode 100644 index 00000000..c3b97f1a Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/UTC differ diff --git a/lib/pytz/zoneinfo/Etc/Universal b/lib/pytz/zoneinfo/Etc/Universal new file mode 100644 index 00000000..c3b97f1a Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/Universal differ diff --git a/lib/pytz/zoneinfo/Etc/Zulu b/lib/pytz/zoneinfo/Etc/Zulu new file mode 100644 index 00000000..c3b97f1a Binary files /dev/null and b/lib/pytz/zoneinfo/Etc/Zulu differ diff --git a/lib/pytz/zoneinfo/Europe/Amsterdam b/lib/pytz/zoneinfo/Europe/Amsterdam new file mode 100644 index 00000000..f74769d4 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Amsterdam differ diff --git a/lib/pytz/zoneinfo/Europe/Andorra b/lib/pytz/zoneinfo/Europe/Andorra new file mode 100644 index 00000000..b06de7a5 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Andorra differ diff --git a/lib/pytz/zoneinfo/Europe/Athens b/lib/pytz/zoneinfo/Europe/Athens new file mode 100644 index 00000000..0001602f Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Athens differ diff --git a/lib/pytz/zoneinfo/Europe/Belfast b/lib/pytz/zoneinfo/Europe/Belfast new file mode 100644 index 00000000..4527515c Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Belfast differ diff --git a/lib/pytz/zoneinfo/Europe/Belgrade b/lib/pytz/zoneinfo/Europe/Belgrade new file mode 100644 index 00000000..79c25d70 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Belgrade differ diff --git a/lib/pytz/zoneinfo/Europe/Berlin b/lib/pytz/zoneinfo/Europe/Berlin new file mode 100644 index 00000000..b4f2a2af Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Berlin differ diff --git a/lib/pytz/zoneinfo/Europe/Bratislava b/lib/pytz/zoneinfo/Europe/Bratislava new file mode 100644 index 00000000..4eabe5c8 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Bratislava differ diff --git a/lib/pytz/zoneinfo/Europe/Brussels b/lib/pytz/zoneinfo/Europe/Brussels new file mode 100644 index 00000000..d8f19a63 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Brussels differ diff --git a/lib/pytz/zoneinfo/Europe/Bucharest b/lib/pytz/zoneinfo/Europe/Bucharest new file mode 100644 index 00000000..e0eac4ce Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Bucharest differ diff --git a/lib/pytz/zoneinfo/Europe/Budapest b/lib/pytz/zoneinfo/Europe/Budapest new file mode 100644 index 00000000..3ddf6a52 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Budapest differ diff --git a/lib/pytz/zoneinfo/Europe/Busingen b/lib/pytz/zoneinfo/Europe/Busingen new file mode 100644 index 00000000..9c2b600b Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Busingen differ diff --git a/lib/pytz/zoneinfo/Europe/Chisinau b/lib/pytz/zoneinfo/Europe/Chisinau new file mode 100644 index 00000000..7998b2d8 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Chisinau differ diff --git a/lib/pytz/zoneinfo/Europe/Copenhagen b/lib/pytz/zoneinfo/Europe/Copenhagen new file mode 100644 index 00000000..be87cf16 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Copenhagen differ diff --git a/lib/pytz/zoneinfo/Europe/Dublin b/lib/pytz/zoneinfo/Europe/Dublin new file mode 100644 index 00000000..a7cffbbb Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Dublin differ diff --git a/lib/pytz/zoneinfo/Europe/Gibraltar b/lib/pytz/zoneinfo/Europe/Gibraltar new file mode 100644 index 00000000..a7105faa Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Gibraltar differ diff --git a/lib/pytz/zoneinfo/Europe/Guernsey b/lib/pytz/zoneinfo/Europe/Guernsey new file mode 100644 index 00000000..4527515c Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Guernsey differ diff --git a/lib/pytz/zoneinfo/Europe/Helsinki b/lib/pytz/zoneinfo/Europe/Helsinki new file mode 100644 index 00000000..29b3c817 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Helsinki differ diff --git a/lib/pytz/zoneinfo/Europe/Isle_of_Man b/lib/pytz/zoneinfo/Europe/Isle_of_Man new file mode 100644 index 00000000..4527515c Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Isle_of_Man differ diff --git a/lib/pytz/zoneinfo/Europe/Istanbul b/lib/pytz/zoneinfo/Europe/Istanbul new file mode 100644 index 00000000..d89aa3a8 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Istanbul differ diff --git a/lib/pytz/zoneinfo/Europe/Jersey b/lib/pytz/zoneinfo/Europe/Jersey new file mode 100644 index 00000000..4527515c Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Jersey differ diff --git a/lib/pytz/zoneinfo/Europe/Kaliningrad b/lib/pytz/zoneinfo/Europe/Kaliningrad new file mode 100644 index 00000000..4805fe42 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Kaliningrad differ diff --git a/lib/pytz/zoneinfo/Europe/Kiev b/lib/pytz/zoneinfo/Europe/Kiev new file mode 100644 index 00000000..b3e20a7e Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Kiev differ diff --git a/lib/pytz/zoneinfo/Europe/Lisbon b/lib/pytz/zoneinfo/Europe/Lisbon new file mode 100644 index 00000000..b9aff3a5 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Lisbon differ diff --git a/lib/pytz/zoneinfo/Europe/Ljubljana b/lib/pytz/zoneinfo/Europe/Ljubljana new file mode 100644 index 00000000..79c25d70 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Ljubljana differ diff --git a/lib/pytz/zoneinfo/Europe/London b/lib/pytz/zoneinfo/Europe/London new file mode 100644 index 00000000..4527515c Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/London differ diff --git a/lib/pytz/zoneinfo/Europe/Luxembourg b/lib/pytz/zoneinfo/Europe/Luxembourg new file mode 100644 index 00000000..6fae86c5 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Luxembourg differ diff --git a/lib/pytz/zoneinfo/Europe/Madrid b/lib/pytz/zoneinfo/Europe/Madrid new file mode 100644 index 00000000..af474328 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Madrid differ diff --git a/lib/pytz/zoneinfo/Europe/Malta b/lib/pytz/zoneinfo/Europe/Malta new file mode 100644 index 00000000..d2519389 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Malta differ diff --git a/lib/pytz/zoneinfo/Europe/Mariehamn b/lib/pytz/zoneinfo/Europe/Mariehamn new file mode 100644 index 00000000..29b3c817 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Mariehamn differ diff --git a/lib/pytz/zoneinfo/Europe/Minsk b/lib/pytz/zoneinfo/Europe/Minsk new file mode 100644 index 00000000..fa1e2e4e Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Minsk differ diff --git a/lib/pytz/zoneinfo/Europe/Monaco b/lib/pytz/zoneinfo/Europe/Monaco new file mode 100644 index 00000000..0b40f1ec Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Monaco differ diff --git a/lib/pytz/zoneinfo/Europe/Moscow b/lib/pytz/zoneinfo/Europe/Moscow new file mode 100644 index 00000000..bdbbaebe Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Moscow differ diff --git a/lib/pytz/zoneinfo/Europe/Nicosia b/lib/pytz/zoneinfo/Europe/Nicosia new file mode 100644 index 00000000..3e663b21 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Nicosia differ diff --git a/lib/pytz/zoneinfo/Europe/Oslo b/lib/pytz/zoneinfo/Europe/Oslo new file mode 100644 index 00000000..239c0174 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Oslo differ diff --git a/lib/pytz/zoneinfo/Europe/Paris b/lib/pytz/zoneinfo/Europe/Paris new file mode 100644 index 00000000..cf6e2e2e Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Paris differ diff --git a/lib/pytz/zoneinfo/Europe/Podgorica b/lib/pytz/zoneinfo/Europe/Podgorica new file mode 100644 index 00000000..79c25d70 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Podgorica differ diff --git a/lib/pytz/zoneinfo/Europe/Prague b/lib/pytz/zoneinfo/Europe/Prague new file mode 100644 index 00000000..4eabe5c8 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Prague differ diff --git a/lib/pytz/zoneinfo/Europe/Riga b/lib/pytz/zoneinfo/Europe/Riga new file mode 100644 index 00000000..b729ee8c Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Riga differ diff --git a/lib/pytz/zoneinfo/Europe/Rome b/lib/pytz/zoneinfo/Europe/Rome new file mode 100644 index 00000000..5cc30403 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Rome differ diff --git a/lib/pytz/zoneinfo/Europe/Samara b/lib/pytz/zoneinfo/Europe/Samara new file mode 100644 index 00000000..79759f53 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Samara differ diff --git a/lib/pytz/zoneinfo/Europe/San_Marino b/lib/pytz/zoneinfo/Europe/San_Marino new file mode 100644 index 00000000..5cc30403 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/San_Marino differ diff --git a/lib/pytz/zoneinfo/Europe/Sarajevo b/lib/pytz/zoneinfo/Europe/Sarajevo new file mode 100644 index 00000000..79c25d70 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Sarajevo differ diff --git a/lib/pytz/zoneinfo/Europe/Simferopol b/lib/pytz/zoneinfo/Europe/Simferopol new file mode 100644 index 00000000..ebe9017d Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Simferopol differ diff --git a/lib/pytz/zoneinfo/Europe/Skopje b/lib/pytz/zoneinfo/Europe/Skopje new file mode 100644 index 00000000..79c25d70 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Skopje differ diff --git a/lib/pytz/zoneinfo/Europe/Sofia b/lib/pytz/zoneinfo/Europe/Sofia new file mode 100644 index 00000000..763e0747 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Sofia differ diff --git a/lib/pytz/zoneinfo/Europe/Stockholm b/lib/pytz/zoneinfo/Europe/Stockholm new file mode 100644 index 00000000..43c7f2e2 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Stockholm differ diff --git a/lib/pytz/zoneinfo/Europe/Tallinn b/lib/pytz/zoneinfo/Europe/Tallinn new file mode 100644 index 00000000..8a4f1240 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Tallinn differ diff --git a/lib/pytz/zoneinfo/Europe/Tirane b/lib/pytz/zoneinfo/Europe/Tirane new file mode 100644 index 00000000..52c16a42 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Tirane differ diff --git a/lib/pytz/zoneinfo/Europe/Tiraspol b/lib/pytz/zoneinfo/Europe/Tiraspol new file mode 100644 index 00000000..7998b2d8 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Tiraspol differ diff --git a/lib/pytz/zoneinfo/Europe/Uzhgorod b/lib/pytz/zoneinfo/Europe/Uzhgorod new file mode 100644 index 00000000..8ddba909 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Uzhgorod differ diff --git a/lib/pytz/zoneinfo/Europe/Vaduz b/lib/pytz/zoneinfo/Europe/Vaduz new file mode 100644 index 00000000..9c2b600b Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Vaduz differ diff --git a/lib/pytz/zoneinfo/Europe/Vatican b/lib/pytz/zoneinfo/Europe/Vatican new file mode 100644 index 00000000..5cc30403 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Vatican differ diff --git a/lib/pytz/zoneinfo/Europe/Vienna b/lib/pytz/zoneinfo/Europe/Vienna new file mode 100644 index 00000000..9c0fac53 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Vienna differ diff --git a/lib/pytz/zoneinfo/Europe/Vilnius b/lib/pytz/zoneinfo/Europe/Vilnius new file mode 100644 index 00000000..3b11880d Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Vilnius differ diff --git a/lib/pytz/zoneinfo/Europe/Volgograd b/lib/pytz/zoneinfo/Europe/Volgograd new file mode 100644 index 00000000..c62c32a6 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Volgograd differ diff --git a/lib/pytz/zoneinfo/Europe/Warsaw b/lib/pytz/zoneinfo/Europe/Warsaw new file mode 100644 index 00000000..5cbba412 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Warsaw differ diff --git a/lib/pytz/zoneinfo/Europe/Zagreb b/lib/pytz/zoneinfo/Europe/Zagreb new file mode 100644 index 00000000..79c25d70 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Zagreb differ diff --git a/lib/pytz/zoneinfo/Europe/Zaporozhye b/lib/pytz/zoneinfo/Europe/Zaporozhye new file mode 100644 index 00000000..49b568e7 Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Zaporozhye differ diff --git a/lib/pytz/zoneinfo/Europe/Zurich b/lib/pytz/zoneinfo/Europe/Zurich new file mode 100644 index 00000000..9c2b600b Binary files /dev/null and b/lib/pytz/zoneinfo/Europe/Zurich differ diff --git a/lib/pytz/zoneinfo/Factory b/lib/pytz/zoneinfo/Factory new file mode 100644 index 00000000..6e6c452e Binary files /dev/null and b/lib/pytz/zoneinfo/Factory differ diff --git a/lib/pytz/zoneinfo/GB b/lib/pytz/zoneinfo/GB new file mode 100644 index 00000000..4527515c Binary files /dev/null and b/lib/pytz/zoneinfo/GB differ diff --git a/lib/pytz/zoneinfo/GB-Eire b/lib/pytz/zoneinfo/GB-Eire new file mode 100644 index 00000000..4527515c Binary files /dev/null and b/lib/pytz/zoneinfo/GB-Eire differ diff --git a/lib/pytz/zoneinfo/GMT b/lib/pytz/zoneinfo/GMT new file mode 100644 index 00000000..c05e45fd Binary files /dev/null and b/lib/pytz/zoneinfo/GMT differ diff --git a/lib/pytz/zoneinfo/GMT+0 b/lib/pytz/zoneinfo/GMT+0 new file mode 100644 index 00000000..c05e45fd Binary files /dev/null and b/lib/pytz/zoneinfo/GMT+0 differ diff --git a/lib/pytz/zoneinfo/GMT-0 b/lib/pytz/zoneinfo/GMT-0 new file mode 100644 index 00000000..c05e45fd Binary files /dev/null and b/lib/pytz/zoneinfo/GMT-0 differ diff --git a/lib/pytz/zoneinfo/GMT0 b/lib/pytz/zoneinfo/GMT0 new file mode 100644 index 00000000..c05e45fd Binary files /dev/null and b/lib/pytz/zoneinfo/GMT0 differ diff --git a/lib/pytz/zoneinfo/Greenwich b/lib/pytz/zoneinfo/Greenwich new file mode 100644 index 00000000..c05e45fd Binary files /dev/null and b/lib/pytz/zoneinfo/Greenwich differ diff --git a/lib/pytz/zoneinfo/HST b/lib/pytz/zoneinfo/HST new file mode 100644 index 00000000..03e4db07 Binary files /dev/null and b/lib/pytz/zoneinfo/HST differ diff --git a/lib/pytz/zoneinfo/Hongkong b/lib/pytz/zoneinfo/Hongkong new file mode 100644 index 00000000..dc9058e4 Binary files /dev/null and b/lib/pytz/zoneinfo/Hongkong differ diff --git a/lib/pytz/zoneinfo/Iceland b/lib/pytz/zoneinfo/Iceland new file mode 100644 index 00000000..35ba7a15 Binary files /dev/null and b/lib/pytz/zoneinfo/Iceland differ diff --git a/lib/pytz/zoneinfo/Indian/Antananarivo b/lib/pytz/zoneinfo/Indian/Antananarivo new file mode 100644 index 00000000..33d59cc9 Binary files /dev/null and b/lib/pytz/zoneinfo/Indian/Antananarivo differ diff --git a/lib/pytz/zoneinfo/Indian/Chagos b/lib/pytz/zoneinfo/Indian/Chagos new file mode 100644 index 00000000..a616bdfb Binary files /dev/null and b/lib/pytz/zoneinfo/Indian/Chagos differ diff --git a/lib/pytz/zoneinfo/Indian/Christmas b/lib/pytz/zoneinfo/Indian/Christmas new file mode 100644 index 00000000..ebcd2624 Binary files /dev/null and b/lib/pytz/zoneinfo/Indian/Christmas differ diff --git a/lib/pytz/zoneinfo/Indian/Cocos b/lib/pytz/zoneinfo/Indian/Cocos new file mode 100644 index 00000000..cd603f24 Binary files /dev/null and b/lib/pytz/zoneinfo/Indian/Cocos differ diff --git a/lib/pytz/zoneinfo/Indian/Comoro b/lib/pytz/zoneinfo/Indian/Comoro new file mode 100644 index 00000000..298db9b7 Binary files /dev/null and b/lib/pytz/zoneinfo/Indian/Comoro differ diff --git a/lib/pytz/zoneinfo/Indian/Kerguelen b/lib/pytz/zoneinfo/Indian/Kerguelen new file mode 100644 index 00000000..462851eb Binary files /dev/null and b/lib/pytz/zoneinfo/Indian/Kerguelen differ diff --git a/lib/pytz/zoneinfo/Indian/Mahe b/lib/pytz/zoneinfo/Indian/Mahe new file mode 100644 index 00000000..5f42819b Binary files /dev/null and b/lib/pytz/zoneinfo/Indian/Mahe differ diff --git a/lib/pytz/zoneinfo/Indian/Maldives b/lib/pytz/zoneinfo/Indian/Maldives new file mode 100644 index 00000000..cec224ff Binary files /dev/null and b/lib/pytz/zoneinfo/Indian/Maldives differ diff --git a/lib/pytz/zoneinfo/Indian/Mauritius b/lib/pytz/zoneinfo/Indian/Mauritius new file mode 100644 index 00000000..66ecc8f5 Binary files /dev/null and b/lib/pytz/zoneinfo/Indian/Mauritius differ diff --git a/lib/pytz/zoneinfo/Indian/Mayotte b/lib/pytz/zoneinfo/Indian/Mayotte new file mode 100644 index 00000000..c915d909 Binary files /dev/null and b/lib/pytz/zoneinfo/Indian/Mayotte differ diff --git a/lib/pytz/zoneinfo/Indian/Reunion b/lib/pytz/zoneinfo/Indian/Reunion new file mode 100644 index 00000000..c4d0da90 Binary files /dev/null and b/lib/pytz/zoneinfo/Indian/Reunion differ diff --git a/lib/pytz/zoneinfo/Iran b/lib/pytz/zoneinfo/Iran new file mode 100644 index 00000000..87107811 Binary files /dev/null and b/lib/pytz/zoneinfo/Iran differ diff --git a/lib/pytz/zoneinfo/Israel b/lib/pytz/zoneinfo/Israel new file mode 100644 index 00000000..df511993 Binary files /dev/null and b/lib/pytz/zoneinfo/Israel differ diff --git a/lib/pytz/zoneinfo/Jamaica b/lib/pytz/zoneinfo/Jamaica new file mode 100644 index 00000000..24ea5dc0 Binary files /dev/null and b/lib/pytz/zoneinfo/Jamaica differ diff --git a/lib/pytz/zoneinfo/Japan b/lib/pytz/zoneinfo/Japan new file mode 100644 index 00000000..02441403 Binary files /dev/null and b/lib/pytz/zoneinfo/Japan differ diff --git a/lib/pytz/zoneinfo/Kwajalein b/lib/pytz/zoneinfo/Kwajalein new file mode 100644 index 00000000..094c3cfd Binary files /dev/null and b/lib/pytz/zoneinfo/Kwajalein differ diff --git a/lib/pytz/zoneinfo/Libya b/lib/pytz/zoneinfo/Libya new file mode 100644 index 00000000..b32e2202 Binary files /dev/null and b/lib/pytz/zoneinfo/Libya differ diff --git a/lib/pytz/zoneinfo/MET b/lib/pytz/zoneinfo/MET new file mode 100644 index 00000000..71963d53 Binary files /dev/null and b/lib/pytz/zoneinfo/MET differ diff --git a/lib/pytz/zoneinfo/MST b/lib/pytz/zoneinfo/MST new file mode 100644 index 00000000..a1bee7c6 Binary files /dev/null and b/lib/pytz/zoneinfo/MST differ diff --git a/lib/pytz/zoneinfo/MST7MDT b/lib/pytz/zoneinfo/MST7MDT new file mode 100644 index 00000000..726a7e57 Binary files /dev/null and b/lib/pytz/zoneinfo/MST7MDT differ diff --git a/lib/pytz/zoneinfo/Mexico/BajaNorte b/lib/pytz/zoneinfo/Mexico/BajaNorte new file mode 100644 index 00000000..13874753 Binary files /dev/null and b/lib/pytz/zoneinfo/Mexico/BajaNorte differ diff --git a/lib/pytz/zoneinfo/Mexico/BajaSur b/lib/pytz/zoneinfo/Mexico/BajaSur new file mode 100644 index 00000000..afa94c2a Binary files /dev/null and b/lib/pytz/zoneinfo/Mexico/BajaSur differ diff --git a/lib/pytz/zoneinfo/Mexico/General b/lib/pytz/zoneinfo/Mexico/General new file mode 100644 index 00000000..f11e3d2d Binary files /dev/null and b/lib/pytz/zoneinfo/Mexico/General differ diff --git a/lib/pytz/zoneinfo/NZ b/lib/pytz/zoneinfo/NZ new file mode 100644 index 00000000..a5f5b6d5 Binary files /dev/null and b/lib/pytz/zoneinfo/NZ differ diff --git a/lib/pytz/zoneinfo/NZ-CHAT b/lib/pytz/zoneinfo/NZ-CHAT new file mode 100644 index 00000000..59bc4ede Binary files /dev/null and b/lib/pytz/zoneinfo/NZ-CHAT differ diff --git a/lib/pytz/zoneinfo/Navajo b/lib/pytz/zoneinfo/Navajo new file mode 100644 index 00000000..7fc66917 Binary files /dev/null and b/lib/pytz/zoneinfo/Navajo differ diff --git a/lib/pytz/zoneinfo/PRC b/lib/pytz/zoneinfo/PRC new file mode 100644 index 00000000..dbd132f2 Binary files /dev/null and b/lib/pytz/zoneinfo/PRC differ diff --git a/lib/pytz/zoneinfo/PST8PDT b/lib/pytz/zoneinfo/PST8PDT new file mode 100644 index 00000000..6242ac04 Binary files /dev/null and b/lib/pytz/zoneinfo/PST8PDT differ diff --git a/lib/pytz/zoneinfo/Pacific/Apia b/lib/pytz/zoneinfo/Pacific/Apia new file mode 100644 index 00000000..cc5d2cd2 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Apia differ diff --git a/lib/pytz/zoneinfo/Pacific/Auckland b/lib/pytz/zoneinfo/Pacific/Auckland new file mode 100644 index 00000000..a5f5b6d5 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Auckland differ diff --git a/lib/pytz/zoneinfo/Pacific/Chatham b/lib/pytz/zoneinfo/Pacific/Chatham new file mode 100644 index 00000000..59bc4ede Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Chatham differ diff --git a/lib/pytz/zoneinfo/Pacific/Chuuk b/lib/pytz/zoneinfo/Pacific/Chuuk new file mode 100644 index 00000000..28356bbf Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Chuuk differ diff --git a/lib/pytz/zoneinfo/Pacific/Easter b/lib/pytz/zoneinfo/Pacific/Easter new file mode 100644 index 00000000..8c8a6c7d Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Easter differ diff --git a/lib/pytz/zoneinfo/Pacific/Efate b/lib/pytz/zoneinfo/Pacific/Efate new file mode 100644 index 00000000..1d99519b Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Efate differ diff --git a/lib/pytz/zoneinfo/Pacific/Enderbury b/lib/pytz/zoneinfo/Pacific/Enderbury new file mode 100644 index 00000000..48610523 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Enderbury differ diff --git a/lib/pytz/zoneinfo/Pacific/Fakaofo b/lib/pytz/zoneinfo/Pacific/Fakaofo new file mode 100644 index 00000000..e02e18e2 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Fakaofo differ diff --git a/lib/pytz/zoneinfo/Pacific/Fiji b/lib/pytz/zoneinfo/Pacific/Fiji new file mode 100644 index 00000000..d91c7e5d Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Fiji differ diff --git a/lib/pytz/zoneinfo/Pacific/Funafuti b/lib/pytz/zoneinfo/Pacific/Funafuti new file mode 100644 index 00000000..576dea30 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Funafuti differ diff --git a/lib/pytz/zoneinfo/Pacific/Galapagos b/lib/pytz/zoneinfo/Pacific/Galapagos new file mode 100644 index 00000000..c9a7371d Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Galapagos differ diff --git a/lib/pytz/zoneinfo/Pacific/Gambier b/lib/pytz/zoneinfo/Pacific/Gambier new file mode 100644 index 00000000..4ab6c206 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Gambier differ diff --git a/lib/pytz/zoneinfo/Pacific/Guadalcanal b/lib/pytz/zoneinfo/Pacific/Guadalcanal new file mode 100644 index 00000000..b183d1ea Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Guadalcanal differ diff --git a/lib/pytz/zoneinfo/Pacific/Guam b/lib/pytz/zoneinfo/Pacific/Guam new file mode 100644 index 00000000..4286e6ba Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Guam differ diff --git a/lib/pytz/zoneinfo/Pacific/Honolulu b/lib/pytz/zoneinfo/Pacific/Honolulu new file mode 100644 index 00000000..bd855772 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Honolulu differ diff --git a/lib/pytz/zoneinfo/Pacific/Johnston b/lib/pytz/zoneinfo/Pacific/Johnston new file mode 100644 index 00000000..bd855772 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Johnston differ diff --git a/lib/pytz/zoneinfo/Pacific/Kiritimati b/lib/pytz/zoneinfo/Pacific/Kiritimati new file mode 100644 index 00000000..c2eafbc7 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Kiritimati differ diff --git a/lib/pytz/zoneinfo/Pacific/Kosrae b/lib/pytz/zoneinfo/Pacific/Kosrae new file mode 100644 index 00000000..66c4d658 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Kosrae differ diff --git a/lib/pytz/zoneinfo/Pacific/Kwajalein b/lib/pytz/zoneinfo/Pacific/Kwajalein new file mode 100644 index 00000000..094c3cfd Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Kwajalein differ diff --git a/lib/pytz/zoneinfo/Pacific/Majuro b/lib/pytz/zoneinfo/Pacific/Majuro new file mode 100644 index 00000000..d53b7c2d Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Majuro differ diff --git a/lib/pytz/zoneinfo/Pacific/Marquesas b/lib/pytz/zoneinfo/Pacific/Marquesas new file mode 100644 index 00000000..c717c122 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Marquesas differ diff --git a/lib/pytz/zoneinfo/Pacific/Midway b/lib/pytz/zoneinfo/Pacific/Midway new file mode 100644 index 00000000..f2a2f63c Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Midway differ diff --git a/lib/pytz/zoneinfo/Pacific/Nauru b/lib/pytz/zoneinfo/Pacific/Nauru new file mode 100644 index 00000000..896ffeee Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Nauru differ diff --git a/lib/pytz/zoneinfo/Pacific/Niue b/lib/pytz/zoneinfo/Pacific/Niue new file mode 100644 index 00000000..d772edf5 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Niue differ diff --git a/lib/pytz/zoneinfo/Pacific/Norfolk b/lib/pytz/zoneinfo/Pacific/Norfolk new file mode 100644 index 00000000..3a286be3 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Norfolk differ diff --git a/lib/pytz/zoneinfo/Pacific/Noumea b/lib/pytz/zoneinfo/Pacific/Noumea new file mode 100644 index 00000000..fcc44e60 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Noumea differ diff --git a/lib/pytz/zoneinfo/Pacific/Pago_Pago b/lib/pytz/zoneinfo/Pacific/Pago_Pago new file mode 100644 index 00000000..1d7649ff Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Pago_Pago differ diff --git a/lib/pytz/zoneinfo/Pacific/Palau b/lib/pytz/zoneinfo/Pacific/Palau new file mode 100644 index 00000000..28992d2d Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Palau differ diff --git a/lib/pytz/zoneinfo/Pacific/Pitcairn b/lib/pytz/zoneinfo/Pacific/Pitcairn new file mode 100644 index 00000000..d62c648b Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Pitcairn differ diff --git a/lib/pytz/zoneinfo/Pacific/Pohnpei b/lib/pytz/zoneinfo/Pacific/Pohnpei new file mode 100644 index 00000000..59bd7646 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Pohnpei differ diff --git a/lib/pytz/zoneinfo/Pacific/Ponape b/lib/pytz/zoneinfo/Pacific/Ponape new file mode 100644 index 00000000..59bd7646 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Ponape differ diff --git a/lib/pytz/zoneinfo/Pacific/Port_Moresby b/lib/pytz/zoneinfo/Pacific/Port_Moresby new file mode 100644 index 00000000..dffa4573 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Port_Moresby differ diff --git a/lib/pytz/zoneinfo/Pacific/Rarotonga b/lib/pytz/zoneinfo/Pacific/Rarotonga new file mode 100644 index 00000000..2a254902 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Rarotonga differ diff --git a/lib/pytz/zoneinfo/Pacific/Saipan b/lib/pytz/zoneinfo/Pacific/Saipan new file mode 100644 index 00000000..c54473cd Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Saipan differ diff --git a/lib/pytz/zoneinfo/Pacific/Samoa b/lib/pytz/zoneinfo/Pacific/Samoa new file mode 100644 index 00000000..1d7649ff Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Samoa differ diff --git a/lib/pytz/zoneinfo/Pacific/Tahiti b/lib/pytz/zoneinfo/Pacific/Tahiti new file mode 100644 index 00000000..bfc9a7c9 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Tahiti differ diff --git a/lib/pytz/zoneinfo/Pacific/Tarawa b/lib/pytz/zoneinfo/Pacific/Tarawa new file mode 100644 index 00000000..1e8189ce Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Tarawa differ diff --git a/lib/pytz/zoneinfo/Pacific/Tongatapu b/lib/pytz/zoneinfo/Pacific/Tongatapu new file mode 100644 index 00000000..71d899bb Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Tongatapu differ diff --git a/lib/pytz/zoneinfo/Pacific/Truk b/lib/pytz/zoneinfo/Pacific/Truk new file mode 100644 index 00000000..28356bbf Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Truk differ diff --git a/lib/pytz/zoneinfo/Pacific/Wake b/lib/pytz/zoneinfo/Pacific/Wake new file mode 100644 index 00000000..9e2a37cc Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Wake differ diff --git a/lib/pytz/zoneinfo/Pacific/Wallis b/lib/pytz/zoneinfo/Pacific/Wallis new file mode 100644 index 00000000..b8944715 Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Wallis differ diff --git a/lib/pytz/zoneinfo/Pacific/Yap b/lib/pytz/zoneinfo/Pacific/Yap new file mode 100644 index 00000000..28356bbf Binary files /dev/null and b/lib/pytz/zoneinfo/Pacific/Yap differ diff --git a/lib/pytz/zoneinfo/Poland b/lib/pytz/zoneinfo/Poland new file mode 100644 index 00000000..5cbba412 Binary files /dev/null and b/lib/pytz/zoneinfo/Poland differ diff --git a/lib/pytz/zoneinfo/Portugal b/lib/pytz/zoneinfo/Portugal new file mode 100644 index 00000000..b9aff3a5 Binary files /dev/null and b/lib/pytz/zoneinfo/Portugal differ diff --git a/lib/pytz/zoneinfo/ROC b/lib/pytz/zoneinfo/ROC new file mode 100644 index 00000000..4810a0b6 Binary files /dev/null and b/lib/pytz/zoneinfo/ROC differ diff --git a/lib/pytz/zoneinfo/ROK b/lib/pytz/zoneinfo/ROK new file mode 100644 index 00000000..6931d782 Binary files /dev/null and b/lib/pytz/zoneinfo/ROK differ diff --git a/lib/pytz/zoneinfo/Singapore b/lib/pytz/zoneinfo/Singapore new file mode 100644 index 00000000..9dd49cb7 Binary files /dev/null and b/lib/pytz/zoneinfo/Singapore differ diff --git a/lib/pytz/zoneinfo/Turkey b/lib/pytz/zoneinfo/Turkey new file mode 100644 index 00000000..d89aa3a8 Binary files /dev/null and b/lib/pytz/zoneinfo/Turkey differ diff --git a/lib/pytz/zoneinfo/UCT b/lib/pytz/zoneinfo/UCT new file mode 100644 index 00000000..40147b9e Binary files /dev/null and b/lib/pytz/zoneinfo/UCT differ diff --git a/lib/pytz/zoneinfo/US/Alaska b/lib/pytz/zoneinfo/US/Alaska new file mode 100644 index 00000000..a4627cac Binary files /dev/null and b/lib/pytz/zoneinfo/US/Alaska differ diff --git a/lib/pytz/zoneinfo/US/Aleutian b/lib/pytz/zoneinfo/US/Aleutian new file mode 100644 index 00000000..b0a5dd60 Binary files /dev/null and b/lib/pytz/zoneinfo/US/Aleutian differ diff --git a/lib/pytz/zoneinfo/US/Arizona b/lib/pytz/zoneinfo/US/Arizona new file mode 100644 index 00000000..adf28236 Binary files /dev/null and b/lib/pytz/zoneinfo/US/Arizona differ diff --git a/lib/pytz/zoneinfo/US/Central b/lib/pytz/zoneinfo/US/Central new file mode 100644 index 00000000..3dd8f0fa Binary files /dev/null and b/lib/pytz/zoneinfo/US/Central differ diff --git a/lib/pytz/zoneinfo/US/East-Indiana b/lib/pytz/zoneinfo/US/East-Indiana new file mode 100644 index 00000000..4a92c065 Binary files /dev/null and b/lib/pytz/zoneinfo/US/East-Indiana differ diff --git a/lib/pytz/zoneinfo/US/Eastern b/lib/pytz/zoneinfo/US/Eastern new file mode 100644 index 00000000..7553fee3 Binary files /dev/null and b/lib/pytz/zoneinfo/US/Eastern differ diff --git a/lib/pytz/zoneinfo/US/Hawaii b/lib/pytz/zoneinfo/US/Hawaii new file mode 100644 index 00000000..bd855772 Binary files /dev/null and b/lib/pytz/zoneinfo/US/Hawaii differ diff --git a/lib/pytz/zoneinfo/US/Indiana-Starke b/lib/pytz/zoneinfo/US/Indiana-Starke new file mode 100644 index 00000000..cc785da9 Binary files /dev/null and b/lib/pytz/zoneinfo/US/Indiana-Starke differ diff --git a/lib/pytz/zoneinfo/US/Michigan b/lib/pytz/zoneinfo/US/Michigan new file mode 100644 index 00000000..a123b331 Binary files /dev/null and b/lib/pytz/zoneinfo/US/Michigan differ diff --git a/lib/pytz/zoneinfo/US/Mountain b/lib/pytz/zoneinfo/US/Mountain new file mode 100644 index 00000000..7fc66917 Binary files /dev/null and b/lib/pytz/zoneinfo/US/Mountain differ diff --git a/lib/pytz/zoneinfo/US/Pacific b/lib/pytz/zoneinfo/US/Pacific new file mode 100644 index 00000000..1fa9149f Binary files /dev/null and b/lib/pytz/zoneinfo/US/Pacific differ diff --git a/lib/pytz/zoneinfo/US/Pacific-New b/lib/pytz/zoneinfo/US/Pacific-New new file mode 100644 index 00000000..1fa9149f Binary files /dev/null and b/lib/pytz/zoneinfo/US/Pacific-New differ diff --git a/lib/pytz/zoneinfo/US/Samoa b/lib/pytz/zoneinfo/US/Samoa new file mode 100644 index 00000000..1d7649ff Binary files /dev/null and b/lib/pytz/zoneinfo/US/Samoa differ diff --git a/lib/pytz/zoneinfo/UTC b/lib/pytz/zoneinfo/UTC new file mode 100644 index 00000000..c3b97f1a Binary files /dev/null and b/lib/pytz/zoneinfo/UTC differ diff --git a/lib/pytz/zoneinfo/Universal b/lib/pytz/zoneinfo/Universal new file mode 100644 index 00000000..c3b97f1a Binary files /dev/null and b/lib/pytz/zoneinfo/Universal differ diff --git a/lib/pytz/zoneinfo/W-SU b/lib/pytz/zoneinfo/W-SU new file mode 100644 index 00000000..bdbbaebe Binary files /dev/null and b/lib/pytz/zoneinfo/W-SU differ diff --git a/lib/pytz/zoneinfo/WET b/lib/pytz/zoneinfo/WET new file mode 100644 index 00000000..444a1933 Binary files /dev/null and b/lib/pytz/zoneinfo/WET differ diff --git a/lib/pytz/zoneinfo/Zulu b/lib/pytz/zoneinfo/Zulu new file mode 100644 index 00000000..c3b97f1a Binary files /dev/null and b/lib/pytz/zoneinfo/Zulu differ diff --git a/lib/pytz/zoneinfo/iso3166.tab b/lib/pytz/zoneinfo/iso3166.tab new file mode 100644 index 00000000..0b0b8426 --- /dev/null +++ b/lib/pytz/zoneinfo/iso3166.tab @@ -0,0 +1,275 @@ +# ISO 3166 alpha-2 country codes +# +# This file is in the public domain, so clarified as of +# 2009-05-17 by Arthur David Olson. +# +# From Paul Eggert (2014-07-18): +# This file contains a table of two-letter country codes. Columns are +# separated by a single tab. Lines beginning with '#' are comments. +# Although all text currently uses ASCII encoding, this is planned to +# change to UTF-8 soon. The columns of the table are as follows: +# +# 1. ISO 3166-1 alpha-2 country code, current as of +# ISO 3166-1 Newsletter VI-16 (2013-07-11). See: Updates on ISO 3166 +# http://www.iso.org/iso/home/standards/country_codes/updates_on_iso_3166.htm +# 2. The usual English name for the coded region, +# chosen so that alphabetic sorting of subsets produces helpful lists. +# This is not the same as the English name in the ISO 3166 tables. +# +# The table is sorted by country code. +# +# This table is intended as an aid for users, to help them select time +# zone data appropriate for their practical needs. It is not intended +# to take or endorse any position on legal or territorial claims. +# +#country- +#code name of country, territory, area, or subdivision +AD Andorra +AE United Arab Emirates +AF Afghanistan +AG Antigua & Barbuda +AI Anguilla +AL Albania +AM Armenia +AO Angola +AQ Antarctica +AR Argentina +AS Samoa (American) +AT Austria +AU Australia +AW Aruba +AX Aaland Islands +AZ Azerbaijan +BA Bosnia & Herzegovina +BB Barbados +BD Bangladesh +BE Belgium +BF Burkina Faso +BG Bulgaria +BH Bahrain +BI Burundi +BJ Benin +BL St Barthelemy +BM Bermuda +BN Brunei +BO Bolivia +BQ Caribbean Netherlands +BR Brazil +BS Bahamas +BT Bhutan +BV Bouvet Island +BW Botswana +BY Belarus +BZ Belize +CA Canada +CC Cocos (Keeling) Islands +CD Congo (Dem. Rep.) +CF Central African Rep. +CG Congo (Rep.) +CH Switzerland +CI Cote d'Ivoire +CK Cook Islands +CL Chile +CM Cameroon +CN China +CO Colombia +CR Costa Rica +CU Cuba +CV Cape Verde +CW Curacao +CX Christmas Island +CY Cyprus +CZ Czech Republic +DE Germany +DJ Djibouti +DK Denmark +DM Dominica +DO Dominican Republic +DZ Algeria +EC Ecuador +EE Estonia +EG Egypt +EH Western Sahara +ER Eritrea +ES Spain +ET Ethiopia +FI Finland +FJ Fiji +FK Falkland Islands +FM Micronesia +FO Faroe Islands +FR France +GA Gabon +GB Britain (UK) +GD Grenada +GE Georgia +GF French Guiana +GG Guernsey +GH Ghana +GI Gibraltar +GL Greenland +GM Gambia +GN Guinea +GP Guadeloupe +GQ Equatorial Guinea +GR Greece +GS South Georgia & the South Sandwich Islands +GT Guatemala +GU Guam +GW Guinea-Bissau +GY Guyana +HK Hong Kong +HM Heard Island & McDonald Islands +HN Honduras +HR Croatia +HT Haiti +HU Hungary +ID Indonesia +IE Ireland +IL Israel +IM Isle of Man +IN India +IO British Indian Ocean Territory +IQ Iraq +IR Iran +IS Iceland +IT Italy +JE Jersey +JM Jamaica +JO Jordan +JP Japan +KE Kenya +KG Kyrgyzstan +KH Cambodia +KI Kiribati +KM Comoros +KN St Kitts & Nevis +KP Korea (North) +KR Korea (South) +KW Kuwait +KY Cayman Islands +KZ Kazakhstan +LA Laos +LB Lebanon +LC St Lucia +LI Liechtenstein +LK Sri Lanka +LR Liberia +LS Lesotho +LT Lithuania +LU Luxembourg +LV Latvia +LY Libya +MA Morocco +MC Monaco +MD Moldova +ME Montenegro +MF St Martin (French part) +MG Madagascar +MH Marshall Islands +MK Macedonia +ML Mali +MM Myanmar (Burma) +MN Mongolia +MO Macau +MP Northern Mariana Islands +MQ Martinique +MR Mauritania +MS Montserrat +MT Malta +MU Mauritius +MV Maldives +MW Malawi +MX Mexico +MY Malaysia +MZ Mozambique +NA Namibia +NC New Caledonia +NE Niger +NF Norfolk Island +NG Nigeria +NI Nicaragua +NL Netherlands +NO Norway +NP Nepal +NR Nauru +NU Niue +NZ New Zealand +OM Oman +PA Panama +PE Peru +PF French Polynesia +PG Papua New Guinea +PH Philippines +PK Pakistan +PL Poland +PM St Pierre & Miquelon +PN Pitcairn +PR Puerto Rico +PS Palestine +PT Portugal +PW Palau +PY Paraguay +QA Qatar +RE Reunion +RO Romania +RS Serbia +RU Russia +RW Rwanda +SA Saudi Arabia +SB Solomon Islands +SC Seychelles +SD Sudan +SE Sweden +SG Singapore +SH St Helena +SI Slovenia +SJ Svalbard & Jan Mayen +SK Slovakia +SL Sierra Leone +SM San Marino +SN Senegal +SO Somalia +SR Suriname +SS South Sudan +ST Sao Tome & Principe +SV El Salvador +SX St Maarten (Dutch part) +SY Syria +SZ Swaziland +TC Turks & Caicos Is +TD Chad +TF French Southern & Antarctic Lands +TG Togo +TH Thailand +TJ Tajikistan +TK Tokelau +TL East Timor +TM Turkmenistan +TN Tunisia +TO Tonga +TR Turkey +TT Trinidad & Tobago +TV Tuvalu +TW Taiwan +TZ Tanzania +UA Ukraine +UG Uganda +UM US minor outlying islands +US United States +UY Uruguay +UZ Uzbekistan +VA Vatican City +VC St Vincent +VE Venezuela +VG Virgin Islands (UK) +VI Virgin Islands (US) +VN Vietnam +VU Vanuatu +WF Wallis & Futuna +WS Samoa (western) +YE Yemen +YT Mayotte +ZA South Africa +ZM Zambia +ZW Zimbabwe diff --git a/lib/pytz/zoneinfo/localtime b/lib/pytz/zoneinfo/localtime new file mode 100644 index 00000000..c05e45fd Binary files /dev/null and b/lib/pytz/zoneinfo/localtime differ diff --git a/lib/pytz/zoneinfo/posixrules b/lib/pytz/zoneinfo/posixrules new file mode 100644 index 00000000..7553fee3 Binary files /dev/null and b/lib/pytz/zoneinfo/posixrules differ diff --git a/lib/pytz/zoneinfo/zone.tab b/lib/pytz/zoneinfo/zone.tab new file mode 100644 index 00000000..084bb2fb --- /dev/null +++ b/lib/pytz/zoneinfo/zone.tab @@ -0,0 +1,439 @@ +# tz zone descriptions (deprecated version) +# +# This file is in the public domain, so clarified as of +# 2009-05-17 by Arthur David Olson. +# +# From Paul Eggert (2014-07-31): +# This file is intended as a backward-compatibility aid for older programs. +# New programs should use zone1970.tab. This file is like zone1970.tab (see +# zone1970.tab's comments), but with the following additional restrictions: +# +# 1. This file contains only ASCII characters. +# 2. The first data column contains exactly one country code. +# +# Because of (2), each row stands for an area that is the intersection +# of a region identified by a country code and of a zone where civil +# clocks have agreed since 1970; this is a narrower definition than +# that of zone1970.tab. +# +# This table is intended as an aid for users, to help them select time +# zone data entries appropriate for their practical needs. It is not +# intended to take or endorse any position on legal or territorial claims. +# +#country- +#code coordinates TZ comments +AD +4230+00131 Europe/Andorra +AE +2518+05518 Asia/Dubai +AF +3431+06912 Asia/Kabul +AG +1703-06148 America/Antigua +AI +1812-06304 America/Anguilla +AL +4120+01950 Europe/Tirane +AM +4011+04430 Asia/Yerevan +AO -0848+01314 Africa/Luanda +AQ -7750+16636 Antarctica/McMurdo McMurdo, South Pole, Scott (New Zealand time) +AQ -6734-06808 Antarctica/Rothera Rothera Station, Adelaide Island +AQ -6448-06406 Antarctica/Palmer Palmer Station, Anvers Island +AQ -6736+06253 Antarctica/Mawson Mawson Station, Holme Bay +AQ -6835+07758 Antarctica/Davis Davis Station, Vestfold Hills +AQ -6617+11031 Antarctica/Casey Casey Station, Bailey Peninsula +AQ -7824+10654 Antarctica/Vostok Vostok Station, Lake Vostok +AQ -6640+14001 Antarctica/DumontDUrville Dumont-d'Urville Station, Adelie Land +AQ -690022+0393524 Antarctica/Syowa Syowa Station, E Ongul I +AQ -720041+0023206 Antarctica/Troll Troll Station, Queen Maud Land +AR -3436-05827 America/Argentina/Buenos_Aires Buenos Aires (BA, CF) +AR -3124-06411 America/Argentina/Cordoba most locations (CB, CC, CN, ER, FM, MN, SE, SF) +AR -2447-06525 America/Argentina/Salta (SA, LP, NQ, RN) +AR -2411-06518 America/Argentina/Jujuy Jujuy (JY) +AR -2649-06513 America/Argentina/Tucuman Tucuman (TM) +AR -2828-06547 America/Argentina/Catamarca Catamarca (CT), Chubut (CH) +AR -2926-06651 America/Argentina/La_Rioja La Rioja (LR) +AR -3132-06831 America/Argentina/San_Juan San Juan (SJ) +AR -3253-06849 America/Argentina/Mendoza Mendoza (MZ) +AR -3319-06621 America/Argentina/San_Luis San Luis (SL) +AR -5138-06913 America/Argentina/Rio_Gallegos Santa Cruz (SC) +AR -5448-06818 America/Argentina/Ushuaia Tierra del Fuego (TF) +AS -1416-17042 Pacific/Pago_Pago +AT +4813+01620 Europe/Vienna +AU -3133+15905 Australia/Lord_Howe Lord Howe Island +AU -5430+15857 Antarctica/Macquarie Macquarie Island +AU -4253+14719 Australia/Hobart Tasmania - most locations +AU -3956+14352 Australia/Currie Tasmania - King Island +AU -3749+14458 Australia/Melbourne Victoria +AU -3352+15113 Australia/Sydney New South Wales - most locations +AU -3157+14127 Australia/Broken_Hill New South Wales - Yancowinna +AU -2728+15302 Australia/Brisbane Queensland - most locations +AU -2016+14900 Australia/Lindeman Queensland - Holiday Islands +AU -3455+13835 Australia/Adelaide South Australia +AU -1228+13050 Australia/Darwin Northern Territory +AU -3157+11551 Australia/Perth Western Australia - most locations +AU -3143+12852 Australia/Eucla Western Australia - Eucla area +AW +1230-06958 America/Aruba +AX +6006+01957 Europe/Mariehamn +AZ +4023+04951 Asia/Baku +BA +4352+01825 Europe/Sarajevo +BB +1306-05937 America/Barbados +BD +2343+09025 Asia/Dhaka +BE +5050+00420 Europe/Brussels +BF +1222-00131 Africa/Ouagadougou +BG +4241+02319 Europe/Sofia +BH +2623+05035 Asia/Bahrain +BI -0323+02922 Africa/Bujumbura +BJ +0629+00237 Africa/Porto-Novo +BL +1753-06251 America/St_Barthelemy +BM +3217-06446 Atlantic/Bermuda +BN +0456+11455 Asia/Brunei +BO -1630-06809 America/La_Paz +BQ +120903-0681636 America/Kralendijk +BR -0351-03225 America/Noronha Atlantic islands +BR -0127-04829 America/Belem Amapa, E Para +BR -0343-03830 America/Fortaleza NE Brazil (MA, PI, CE, RN, PB) +BR -0803-03454 America/Recife Pernambuco +BR -0712-04812 America/Araguaina Tocantins +BR -0940-03543 America/Maceio Alagoas, Sergipe +BR -1259-03831 America/Bahia Bahia +BR -2332-04637 America/Sao_Paulo S & SE Brazil (GO, DF, MG, ES, RJ, SP, PR, SC, RS) +BR -2027-05437 America/Campo_Grande Mato Grosso do Sul +BR -1535-05605 America/Cuiaba Mato Grosso +BR -0226-05452 America/Santarem W Para +BR -0846-06354 America/Porto_Velho Rondonia +BR +0249-06040 America/Boa_Vista Roraima +BR -0308-06001 America/Manaus E Amazonas +BR -0640-06952 America/Eirunepe W Amazonas +BR -0958-06748 America/Rio_Branco Acre +BS +2505-07721 America/Nassau +BT +2728+08939 Asia/Thimphu +BW -2439+02555 Africa/Gaborone +BY +5354+02734 Europe/Minsk +BZ +1730-08812 America/Belize +CA +4734-05243 America/St_Johns Newfoundland Time, including SE Labrador +CA +4439-06336 America/Halifax Atlantic Time - Nova Scotia (most places), PEI +CA +4612-05957 America/Glace_Bay Atlantic Time - Nova Scotia - places that did not observe DST 1966-1971 +CA +4606-06447 America/Moncton Atlantic Time - New Brunswick +CA +5320-06025 America/Goose_Bay Atlantic Time - Labrador - most locations +CA +5125-05707 America/Blanc-Sablon Atlantic Standard Time - Quebec - Lower North Shore +CA +4339-07923 America/Toronto Eastern Time - Ontario & Quebec - most locations +CA +4901-08816 America/Nipigon Eastern Time - Ontario & Quebec - places that did not observe DST 1967-1973 +CA +4823-08915 America/Thunder_Bay Eastern Time - Thunder Bay, Ontario +CA +6344-06828 America/Iqaluit Eastern Time - east Nunavut - most locations +CA +6608-06544 America/Pangnirtung Eastern Time - Pangnirtung, Nunavut +CA +744144-0944945 America/Resolute Central Time - Resolute, Nunavut +CA +484531-0913718 America/Atikokan Eastern Standard Time - Atikokan, Ontario and Southampton I, Nunavut +CA +624900-0920459 America/Rankin_Inlet Central Time - central Nunavut +CA +4953-09709 America/Winnipeg Central Time - Manitoba & west Ontario +CA +4843-09434 America/Rainy_River Central Time - Rainy River & Fort Frances, Ontario +CA +5024-10439 America/Regina Central Standard Time - Saskatchewan - most locations +CA +5017-10750 America/Swift_Current Central Standard Time - Saskatchewan - midwest +CA +5333-11328 America/Edmonton Mountain Time - Alberta, east British Columbia & west Saskatchewan +CA +690650-1050310 America/Cambridge_Bay Mountain Time - west Nunavut +CA +6227-11421 America/Yellowknife Mountain Time - central Northwest Territories +CA +682059-1334300 America/Inuvik Mountain Time - west Northwest Territories +CA +4906-11631 America/Creston Mountain Standard Time - Creston, British Columbia +CA +5946-12014 America/Dawson_Creek Mountain Standard Time - Dawson Creek & Fort Saint John, British Columbia +CA +4916-12307 America/Vancouver Pacific Time - west British Columbia +CA +6043-13503 America/Whitehorse Pacific Time - south Yukon +CA +6404-13925 America/Dawson Pacific Time - north Yukon +CC -1210+09655 Indian/Cocos +CD -0418+01518 Africa/Kinshasa west Dem. Rep. of Congo +CD -1140+02728 Africa/Lubumbashi east Dem. Rep. of Congo +CF +0422+01835 Africa/Bangui +CG -0416+01517 Africa/Brazzaville +CH +4723+00832 Europe/Zurich +CI +0519-00402 Africa/Abidjan +CK -2114-15946 Pacific/Rarotonga +CL -3327-07040 America/Santiago most locations +CL -2709-10926 Pacific/Easter Easter Island +CM +0403+00942 Africa/Douala +CN +3114+12128 Asia/Shanghai Beijing Time +CN +4348+08735 Asia/Urumqi Xinjiang Time +CO +0436-07405 America/Bogota +CR +0956-08405 America/Costa_Rica +CU +2308-08222 America/Havana +CV +1455-02331 Atlantic/Cape_Verde +CW +1211-06900 America/Curacao +CX -1025+10543 Indian/Christmas +CY +3510+03322 Asia/Nicosia +CZ +5005+01426 Europe/Prague +DE +5230+01322 Europe/Berlin most locations +DE +4742+00841 Europe/Busingen Busingen +DJ +1136+04309 Africa/Djibouti +DK +5540+01235 Europe/Copenhagen +DM +1518-06124 America/Dominica +DO +1828-06954 America/Santo_Domingo +DZ +3647+00303 Africa/Algiers +EC -0210-07950 America/Guayaquil mainland +EC -0054-08936 Pacific/Galapagos Galapagos Islands +EE +5925+02445 Europe/Tallinn +EG +3003+03115 Africa/Cairo +EH +2709-01312 Africa/El_Aaiun +ER +1520+03853 Africa/Asmara +ES +4024-00341 Europe/Madrid mainland +ES +3553-00519 Africa/Ceuta Ceuta & Melilla +ES +2806-01524 Atlantic/Canary Canary Islands +ET +0902+03842 Africa/Addis_Ababa +FI +6010+02458 Europe/Helsinki +FJ -1808+17825 Pacific/Fiji +FK -5142-05751 Atlantic/Stanley +FM +0725+15147 Pacific/Chuuk Chuuk (Truk) and Yap +FM +0658+15813 Pacific/Pohnpei Pohnpei (Ponape) +FM +0519+16259 Pacific/Kosrae Kosrae +FO +6201-00646 Atlantic/Faroe +FR +4852+00220 Europe/Paris +GA +0023+00927 Africa/Libreville +GB +513030-0000731 Europe/London +GD +1203-06145 America/Grenada +GE +4143+04449 Asia/Tbilisi +GF +0456-05220 America/Cayenne +GG +4927-00232 Europe/Guernsey +GH +0533-00013 Africa/Accra +GI +3608-00521 Europe/Gibraltar +GL +6411-05144 America/Godthab most locations +GL +7646-01840 America/Danmarkshavn east coast, north of Scoresbysund +GL +7029-02158 America/Scoresbysund Scoresbysund / Ittoqqortoormiit +GL +7634-06847 America/Thule Thule / Pituffik +GM +1328-01639 Africa/Banjul +GN +0931-01343 Africa/Conakry +GP +1614-06132 America/Guadeloupe +GQ +0345+00847 Africa/Malabo +GR +3758+02343 Europe/Athens +GS -5416-03632 Atlantic/South_Georgia +GT +1438-09031 America/Guatemala +GU +1328+14445 Pacific/Guam +GW +1151-01535 Africa/Bissau +GY +0648-05810 America/Guyana +HK +2217+11409 Asia/Hong_Kong +HN +1406-08713 America/Tegucigalpa +HR +4548+01558 Europe/Zagreb +HT +1832-07220 America/Port-au-Prince +HU +4730+01905 Europe/Budapest +ID -0610+10648 Asia/Jakarta Java & Sumatra +ID -0002+10920 Asia/Pontianak west & central Borneo +ID -0507+11924 Asia/Makassar east & south Borneo, Sulawesi (Celebes), Bali, Nusa Tengarra, west Timor +ID -0232+14042 Asia/Jayapura west New Guinea (Irian Jaya) & Malukus (Moluccas) +IE +5320-00615 Europe/Dublin +IL +314650+0351326 Asia/Jerusalem +IM +5409-00428 Europe/Isle_of_Man +IN +2232+08822 Asia/Kolkata +IO -0720+07225 Indian/Chagos +IQ +3321+04425 Asia/Baghdad +IR +3540+05126 Asia/Tehran +IS +6409-02151 Atlantic/Reykjavik +IT +4154+01229 Europe/Rome +JE +4912-00207 Europe/Jersey +JM +175805-0764736 America/Jamaica +JO +3157+03556 Asia/Amman +JP +353916+1394441 Asia/Tokyo +KE -0117+03649 Africa/Nairobi +KG +4254+07436 Asia/Bishkek +KH +1133+10455 Asia/Phnom_Penh +KI +0125+17300 Pacific/Tarawa Gilbert Islands +KI -0308-17105 Pacific/Enderbury Phoenix Islands +KI +0152-15720 Pacific/Kiritimati Line Islands +KM -1141+04316 Indian/Comoro +KN +1718-06243 America/St_Kitts +KP +3901+12545 Asia/Pyongyang +KR +3733+12658 Asia/Seoul +KW +2920+04759 Asia/Kuwait +KY +1918-08123 America/Cayman +KZ +4315+07657 Asia/Almaty most locations +KZ +4448+06528 Asia/Qyzylorda Qyzylorda (Kyzylorda, Kzyl-Orda) +KZ +5017+05710 Asia/Aqtobe Aqtobe (Aktobe) +KZ +4431+05016 Asia/Aqtau Atyrau (Atirau, Gur'yev), Mangghystau (Mankistau) +KZ +5113+05121 Asia/Oral West Kazakhstan +LA +1758+10236 Asia/Vientiane +LB +3353+03530 Asia/Beirut +LC +1401-06100 America/St_Lucia +LI +4709+00931 Europe/Vaduz +LK +0656+07951 Asia/Colombo +LR +0618-01047 Africa/Monrovia +LS -2928+02730 Africa/Maseru +LT +5441+02519 Europe/Vilnius +LU +4936+00609 Europe/Luxembourg +LV +5657+02406 Europe/Riga +LY +3254+01311 Africa/Tripoli +MA +3339-00735 Africa/Casablanca +MC +4342+00723 Europe/Monaco +MD +4700+02850 Europe/Chisinau +ME +4226+01916 Europe/Podgorica +MF +1804-06305 America/Marigot +MG -1855+04731 Indian/Antananarivo +MH +0709+17112 Pacific/Majuro most locations +MH +0905+16720 Pacific/Kwajalein Kwajalein +MK +4159+02126 Europe/Skopje +ML +1239-00800 Africa/Bamako +MM +1647+09610 Asia/Rangoon +MN +4755+10653 Asia/Ulaanbaatar most locations +MN +4801+09139 Asia/Hovd Bayan-Olgiy, Govi-Altai, Hovd, Uvs, Zavkhan +MN +4804+11430 Asia/Choibalsan Dornod, Sukhbaatar +MO +2214+11335 Asia/Macau +MP +1512+14545 Pacific/Saipan +MQ +1436-06105 America/Martinique +MR +1806-01557 Africa/Nouakchott +MS +1643-06213 America/Montserrat +MT +3554+01431 Europe/Malta +MU -2010+05730 Indian/Mauritius +MV +0410+07330 Indian/Maldives +MW -1547+03500 Africa/Blantyre +MX +1924-09909 America/Mexico_City Central Time - most locations +MX +2105-08646 America/Cancun Central Time - Quintana Roo +MX +2058-08937 America/Merida Central Time - Campeche, Yucatan +MX +2540-10019 America/Monterrey Mexican Central Time - Coahuila, Durango, Nuevo Leon, Tamaulipas away from US border +MX +2550-09730 America/Matamoros US Central Time - Coahuila, Durango, Nuevo Leon, Tamaulipas near US border +MX +2313-10625 America/Mazatlan Mountain Time - S Baja, Nayarit, Sinaloa +MX +2838-10605 America/Chihuahua Mexican Mountain Time - Chihuahua away from US border +MX +2934-10425 America/Ojinaga US Mountain Time - Chihuahua near US border +MX +2904-11058 America/Hermosillo Mountain Standard Time - Sonora +MX +3232-11701 America/Tijuana US Pacific Time - Baja California near US border +MX +3018-11452 America/Santa_Isabel Mexican Pacific Time - Baja California away from US border +MX +2048-10515 America/Bahia_Banderas Mexican Central Time - Bahia de Banderas +MY +0310+10142 Asia/Kuala_Lumpur peninsular Malaysia +MY +0133+11020 Asia/Kuching Sabah & Sarawak +MZ -2558+03235 Africa/Maputo +NA -2234+01706 Africa/Windhoek +NC -2216+16627 Pacific/Noumea +NE +1331+00207 Africa/Niamey +NF -2903+16758 Pacific/Norfolk +NG +0627+00324 Africa/Lagos +NI +1209-08617 America/Managua +NL +5222+00454 Europe/Amsterdam +NO +5955+01045 Europe/Oslo +NP +2743+08519 Asia/Kathmandu +NR -0031+16655 Pacific/Nauru +NU -1901-16955 Pacific/Niue +NZ -3652+17446 Pacific/Auckland most locations +NZ -4357-17633 Pacific/Chatham Chatham Islands +OM +2336+05835 Asia/Muscat +PA +0858-07932 America/Panama +PE -1203-07703 America/Lima +PF -1732-14934 Pacific/Tahiti Society Islands +PF -0900-13930 Pacific/Marquesas Marquesas Islands +PF -2308-13457 Pacific/Gambier Gambier Islands +PG -0930+14710 Pacific/Port_Moresby +PH +1435+12100 Asia/Manila +PK +2452+06703 Asia/Karachi +PL +5215+02100 Europe/Warsaw +PM +4703-05620 America/Miquelon +PN -2504-13005 Pacific/Pitcairn +PR +182806-0660622 America/Puerto_Rico +PS +3130+03428 Asia/Gaza Gaza Strip +PS +313200+0350542 Asia/Hebron West Bank +PT +3843-00908 Europe/Lisbon mainland +PT +3238-01654 Atlantic/Madeira Madeira Islands +PT +3744-02540 Atlantic/Azores Azores +PW +0720+13429 Pacific/Palau +PY -2516-05740 America/Asuncion +QA +2517+05132 Asia/Qatar +RE -2052+05528 Indian/Reunion +RO +4426+02606 Europe/Bucharest +RS +4450+02030 Europe/Belgrade +RU +5443+02030 Europe/Kaliningrad Moscow-01 - Kaliningrad +RU +554521+0373704 Europe/Moscow Moscow+00 - west Russia +RU +4457+03406 Europe/Simferopol Moscow+00 - Crimea +RU +4844+04425 Europe/Volgograd Moscow+00 - Caspian Sea +RU +5312+05009 Europe/Samara Moscow+00 (Moscow+01 after 2014-10-26) - Samara, Udmurtia +RU +5651+06036 Asia/Yekaterinburg Moscow+02 - Urals +RU +5500+07324 Asia/Omsk Moscow+03 - west Siberia +RU +5502+08255 Asia/Novosibirsk Moscow+03 - Novosibirsk +RU +5345+08707 Asia/Novokuznetsk Moscow+03 (Moscow+04 after 2014-10-26) - Kemerovo +RU +5601+09250 Asia/Krasnoyarsk Moscow+04 - Yenisei River +RU +5216+10420 Asia/Irkutsk Moscow+05 - Lake Baikal +RU +5203+11328 Asia/Chita Moscow+06 (Moscow+05 after 2014-10-26) - Zabaykalsky +RU +6200+12940 Asia/Yakutsk Moscow+06 - Lena River +RU +623923+1353314 Asia/Khandyga Moscow+06 - Tomponsky, Ust-Maysky +RU +4310+13156 Asia/Vladivostok Moscow+07 - Amur River +RU +4658+14242 Asia/Sakhalin Moscow+07 - Sakhalin Island +RU +643337+1431336 Asia/Ust-Nera Moscow+07 - Oymyakonsky +RU +5934+15048 Asia/Magadan Moscow+08 (Moscow+07 after 2014-10-26) - Magadan +RU +6728+15343 Asia/Srednekolymsk Moscow+08 - E Sakha, N Kuril Is +RU +5301+15839 Asia/Kamchatka Moscow+08 (Moscow+09 after 2014-10-26) - Kamchatka +RU +6445+17729 Asia/Anadyr Moscow+08 (Moscow+09 after 2014-10-26) - Bering Sea +RW -0157+03004 Africa/Kigali +SA +2438+04643 Asia/Riyadh +SB -0932+16012 Pacific/Guadalcanal +SC -0440+05528 Indian/Mahe +SD +1536+03232 Africa/Khartoum +SE +5920+01803 Europe/Stockholm +SG +0117+10351 Asia/Singapore +SH -1555-00542 Atlantic/St_Helena +SI +4603+01431 Europe/Ljubljana +SJ +7800+01600 Arctic/Longyearbyen +SK +4809+01707 Europe/Bratislava +SL +0830-01315 Africa/Freetown +SM +4355+01228 Europe/San_Marino +SN +1440-01726 Africa/Dakar +SO +0204+04522 Africa/Mogadishu +SR +0550-05510 America/Paramaribo +SS +0451+03136 Africa/Juba +ST +0020+00644 Africa/Sao_Tome +SV +1342-08912 America/El_Salvador +SX +180305-0630250 America/Lower_Princes +SY +3330+03618 Asia/Damascus +SZ -2618+03106 Africa/Mbabane +TC +2128-07108 America/Grand_Turk +TD +1207+01503 Africa/Ndjamena +TF -492110+0701303 Indian/Kerguelen +TG +0608+00113 Africa/Lome +TH +1345+10031 Asia/Bangkok +TJ +3835+06848 Asia/Dushanbe +TK -0922-17114 Pacific/Fakaofo +TL -0833+12535 Asia/Dili +TM +3757+05823 Asia/Ashgabat +TN +3648+01011 Africa/Tunis +TO -2110-17510 Pacific/Tongatapu +TR +4101+02858 Europe/Istanbul +TT +1039-06131 America/Port_of_Spain +TV -0831+17913 Pacific/Funafuti +TW +2503+12130 Asia/Taipei +TZ -0648+03917 Africa/Dar_es_Salaam +UA +5026+03031 Europe/Kiev most locations +UA +4837+02218 Europe/Uzhgorod Ruthenia +UA +4750+03510 Europe/Zaporozhye Zaporozh'ye, E Lugansk / Zaporizhia, E Luhansk +UG +0019+03225 Africa/Kampala +UM +1645-16931 Pacific/Johnston Johnston Atoll +UM +2813-17722 Pacific/Midway Midway Islands +UM +1917+16637 Pacific/Wake Wake Island +US +404251-0740023 America/New_York Eastern Time +US +421953-0830245 America/Detroit Eastern Time - Michigan - most locations +US +381515-0854534 America/Kentucky/Louisville Eastern Time - Kentucky - Louisville area +US +364947-0845057 America/Kentucky/Monticello Eastern Time - Kentucky - Wayne County +US +394606-0860929 America/Indiana/Indianapolis Eastern Time - Indiana - most locations +US +384038-0873143 America/Indiana/Vincennes Eastern Time - Indiana - Daviess, Dubois, Knox & Martin Counties +US +410305-0863611 America/Indiana/Winamac Eastern Time - Indiana - Pulaski County +US +382232-0862041 America/Indiana/Marengo Eastern Time - Indiana - Crawford County +US +382931-0871643 America/Indiana/Petersburg Eastern Time - Indiana - Pike County +US +384452-0850402 America/Indiana/Vevay Eastern Time - Indiana - Switzerland County +US +415100-0873900 America/Chicago Central Time +US +375711-0864541 America/Indiana/Tell_City Central Time - Indiana - Perry County +US +411745-0863730 America/Indiana/Knox Central Time - Indiana - Starke County +US +450628-0873651 America/Menominee Central Time - Michigan - Dickinson, Gogebic, Iron & Menominee Counties +US +470659-1011757 America/North_Dakota/Center Central Time - North Dakota - Oliver County +US +465042-1012439 America/North_Dakota/New_Salem Central Time - North Dakota - Morton County (except Mandan area) +US +471551-1014640 America/North_Dakota/Beulah Central Time - North Dakota - Mercer County +US +394421-1045903 America/Denver Mountain Time +US +433649-1161209 America/Boise Mountain Time - south Idaho & east Oregon +US +332654-1120424 America/Phoenix Mountain Standard Time - Arizona (except Navajo) +US +340308-1181434 America/Los_Angeles Pacific Time +US +550737-1313435 America/Metlakatla Pacific Standard Time - Annette Island, Alaska +US +611305-1495401 America/Anchorage Alaska Time +US +581807-1342511 America/Juneau Alaska Time - Alaska panhandle +US +571035-1351807 America/Sitka Alaska Time - southeast Alaska panhandle +US +593249-1394338 America/Yakutat Alaska Time - Alaska panhandle neck +US +643004-1652423 America/Nome Alaska Time - west Alaska +US +515248-1763929 America/Adak Aleutian Islands +US +211825-1575130 Pacific/Honolulu Hawaii +UY -3453-05611 America/Montevideo +UZ +3940+06648 Asia/Samarkand west Uzbekistan +UZ +4120+06918 Asia/Tashkent east Uzbekistan +VA +415408+0122711 Europe/Vatican +VC +1309-06114 America/St_Vincent +VE +1030-06656 America/Caracas +VG +1827-06437 America/Tortola +VI +1821-06456 America/St_Thomas +VN +1045+10640 Asia/Ho_Chi_Minh +VU -1740+16825 Pacific/Efate +WF -1318-17610 Pacific/Wallis +WS -1350-17144 Pacific/Apia +YE +1245+04512 Asia/Aden +YT -1247+04514 Indian/Mayotte +ZA -2615+02800 Africa/Johannesburg +ZM -1525+02817 Africa/Lusaka +ZW -1750+03103 Africa/Harare diff --git a/lib/pytz/zoneinfo/zone1970.tab b/lib/pytz/zoneinfo/zone1970.tab new file mode 100644 index 00000000..03c50d89 --- /dev/null +++ b/lib/pytz/zoneinfo/zone1970.tab @@ -0,0 +1,369 @@ +# tz zone descriptions +# +# This file is in the public domain. +# +# From Paul Eggert (2014-07-31): +# This file contains a table where each row stands for a zone where +# civil time stamps have agreed since 1970. Columns are separated by +# a single tab. Lines beginning with '#' are comments. All text uses +# UTF-8 encoding. The columns of the table are as follows: +# +# 1. The countries that overlap the zone, as a comma-separated list +# of ISO 3166 2-character country codes. See the file 'iso3166.tab'. +# 2. Latitude and longitude of the zone's principal location +# in ISO 6709 sign-degrees-minutes-seconds format, +# either +-DDMM+-DDDMM or +-DDMMSS+-DDDMMSS, +# first latitude (+ is north), then longitude (+ is east). +# 3. Zone name used in value of TZ environment variable. +# Please see the 'Theory' file for how zone names are chosen. +# If multiple zones overlap a country, each has a row in the +# table, with each column 1 containing the country code. +# 4. Comments; present if and only if a country has multiple zones. +# +# If a zone covers multiple countries, the most-populous city is used, +# and that country is listed first in column 1; any other countries +# are listed alphabetically by country code. The table is sorted +# first by country code, then (if possible) by an order within the +# country that (1) makes some geographical sense, and (2) puts the +# most populous zones first, where that does not contradict (1). +# +# This table is intended as an aid for users, to help them select time +# zone data entries appropriate for their practical needs. It is not +# intended to take or endorse any position on legal or territorial claims. +# +#country- +#codes coordinates TZ comments +AD +4230+00131 Europe/Andorra +AE,OM +2518+05518 Asia/Dubai +AF +3431+06912 Asia/Kabul +AL +4120+01950 Europe/Tirane +AM +4011+04430 Asia/Yerevan +AQ -6734-06808 Antarctica/Rothera Rothera Station, Adelaide Island +AQ -6448-06406 Antarctica/Palmer Palmer Station, Anvers Island +AQ -6736+06253 Antarctica/Mawson Mawson Station, Holme Bay +AQ -6835+07758 Antarctica/Davis Davis Station, Vestfold Hills +AQ -6617+11031 Antarctica/Casey Casey Station, Bailey Peninsula +AQ -7824+10654 Antarctica/Vostok Vostok Station, Lake Vostok +AQ -6640+14001 Antarctica/DumontDUrville Dumont-d'Urville Station, Adélie Land +AQ -690022+0393524 Antarctica/Syowa Syowa Station, E Ongul I +AQ -720041+0023206 Antarctica/Troll Troll Station, Queen Maud Land +AR -3436-05827 America/Argentina/Buenos_Aires Buenos Aires (BA, CF) +AR -3124-06411 America/Argentina/Cordoba most locations (CB, CC, CN, ER, FM, MN, SE, SF) +AR -2447-06525 America/Argentina/Salta (SA, LP, NQ, RN) +AR -2411-06518 America/Argentina/Jujuy Jujuy (JY) +AR -2649-06513 America/Argentina/Tucuman Tucumán (TM) +AR -2828-06547 America/Argentina/Catamarca Catamarca (CT), Chubut (CH) +AR -2926-06651 America/Argentina/La_Rioja La Rioja (LR) +AR -3132-06831 America/Argentina/San_Juan San Juan (SJ) +AR -3253-06849 America/Argentina/Mendoza Mendoza (MZ) +AR -3319-06621 America/Argentina/San_Luis San Luis (SL) +AR -5138-06913 America/Argentina/Rio_Gallegos Santa Cruz (SC) +AR -5448-06818 America/Argentina/Ushuaia Tierra del Fuego (TF) +AS,UM -1416-17042 Pacific/Pago_Pago Samoa, Midway +AT +4813+01620 Europe/Vienna +AU -3133+15905 Australia/Lord_Howe Lord Howe Island +AU -5430+15857 Antarctica/Macquarie Macquarie Island +AU -4253+14719 Australia/Hobart Tasmania - most locations +AU -3956+14352 Australia/Currie Tasmania - King Island +AU -3749+14458 Australia/Melbourne Victoria +AU -3352+15113 Australia/Sydney New South Wales - most locations +AU -3157+14127 Australia/Broken_Hill New South Wales - Yancowinna +AU -2728+15302 Australia/Brisbane Queensland - most locations +AU -2016+14900 Australia/Lindeman Queensland - Holiday Islands +AU -3455+13835 Australia/Adelaide South Australia +AU -1228+13050 Australia/Darwin Northern Territory +AU -3157+11551 Australia/Perth Western Australia - most locations +AU -3143+12852 Australia/Eucla Western Australia - Eucla area +AZ +4023+04951 Asia/Baku +BB +1306-05937 America/Barbados +BD +2343+09025 Asia/Dhaka +BE +5050+00420 Europe/Brussels +BG +4241+02319 Europe/Sofia +BM +3217-06446 Atlantic/Bermuda +BN +0456+11455 Asia/Brunei +BO -1630-06809 America/La_Paz +BR -0351-03225 America/Noronha Atlantic islands +BR -0127-04829 America/Belem Amapá, E Pará +BR -0343-03830 America/Fortaleza NE Brazil (MA, PI, CE, RN, PB) +BR -0803-03454 America/Recife Pernambuco +BR -0712-04812 America/Araguaina Tocantins +BR -0940-03543 America/Maceio Alagoas, Sergipe +BR -1259-03831 America/Bahia Bahia +BR -2332-04637 America/Sao_Paulo S & SE Brazil (GO, DF, MG, ES, RJ, SP, PR, SC, RS) +BR -2027-05437 America/Campo_Grande Mato Grosso do Sul +BR -1535-05605 America/Cuiaba Mato Grosso +BR -0226-05452 America/Santarem W Pará +BR -0846-06354 America/Porto_Velho Rondônia +BR +0249-06040 America/Boa_Vista Roraima +BR -0308-06001 America/Manaus E Amazonas +BR -0640-06952 America/Eirunepe W Amazonas +BR -0958-06748 America/Rio_Branco Acre +BS +2505-07721 America/Nassau +BT +2728+08939 Asia/Thimphu +BY +5354+02734 Europe/Minsk +BZ +1730-08812 America/Belize +CA +4734-05243 America/St_Johns Newfoundland Time, including SE Labrador +CA +4439-06336 America/Halifax Atlantic Time - Nova Scotia (most places), PEI +CA +4612-05957 America/Glace_Bay Atlantic Time - Nova Scotia - places that did not observe DST 1966-1971 +CA +4606-06447 America/Moncton Atlantic Time - New Brunswick +CA +5320-06025 America/Goose_Bay Atlantic Time - Labrador - most locations +CA +5125-05707 America/Blanc-Sablon Atlantic Standard Time - Quebec - Lower North Shore +CA +4339-07923 America/Toronto Eastern Time - Ontario & Quebec - most locations +CA +4901-08816 America/Nipigon Eastern Time - Ontario & Quebec - places that did not observe DST 1967-1973 +CA +4823-08915 America/Thunder_Bay Eastern Time - Thunder Bay, Ontario +CA +6344-06828 America/Iqaluit Eastern Time - east Nunavut - most locations +CA +6608-06544 America/Pangnirtung Eastern Time - Pangnirtung, Nunavut +CA +744144-0944945 America/Resolute Central Time - Resolute, Nunavut +CA +484531-0913718 America/Atikokan Eastern Standard Time - Atikokan, Ontario and Southampton I, Nunavut +CA +624900-0920459 America/Rankin_Inlet Central Time - central Nunavut +CA +4953-09709 America/Winnipeg Central Time - Manitoba & west Ontario +CA +4843-09434 America/Rainy_River Central Time - Rainy River & Fort Frances, Ontario +CA +5024-10439 America/Regina Central Standard Time - Saskatchewan - most locations +CA +5017-10750 America/Swift_Current Central Standard Time - Saskatchewan - midwest +CA +5333-11328 America/Edmonton Mountain Time - Alberta, east British Columbia & west Saskatchewan +CA +690650-1050310 America/Cambridge_Bay Mountain Time - west Nunavut +CA +6227-11421 America/Yellowknife Mountain Time - central Northwest Territories +CA +682059-1334300 America/Inuvik Mountain Time - west Northwest Territories +CA +4906-11631 America/Creston Mountain Standard Time - Creston, British Columbia +CA +5946-12014 America/Dawson_Creek Mountain Standard Time - Dawson Creek & Fort Saint John, British Columbia +CA +4916-12307 America/Vancouver Pacific Time - west British Columbia +CA +6043-13503 America/Whitehorse Pacific Time - south Yukon +CA +6404-13925 America/Dawson Pacific Time - north Yukon +CC -1210+09655 Indian/Cocos +CH,DE,LI +4723+00832 Europe/Zurich Swiss time +CI,BF,GM,GN,ML,MR,SH,SL,SN,ST,TG +0519-00402 Africa/Abidjan +CK -2114-15946 Pacific/Rarotonga +CL -3327-07040 America/Santiago most locations +CL -2709-10926 Pacific/Easter Easter Island +CN +3114+12128 Asia/Shanghai Beijing Time +CN +4348+08735 Asia/Urumqi Xinjiang Time +CO +0436-07405 America/Bogota +CR +0956-08405 America/Costa_Rica +CU +2308-08222 America/Havana +CV +1455-02331 Atlantic/Cape_Verde +CW,AW,BQ,SX +1211-06900 America/Curacao +CX -1025+10543 Indian/Christmas +CY +3510+03322 Asia/Nicosia +CZ,SK +5005+01426 Europe/Prague +DE +5230+01322 Europe/Berlin Berlin time +DK +5540+01235 Europe/Copenhagen +DO +1828-06954 America/Santo_Domingo +DZ +3647+00303 Africa/Algiers +EC -0210-07950 America/Guayaquil mainland +EC -0054-08936 Pacific/Galapagos Galápagos Islands +EE +5925+02445 Europe/Tallinn +EG +3003+03115 Africa/Cairo +EH +2709-01312 Africa/El_Aaiun +ES +4024-00341 Europe/Madrid mainland +ES +3553-00519 Africa/Ceuta Ceuta & Melilla +ES +2806-01524 Atlantic/Canary Canary Islands +FI,AX +6010+02458 Europe/Helsinki +FJ -1808+17825 Pacific/Fiji +FK -5142-05751 Atlantic/Stanley +FM +0725+15147 Pacific/Chuuk Chuuk (Truk) and Yap +FM +0658+15813 Pacific/Pohnpei Pohnpei (Ponape) +FM +0519+16259 Pacific/Kosrae Kosrae +FO +6201-00646 Atlantic/Faroe +FR +4852+00220 Europe/Paris +GB,GG,IM,JE +513030-0000731 Europe/London +GE +4143+04449 Asia/Tbilisi +GF +0456-05220 America/Cayenne +GH +0533-00013 Africa/Accra +GI +3608-00521 Europe/Gibraltar +GL +6411-05144 America/Godthab most locations +GL +7646-01840 America/Danmarkshavn east coast, north of Scoresbysund +GL +7029-02158 America/Scoresbysund Scoresbysund / Ittoqqortoormiit +GL +7634-06847 America/Thule Thule / Pituffik +GR +3758+02343 Europe/Athens +GS -5416-03632 Atlantic/South_Georgia +GT +1438-09031 America/Guatemala +GU,MP +1328+14445 Pacific/Guam +GW +1151-01535 Africa/Bissau +GY +0648-05810 America/Guyana +HK +2217+11409 Asia/Hong_Kong +HN +1406-08713 America/Tegucigalpa +HT +1832-07220 America/Port-au-Prince +HU +4730+01905 Europe/Budapest +ID -0610+10648 Asia/Jakarta Java & Sumatra +ID -0002+10920 Asia/Pontianak west & central Borneo +ID -0507+11924 Asia/Makassar east & south Borneo, Sulawesi (Celebes), Bali, Nusa Tengarra, west Timor +ID -0232+14042 Asia/Jayapura west New Guinea (Irian Jaya) & Malukus (Moluccas) +IE +5320-00615 Europe/Dublin +IL +314650+0351326 Asia/Jerusalem +IN +2232+08822 Asia/Kolkata +IO -0720+07225 Indian/Chagos +IQ +3321+04425 Asia/Baghdad +IR +3540+05126 Asia/Tehran +IS +6409-02151 Atlantic/Reykjavik +IT,SM,VA +4154+01229 Europe/Rome +JM +175805-0764736 America/Jamaica +JO +3157+03556 Asia/Amman +JP +353916+1394441 Asia/Tokyo +KE,DJ,ER,ET,KM,MG,SO,TZ,UG,YT -0117+03649 Africa/Nairobi +KG +4254+07436 Asia/Bishkek +KI +0125+17300 Pacific/Tarawa Gilbert Islands +KI -0308-17105 Pacific/Enderbury Phoenix Islands +KI +0152-15720 Pacific/Kiritimati Line Islands +KP +3901+12545 Asia/Pyongyang +KR +3733+12658 Asia/Seoul +KZ +4315+07657 Asia/Almaty most locations +KZ +4448+06528 Asia/Qyzylorda Qyzylorda (Kyzylorda, Kzyl-Orda) +KZ +5017+05710 Asia/Aqtobe Aqtobe (Aktobe) +KZ +4431+05016 Asia/Aqtau Atyrau (Atirau, Gur'yev), Mangghystau (Mankistau) +KZ +5113+05121 Asia/Oral West Kazakhstan +LB +3353+03530 Asia/Beirut +LK +0656+07951 Asia/Colombo +LR +0618-01047 Africa/Monrovia +LT +5441+02519 Europe/Vilnius +LU +4936+00609 Europe/Luxembourg +LV +5657+02406 Europe/Riga +LY +3254+01311 Africa/Tripoli +MA +3339-00735 Africa/Casablanca +MC +4342+00723 Europe/Monaco +MD +4700+02850 Europe/Chisinau +MH +0709+17112 Pacific/Majuro most locations +MH +0905+16720 Pacific/Kwajalein Kwajalein +MM +1647+09610 Asia/Rangoon +MN +4755+10653 Asia/Ulaanbaatar most locations +MN +4801+09139 Asia/Hovd Bayan-Ölgii, Govi-Altai, Hovd, Uvs, Zavkhan +MN +4804+11430 Asia/Choibalsan Dornod, Sükhbaatar +MO +2214+11335 Asia/Macau +MQ +1436-06105 America/Martinique +MT +3554+01431 Europe/Malta +MU -2010+05730 Indian/Mauritius +MV +0410+07330 Indian/Maldives +MX +1924-09909 America/Mexico_City Central Time - most locations +MX +2105-08646 America/Cancun Central Time - Quintana Roo +MX +2058-08937 America/Merida Central Time - Campeche, Yucatán +MX +2540-10019 America/Monterrey Mexican Central Time - Coahuila, Durango, Nuevo León, Tamaulipas away from US border +MX +2550-09730 America/Matamoros US Central Time - Coahuila, Durango, Nuevo León, Tamaulipas near US border +MX +2313-10625 America/Mazatlan Mountain Time - S Baja, Nayarit, Sinaloa +MX +2838-10605 America/Chihuahua Mexican Mountain Time - Chihuahua away from US border +MX +2934-10425 America/Ojinaga US Mountain Time - Chihuahua near US border +MX +2904-11058 America/Hermosillo Mountain Standard Time - Sonora +MX +3232-11701 America/Tijuana US Pacific Time - Baja California near US border +MX +3018-11452 America/Santa_Isabel Mexican Pacific Time - Baja California away from US border +MX +2048-10515 America/Bahia_Banderas Mexican Central Time - Bahía de Banderas +MY +0310+10142 Asia/Kuala_Lumpur peninsular Malaysia +MY +0133+11020 Asia/Kuching Sabah & Sarawak +MZ,BI,BW,CD,MW,RW,ZM,ZW -2558+03235 Africa/Maputo Central Africa Time (UTC+2) +NA -2234+01706 Africa/Windhoek +NC -2216+16627 Pacific/Noumea +NF -2903+16758 Pacific/Norfolk +NG,AO,BJ,CD,CF,CG,CM,GA,GQ,NE +0627+00324 Africa/Lagos West Africa Time (UTC+1) +NI +1209-08617 America/Managua +NL +5222+00454 Europe/Amsterdam +NO,SJ +5955+01045 Europe/Oslo +NP +2743+08519 Asia/Kathmandu +NR -0031+16655 Pacific/Nauru +NU -1901-16955 Pacific/Niue +NZ,AQ -3652+17446 Pacific/Auckland New Zealand time +NZ -4357-17633 Pacific/Chatham Chatham Islands +PA,KY +0858-07932 America/Panama +PE -1203-07703 America/Lima +PF -1732-14934 Pacific/Tahiti Society Islands +PF -0900-13930 Pacific/Marquesas Marquesas Islands +PF -2308-13457 Pacific/Gambier Gambier Islands +PG -0930+14710 Pacific/Port_Moresby +PH +1435+12100 Asia/Manila +PK +2452+06703 Asia/Karachi +PL +5215+02100 Europe/Warsaw +PM +4703-05620 America/Miquelon +PN -2504-13005 Pacific/Pitcairn +PR +182806-0660622 America/Puerto_Rico +PS +3130+03428 Asia/Gaza Gaza Strip +PS +313200+0350542 Asia/Hebron West Bank +PT +3843-00908 Europe/Lisbon mainland +PT +3238-01654 Atlantic/Madeira Madeira Islands +PT +3744-02540 Atlantic/Azores Azores +PW +0720+13429 Pacific/Palau +PY -2516-05740 America/Asuncion +QA,BH +2517+05132 Asia/Qatar +RE,TF -2052+05528 Indian/Reunion Réunion, Crozet Is, Scattered Is +RO +4426+02606 Europe/Bucharest +RS,BA,HR,ME,MK,SI +4450+02030 Europe/Belgrade +RU +5443+02030 Europe/Kaliningrad Moscow-01 - Kaliningrad +RU +554521+0373704 Europe/Moscow Moscow+00 - west Russia +RU +4457+03406 Europe/Simferopol Moscow+00 - Crimea +RU +4844+04425 Europe/Volgograd Moscow+00 - Caspian Sea +RU +5312+05009 Europe/Samara Moscow+00 (Moscow+01 after 2014-10-26) - Samara, Udmurtia +RU +5651+06036 Asia/Yekaterinburg Moscow+02 - Urals +RU +5500+07324 Asia/Omsk Moscow+03 - west Siberia +RU +5502+08255 Asia/Novosibirsk Moscow+03 - Novosibirsk +RU +5345+08707 Asia/Novokuznetsk Moscow+03 (Moscow+04 after 2014-10-26) - Kemerovo +RU +5601+09250 Asia/Krasnoyarsk Moscow+04 - Yenisei River +RU +5216+10420 Asia/Irkutsk Moscow+05 - Lake Baikal +RU +5203+11328 Asia/Chita Moscow+06 (Moscow+05 after 2014-10-26) - Zabaykalsky +RU +6200+12940 Asia/Yakutsk Moscow+06 - Lena River +RU +623923+1353314 Asia/Khandyga Moscow+06 - Tomponsky, Ust-Maysky +RU +4310+13156 Asia/Vladivostok Moscow+07 - Amur River +RU +4658+14242 Asia/Sakhalin Moscow+07 - Sakhalin Island +RU +643337+1431336 Asia/Ust-Nera Moscow+07 - Oymyakonsky +RU +5934+15048 Asia/Magadan Moscow+08 (Moscow+07 after 2014-10-26) - Magadan +RU +6728+15343 Asia/Srednekolymsk Moscow+08 - E Sakha, N Kuril Is +RU +5301+15839 Asia/Kamchatka Moscow+08 (Moscow+09 after 2014-10-26) - Kamchatka +RU +6445+17729 Asia/Anadyr Moscow+08 (Moscow+09 after 2014-10-26) - Bering Sea +SA,KW,YE +2438+04643 Asia/Riyadh +SB -0932+16012 Pacific/Guadalcanal +SC -0440+05528 Indian/Mahe +SD,SS +1536+03232 Africa/Khartoum +SE +5920+01803 Europe/Stockholm +SG +0117+10351 Asia/Singapore +SR +0550-05510 America/Paramaribo +SV +1342-08912 America/El_Salvador +SY +3330+03618 Asia/Damascus +TC +2128-07108 America/Grand_Turk +TD +1207+01503 Africa/Ndjamena +TF -492110+0701303 Indian/Kerguelen Kerguelen, St Paul I, Amsterdam I +TH,KH,LA,VN +1345+10031 Asia/Bangkok +TJ +3835+06848 Asia/Dushanbe +TK -0922-17114 Pacific/Fakaofo +TL -0833+12535 Asia/Dili +TM +3757+05823 Asia/Ashgabat +TN +3648+01011 Africa/Tunis +TO -2110-17510 Pacific/Tongatapu +TR +4101+02858 Europe/Istanbul +TT,AG,AI,BL,DM,GD,GP,MF,LC,KN,MS,VC,VG,VI +1039-06131 America/Port_of_Spain +TV -0831+17913 Pacific/Funafuti +TW +2503+12130 Asia/Taipei +UA +5026+03031 Europe/Kiev most locations +UA +4837+02218 Europe/Uzhgorod Ruthenia +UA +4750+03510 Europe/Zaporozhye Zaporozh'ye, E Lugansk / Zaporizhia, E Luhansk +UM +1917+16637 Pacific/Wake Wake Island +US +404251-0740023 America/New_York Eastern Time +US +421953-0830245 America/Detroit Eastern Time - Michigan - most locations +US +381515-0854534 America/Kentucky/Louisville Eastern Time - Kentucky - Louisville area +US +364947-0845057 America/Kentucky/Monticello Eastern Time - Kentucky - Wayne County +US +394606-0860929 America/Indiana/Indianapolis Eastern Time - Indiana - most locations +US +384038-0873143 America/Indiana/Vincennes Eastern Time - Indiana - Daviess, Dubois, Knox & Martin Counties +US +410305-0863611 America/Indiana/Winamac Eastern Time - Indiana - Pulaski County +US +382232-0862041 America/Indiana/Marengo Eastern Time - Indiana - Crawford County +US +382931-0871643 America/Indiana/Petersburg Eastern Time - Indiana - Pike County +US +384452-0850402 America/Indiana/Vevay Eastern Time - Indiana - Switzerland County +US +415100-0873900 America/Chicago Central Time +US +375711-0864541 America/Indiana/Tell_City Central Time - Indiana - Perry County +US +411745-0863730 America/Indiana/Knox Central Time - Indiana - Starke County +US +450628-0873651 America/Menominee Central Time - Michigan - Dickinson, Gogebic, Iron & Menominee Counties +US +470659-1011757 America/North_Dakota/Center Central Time - North Dakota - Oliver County +US +465042-1012439 America/North_Dakota/New_Salem Central Time - North Dakota - Morton County (except Mandan area) +US +471551-1014640 America/North_Dakota/Beulah Central Time - North Dakota - Mercer County +US +394421-1045903 America/Denver Mountain Time +US +433649-1161209 America/Boise Mountain Time - south Idaho & east Oregon +US +332654-1120424 America/Phoenix Mountain Standard Time - Arizona (except Navajo) +US +340308-1181434 America/Los_Angeles Pacific Time +US +550737-1313435 America/Metlakatla Pacific Standard Time - Annette Island, Alaska +US +611305-1495401 America/Anchorage Alaska Time +US +581807-1342511 America/Juneau Alaska Time - Alaska panhandle +US +571035-1351807 America/Sitka Alaska Time - southeast Alaska panhandle +US +593249-1394338 America/Yakutat Alaska Time - Alaska panhandle neck +US +643004-1652423 America/Nome Alaska Time - west Alaska +US +515248-1763929 America/Adak Aleutian Islands +US,UM +211825-1575130 Pacific/Honolulu Hawaii time +UY -3453-05611 America/Montevideo +UZ +3940+06648 Asia/Samarkand west Uzbekistan +UZ +4120+06918 Asia/Tashkent east Uzbekistan +VE +1030-06656 America/Caracas +VU -1740+16825 Pacific/Efate +WF -1318-17610 Pacific/Wallis +WS -1350-17144 Pacific/Apia +ZA,LS,SZ -2615+02800 Africa/Johannesburg diff --git a/lib/tzlocal/LICENSE.txt b/lib/tzlocal/LICENSE.txt new file mode 100644 index 00000000..0e259d42 --- /dev/null +++ b/lib/tzlocal/LICENSE.txt @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/lib/tzlocal/__init__.py b/lib/tzlocal/__init__.py new file mode 100644 index 00000000..df7a66b9 --- /dev/null +++ b/lib/tzlocal/__init__.py @@ -0,0 +1,7 @@ +import sys +if sys.platform == 'win32': + from tzlocal.win32 import get_localzone, reload_localzone +elif 'darwin' in sys.platform: + from tzlocal.darwin import get_localzone, reload_localzone +else: + from tzlocal.unix import get_localzone, reload_localzone diff --git a/lib/tzlocal/darwin.py b/lib/tzlocal/darwin.py new file mode 100644 index 00000000..86fd906f --- /dev/null +++ b/lib/tzlocal/darwin.py @@ -0,0 +1,27 @@ +from __future__ import with_statement +import os +import pytz + +_cache_tz = None + +def _get_localzone(): + tzname = os.popen("systemsetup -gettimezone").read().replace("Time Zone: ", "").strip() + if not tzname or tzname not in pytz.all_timezones_set: + # link will be something like /usr/share/zoneinfo/America/Los_Angeles. + link = os.readlink("/etc/localtime") + tzname = link[link.rfind("zoneinfo/") + 9:] + return pytz.timezone(tzname) + +def get_localzone(): + """Get the computers configured local timezone, if any.""" + global _cache_tz + if _cache_tz is None: + _cache_tz = _get_localzone() + return _cache_tz + +def reload_localzone(): + """Reload the cached localzone. You need to call this if the timezone has changed.""" + global _cache_tz + _cache_tz = _get_localzone() + return _cache_tz + diff --git a/lib/tzlocal/tests.py b/lib/tzlocal/tests.py new file mode 100644 index 00000000..49dd0aef --- /dev/null +++ b/lib/tzlocal/tests.py @@ -0,0 +1,64 @@ +import sys +import os +from datetime import datetime +import unittest +import pytz +import tzlocal.unix + +class TzLocalTests(unittest.TestCase): + + def test_env(self): + tz_harare = tzlocal.unix._tz_from_env(':Africa/Harare') + self.assertEqual(tz_harare.zone, 'Africa/Harare') + + # Some Unices allow this as well, so we must allow it: + tz_harare = tzlocal.unix._tz_from_env('Africa/Harare') + self.assertEqual(tz_harare.zone, 'Africa/Harare') + + local_path = os.path.split(__file__)[0] + tz_local = tzlocal.unix._tz_from_env(':' + os.path.join(local_path, 'test_data', 'Harare')) + self.assertEqual(tz_local.zone, 'local') + # Make sure the local timezone is the same as the Harare one above. + # We test this with a past date, so that we don't run into future changes + # of the Harare timezone. + dt = datetime(2012, 1, 1, 5) + self.assertEqual(tz_harare.localize(dt), tz_local.localize(dt)) + + # Non-zoneinfo timezones are not supported in the TZ environment. + self.assertRaises(pytz.UnknownTimeZoneError, tzlocal.unix._tz_from_env, 'GMT+03:00') + + def test_timezone(self): + # Most versions of Ubuntu + local_path = os.path.split(__file__)[0] + tz = tzlocal.unix._get_localzone(_root=os.path.join(local_path, 'test_data', 'timezone')) + self.assertEqual(tz.zone, 'Africa/Harare') + + def test_zone_setting(self): + # A ZONE setting in /etc/sysconfig/clock, f ex CentOS + local_path = os.path.split(__file__)[0] + tz = tzlocal.unix._get_localzone(_root=os.path.join(local_path, 'test_data', 'zone_setting')) + self.assertEqual(tz.zone, 'Africa/Harare') + + def test_timezone_setting(self): + # A ZONE setting in /etc/conf.d/clock, f ex Gentoo + local_path = os.path.split(__file__)[0] + tz = tzlocal.unix._get_localzone(_root=os.path.join(local_path, 'test_data', 'timezone_setting')) + self.assertEqual(tz.zone, 'Africa/Harare') + + def test_only_localtime(self): + local_path = os.path.split(__file__)[0] + tz = tzlocal.unix._get_localzone(_root=os.path.join(local_path, 'test_data', 'localtime')) + self.assertEqual(tz.zone, 'local') + dt = datetime(2012, 1, 1, 5) + self.assertEqual(pytz.timezone('Africa/Harare').localize(dt), tz.localize(dt)) + +if sys.platform == 'win32': + + import tzlocal.win32 + class TzWin32Tests(unittest.TestCase): + + def test_win32(self): + tzlocal.win32.get_localzone() + +if __name__ == '__main__': + unittest.main() diff --git a/lib/tzlocal/unix.py b/lib/tzlocal/unix.py new file mode 100644 index 00000000..76c214dd --- /dev/null +++ b/lib/tzlocal/unix.py @@ -0,0 +1,115 @@ +from __future__ import with_statement +import os +import re +import pytz + +_cache_tz = None + +def _tz_from_env(tzenv): + if tzenv[0] == ':': + tzenv = tzenv[1:] + + # TZ specifies a file + if os.path.exists(tzenv): + with open(tzenv, 'rb') as tzfile: + return pytz.tzfile.build_tzinfo('local', tzfile) + + # TZ specifies a zoneinfo zone. + try: + tz = pytz.timezone(tzenv) + # That worked, so we return this: + return tz + except pytz.UnknownTimeZoneError: + raise pytz.UnknownTimeZoneError( + "tzlocal() does not support non-zoneinfo timezones like %s. \n" + "Please use a timezone in the form of Continent/City") + +def _get_localzone(_root='/'): + """Tries to find the local timezone configuration. + + This method prefers finding the timezone name and passing that to pytz, + over passing in the localtime file, as in the later case the zoneinfo + name is unknown. + + The parameter _root makes the function look for files like /etc/localtime + beneath the _root directory. This is primarily used by the tests. + In normal usage you call the function without parameters.""" + + tzenv = os.environ.get('TZ') + if tzenv: + try: + return _tz_from_env(tzenv) + except pytz.UnknownTimeZoneError: + pass + + # Now look for distribution specific configuration files + # that contain the timezone name. + tzpath = os.path.join(_root, 'etc/timezone') + if os.path.exists(tzpath): + with open(tzpath, 'rb') as tzfile: + data = tzfile.read() + + # Issue #3 was that /etc/timezone was a zoneinfo file. + # That's a misconfiguration, but we need to handle it gracefully: + if data[:5] != 'TZif2': + etctz = data.strip().decode() + # Get rid of host definitions and comments: + if ' ' in etctz: + etctz, dummy = etctz.split(' ', 1) + if '#' in etctz: + etctz, dummy = etctz.split('#', 1) + return pytz.timezone(etctz.replace(' ', '_')) + + # CentOS has a ZONE setting in /etc/sysconfig/clock, + # OpenSUSE has a TIMEZONE setting in /etc/sysconfig/clock and + # Gentoo has a TIMEZONE setting in /etc/conf.d/clock + # We look through these files for a timezone: + + zone_re = re.compile('\s*ZONE\s*=\s*\"') + timezone_re = re.compile('\s*TIMEZONE\s*=\s*\"') + end_re = re.compile('\"') + + for filename in ('etc/sysconfig/clock', 'etc/conf.d/clock'): + tzpath = os.path.join(_root, filename) + if not os.path.exists(tzpath): + continue + with open(tzpath, 'rt') as tzfile: + data = tzfile.readlines() + + for line in data: + # Look for the ZONE= setting. + match = zone_re.match(line) + if match is None: + # No ZONE= setting. Look for the TIMEZONE= setting. + match = timezone_re.match(line) + if match is not None: + # Some setting existed + line = line[match.end():] + etctz = line[:end_re.search(line).start()] + + # We found a timezone + return pytz.timezone(etctz.replace(' ', '_')) + + # No explicit setting existed. Use localtime + for filename in ('etc/localtime', 'usr/local/etc/localtime'): + tzpath = os.path.join(_root, filename) + + if not os.path.exists(tzpath): + continue + with open(tzpath, 'rb') as tzfile: + return pytz.tzfile.build_tzinfo('local', tzfile) + + raise pytz.UnknownTimeZoneError('Can not find any timezone configuration') + +def get_localzone(): + """Get the computers configured local timezone, if any.""" + global _cache_tz + if _cache_tz is None: + _cache_tz = _get_localzone() + return _cache_tz + +def reload_localzone(): + """Reload the cached localzone. You need to call this if the timezone has changed.""" + global _cache_tz + _cache_tz = _get_localzone() + return _cache_tz diff --git a/lib/tzlocal/win32.py b/lib/tzlocal/win32.py new file mode 100644 index 00000000..63445cd7 --- /dev/null +++ b/lib/tzlocal/win32.py @@ -0,0 +1,93 @@ +try: + import _winreg as winreg +except ImportError: + import winreg + +from tzlocal.windows_tz import win_tz +import pytz + +_cache_tz = None + +def valuestodict(key): + """Convert a registry key's values to a dictionary.""" + dict = {} + size = winreg.QueryInfoKey(key)[1] + for i in range(size): + data = winreg.EnumValue(key, i) + dict[data[0]] = data[1] + return dict + +def get_localzone_name(): + # Windows is special. It has unique time zone names (in several + # meanings of the word) available, but unfortunately, they can be + # translated to the language of the operating system, so we need to + # do a backwards lookup, by going through all time zones and see which + # one matches. + handle = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) + + TZLOCALKEYNAME = r"SYSTEM\CurrentControlSet\Control\TimeZoneInformation" + localtz = winreg.OpenKey(handle, TZLOCALKEYNAME) + keyvalues = valuestodict(localtz) + localtz.Close() + if 'TimeZoneKeyName' in keyvalues: + # Windows 7 (and Vista?) + + # For some reason this returns a string with loads of NUL bytes at + # least on some systems. I don't know if this is a bug somewhere, I + # just work around it. + tzkeyname = keyvalues['TimeZoneKeyName'].split('\x00', 1)[0] + else: + # Windows 2000 or XP + + # This is the localized name: + tzwin = keyvalues['StandardName'] + + # Open the list of timezones to look up the real name: + TZKEYNAME = r"SOFTWARE\Microsoft\Windows NT\CurrentVersion\Time Zones" + tzkey = winreg.OpenKey(handle, TZKEYNAME) + + # Now, match this value to Time Zone information + tzkeyname = None + for i in range(winreg.QueryInfoKey(tzkey)[0]): + subkey = winreg.EnumKey(tzkey, i) + sub = winreg.OpenKey(tzkey, subkey) + data = valuestodict(sub) + sub.Close() + try: + if data['Std'] == tzwin: + tzkeyname = subkey + break + except KeyError: + # This timezone didn't have proper configuration. + # Ignore it. + pass + + tzkey.Close() + handle.Close() + + if tzkeyname is None: + raise LookupError('Can not find Windows timezone configuration') + + timezone = win_tz.get(tzkeyname) + if timezone is None: + # Nope, that didn't work. Try adding "Standard Time", + # it seems to work a lot of times: + timezone = win_tz.get(tzkeyname + " Standard Time") + + # Return what we have. + if timezone is None: + raise pytz.UnknownTimeZoneError('Can not find timezone ' + tzkeyname) + + return timezone + +def get_localzone(): + """Returns the zoneinfo-based tzinfo object that matches the Windows-configured timezone.""" + global _cache_tz + if _cache_tz is None: + _cache_tz = pytz.timezone(get_localzone_name()) + return _cache_tz + +def reload_localzone(): + """Reload the cached localzone. You need to call this if the timezone has changed.""" + global _cache_tz + _cache_tz = pytz.timezone(get_localzone_name()) diff --git a/lib/tzlocal/windows_tz.py b/lib/tzlocal/windows_tz.py new file mode 100644 index 00000000..0790bb48 --- /dev/null +++ b/lib/tzlocal/windows_tz.py @@ -0,0 +1,542 @@ +# This file is autogenerated by the get_windows_info.py script +# Do not edit. +win_tz = {'AUS Central Standard Time': 'Australia/Darwin', + 'AUS Eastern Standard Time': 'Australia/Sydney', + 'Afghanistan Standard Time': 'Asia/Kabul', + 'Alaskan Standard Time': 'America/Anchorage', + 'Arab Standard Time': 'Asia/Riyadh', + 'Arabian Standard Time': 'Asia/Dubai', + 'Arabic Standard Time': 'Asia/Baghdad', + 'Argentina Standard Time': 'America/Buenos_Aires', + 'Atlantic Standard Time': 'America/Halifax', + 'Azerbaijan Standard Time': 'Asia/Baku', + 'Azores Standard Time': 'Atlantic/Azores', + 'Bahia Standard Time': 'America/Bahia', + 'Bangladesh Standard Time': 'Asia/Dhaka', + 'Belarus Standard Time': 'Europe/Minsk', + 'Canada Central Standard Time': 'America/Regina', + 'Cape Verde Standard Time': 'Atlantic/Cape_Verde', + 'Caucasus Standard Time': 'Asia/Yerevan', + 'Cen. Australia Standard Time': 'Australia/Adelaide', + 'Central America Standard Time': 'America/Guatemala', + 'Central Asia Standard Time': 'Asia/Almaty', + 'Central Brazilian Standard Time': 'America/Cuiaba', + 'Central Europe Standard Time': 'Europe/Budapest', + 'Central European Standard Time': 'Europe/Warsaw', + 'Central Pacific Standard Time': 'Pacific/Guadalcanal', + 'Central Standard Time': 'America/Chicago', + 'Central Standard Time (Mexico)': 'America/Mexico_City', + 'China Standard Time': 'Asia/Shanghai', + 'Dateline Standard Time': 'Etc/GMT+12', + 'E. Africa Standard Time': 'Africa/Nairobi', + 'E. Australia Standard Time': 'Australia/Brisbane', + 'E. South America Standard Time': 'America/Sao_Paulo', + 'Eastern Standard Time': 'America/New_York', + 'Egypt Standard Time': 'Africa/Cairo', + 'Ekaterinburg Standard Time': 'Asia/Yekaterinburg', + 'FLE Standard Time': 'Europe/Kiev', + 'Fiji Standard Time': 'Pacific/Fiji', + 'GMT Standard Time': 'Europe/London', + 'GTB Standard Time': 'Europe/Bucharest', + 'Georgian Standard Time': 'Asia/Tbilisi', + 'Greenland Standard Time': 'America/Godthab', + 'Greenwich Standard Time': 'Atlantic/Reykjavik', + 'Hawaiian Standard Time': 'Pacific/Honolulu', + 'India Standard Time': 'Asia/Calcutta', + 'Iran Standard Time': 'Asia/Tehran', + 'Israel Standard Time': 'Asia/Jerusalem', + 'Jordan Standard Time': 'Asia/Amman', + 'Kaliningrad Standard Time': 'Europe/Kaliningrad', + 'Korea Standard Time': 'Asia/Seoul', + 'Libya Standard Time': 'Africa/Tripoli', + 'Line Islands Standard Time': 'Pacific/Kiritimati', + 'Magadan Standard Time': 'Asia/Magadan', + 'Mauritius Standard Time': 'Indian/Mauritius', + 'Middle East Standard Time': 'Asia/Beirut', + 'Montevideo Standard Time': 'America/Montevideo', + 'Morocco Standard Time': 'Africa/Casablanca', + 'Mountain Standard Time': 'America/Denver', + 'Mountain Standard Time (Mexico)': 'America/Chihuahua', + 'Myanmar Standard Time': 'Asia/Rangoon', + 'N. Central Asia Standard Time': 'Asia/Novosibirsk', + 'Namibia Standard Time': 'Africa/Windhoek', + 'Nepal Standard Time': 'Asia/Katmandu', + 'New Zealand Standard Time': 'Pacific/Auckland', + 'Newfoundland Standard Time': 'America/St_Johns', + 'North Asia East Standard Time': 'Asia/Irkutsk', + 'North Asia Standard Time': 'Asia/Krasnoyarsk', + 'Pacific SA Standard Time': 'America/Santiago', + 'Pacific Standard Time': 'America/Los_Angeles', + 'Pacific Standard Time (Mexico)': 'America/Santa_Isabel', + 'Pakistan Standard Time': 'Asia/Karachi', + 'Paraguay Standard Time': 'America/Asuncion', + 'Romance Standard Time': 'Europe/Paris', + 'Russia Time Zone 10': 'Asia/Srednekolymsk', + 'Russia Time Zone 11': 'Asia/Kamchatka', + 'Russia Time Zone 3': 'Europe/Samara', + 'Russian Standard Time': 'Europe/Moscow', + 'SA Eastern Standard Time': 'America/Cayenne', + 'SA Pacific Standard Time': 'America/Bogota', + 'SA Western Standard Time': 'America/La_Paz', + 'SE Asia Standard Time': 'Asia/Bangkok', + 'Samoa Standard Time': 'Pacific/Apia', + 'Singapore Standard Time': 'Asia/Singapore', + 'South Africa Standard Time': 'Africa/Johannesburg', + 'Sri Lanka Standard Time': 'Asia/Colombo', + 'Syria Standard Time': 'Asia/Damascus', + 'Taipei Standard Time': 'Asia/Taipei', + 'Tasmania Standard Time': 'Australia/Hobart', + 'Tokyo Standard Time': 'Asia/Tokyo', + 'Tonga Standard Time': 'Pacific/Tongatapu', + 'Turkey Standard Time': 'Europe/Istanbul', + 'US Eastern Standard Time': 'America/Indianapolis', + 'US Mountain Standard Time': 'America/Phoenix', + 'UTC': 'Etc/GMT', + 'UTC+12': 'Etc/GMT-12', + 'UTC-02': 'Etc/GMT+2', + 'UTC-11': 'Etc/GMT+11', + 'Ulaanbaatar Standard Time': 'Asia/Ulaanbaatar', + 'Venezuela Standard Time': 'America/Caracas', + 'Vladivostok Standard Time': 'Asia/Vladivostok', + 'W. Australia Standard Time': 'Australia/Perth', + 'W. Central Africa Standard Time': 'Africa/Lagos', + 'W. Europe Standard Time': 'Europe/Berlin', + 'West Asia Standard Time': 'Asia/Tashkent', + 'West Pacific Standard Time': 'Pacific/Port_Moresby', + 'Yakutsk Standard Time': 'Asia/Yakutsk'} + +# Old name for the win_tz variable: +tz_names = win_tz + +tz_win = {'Africa/Abidjan': 'Greenwich Standard Time', + 'Africa/Accra': 'Greenwich Standard Time', + 'Africa/Addis_Ababa': 'E. Africa Standard Time', + 'Africa/Algiers': 'W. Central Africa Standard Time', + 'Africa/Asmera': 'E. Africa Standard Time', + 'Africa/Bamako': 'Greenwich Standard Time', + 'Africa/Bangui': 'W. Central Africa Standard Time', + 'Africa/Banjul': 'Greenwich Standard Time', + 'Africa/Bissau': 'Greenwich Standard Time', + 'Africa/Blantyre': 'South Africa Standard Time', + 'Africa/Brazzaville': 'W. Central Africa Standard Time', + 'Africa/Bujumbura': 'South Africa Standard Time', + 'Africa/Cairo': 'Egypt Standard Time', + 'Africa/Casablanca': 'Morocco Standard Time', + 'Africa/Ceuta': 'Romance Standard Time', + 'Africa/Conakry': 'Greenwich Standard Time', + 'Africa/Dakar': 'Greenwich Standard Time', + 'Africa/Dar_es_Salaam': 'E. Africa Standard Time', + 'Africa/Djibouti': 'E. Africa Standard Time', + 'Africa/Douala': 'W. Central Africa Standard Time', + 'Africa/El_Aaiun': 'Morocco Standard Time', + 'Africa/Freetown': 'Greenwich Standard Time', + 'Africa/Gaborone': 'South Africa Standard Time', + 'Africa/Harare': 'South Africa Standard Time', + 'Africa/Johannesburg': 'South Africa Standard Time', + 'Africa/Juba': 'E. Africa Standard Time', + 'Africa/Kampala': 'E. Africa Standard Time', + 'Africa/Khartoum': 'E. Africa Standard Time', + 'Africa/Kigali': 'South Africa Standard Time', + 'Africa/Kinshasa': 'W. Central Africa Standard Time', + 'Africa/Lagos': 'W. Central Africa Standard Time', + 'Africa/Libreville': 'W. Central Africa Standard Time', + 'Africa/Lome': 'Greenwich Standard Time', + 'Africa/Luanda': 'W. Central Africa Standard Time', + 'Africa/Lubumbashi': 'South Africa Standard Time', + 'Africa/Lusaka': 'South Africa Standard Time', + 'Africa/Malabo': 'W. Central Africa Standard Time', + 'Africa/Maputo': 'South Africa Standard Time', + 'Africa/Maseru': 'South Africa Standard Time', + 'Africa/Mbabane': 'South Africa Standard Time', + 'Africa/Mogadishu': 'E. Africa Standard Time', + 'Africa/Monrovia': 'Greenwich Standard Time', + 'Africa/Nairobi': 'E. Africa Standard Time', + 'Africa/Ndjamena': 'W. Central Africa Standard Time', + 'Africa/Niamey': 'W. Central Africa Standard Time', + 'Africa/Nouakchott': 'Greenwich Standard Time', + 'Africa/Ouagadougou': 'Greenwich Standard Time', + 'Africa/Porto-Novo': 'W. Central Africa Standard Time', + 'Africa/Sao_Tome': 'Greenwich Standard Time', + 'Africa/Tripoli': 'Libya Standard Time', + 'Africa/Tunis': 'W. Central Africa Standard Time', + 'Africa/Windhoek': 'Namibia Standard Time', + 'America/Anchorage': 'Alaskan Standard Time', + 'America/Anguilla': 'SA Western Standard Time', + 'America/Antigua': 'SA Western Standard Time', + 'America/Araguaina': 'SA Eastern Standard Time', + 'America/Argentina/La_Rioja': 'Argentina Standard Time', + 'America/Argentina/Rio_Gallegos': 'Argentina Standard Time', + 'America/Argentina/Salta': 'Argentina Standard Time', + 'America/Argentina/San_Juan': 'Argentina Standard Time', + 'America/Argentina/San_Luis': 'Argentina Standard Time', + 'America/Argentina/Tucuman': 'Argentina Standard Time', + 'America/Argentina/Ushuaia': 'Argentina Standard Time', + 'America/Aruba': 'SA Western Standard Time', + 'America/Asuncion': 'Paraguay Standard Time', + 'America/Bahia': 'Bahia Standard Time', + 'America/Bahia_Banderas': 'Central Standard Time (Mexico)', + 'America/Barbados': 'SA Western Standard Time', + 'America/Belem': 'SA Eastern Standard Time', + 'America/Belize': 'Central America Standard Time', + 'America/Blanc-Sablon': 'SA Western Standard Time', + 'America/Boa_Vista': 'SA Western Standard Time', + 'America/Bogota': 'SA Pacific Standard Time', + 'America/Boise': 'Mountain Standard Time', + 'America/Buenos_Aires': 'Argentina Standard Time', + 'America/Cambridge_Bay': 'Mountain Standard Time', + 'America/Campo_Grande': 'Central Brazilian Standard Time', + 'America/Cancun': 'Central Standard Time (Mexico)', + 'America/Caracas': 'Venezuela Standard Time', + 'America/Catamarca': 'Argentina Standard Time', + 'America/Cayenne': 'SA Eastern Standard Time', + 'America/Cayman': 'SA Pacific Standard Time', + 'America/Chicago': 'Central Standard Time', + 'America/Chihuahua': 'Mountain Standard Time (Mexico)', + 'America/Coral_Harbour': 'SA Pacific Standard Time', + 'America/Cordoba': 'Argentina Standard Time', + 'America/Costa_Rica': 'Central America Standard Time', + 'America/Creston': 'US Mountain Standard Time', + 'America/Cuiaba': 'Central Brazilian Standard Time', + 'America/Curacao': 'SA Western Standard Time', + 'America/Danmarkshavn': 'UTC', + 'America/Dawson': 'Pacific Standard Time', + 'America/Dawson_Creek': 'US Mountain Standard Time', + 'America/Denver': 'Mountain Standard Time', + 'America/Detroit': 'Eastern Standard Time', + 'America/Dominica': 'SA Western Standard Time', + 'America/Edmonton': 'Mountain Standard Time', + 'America/Eirunepe': 'SA Pacific Standard Time', + 'America/El_Salvador': 'Central America Standard Time', + 'America/Fortaleza': 'SA Eastern Standard Time', + 'America/Glace_Bay': 'Atlantic Standard Time', + 'America/Godthab': 'Greenland Standard Time', + 'America/Goose_Bay': 'Atlantic Standard Time', + 'America/Grand_Turk': 'SA Western Standard Time', + 'America/Grenada': 'SA Western Standard Time', + 'America/Guadeloupe': 'SA Western Standard Time', + 'America/Guatemala': 'Central America Standard Time', + 'America/Guayaquil': 'SA Pacific Standard Time', + 'America/Guyana': 'SA Western Standard Time', + 'America/Halifax': 'Atlantic Standard Time', + 'America/Havana': 'Eastern Standard Time', + 'America/Hermosillo': 'US Mountain Standard Time', + 'America/Indiana/Knox': 'Central Standard Time', + 'America/Indiana/Marengo': 'US Eastern Standard Time', + 'America/Indiana/Petersburg': 'Eastern Standard Time', + 'America/Indiana/Tell_City': 'Central Standard Time', + 'America/Indiana/Vevay': 'US Eastern Standard Time', + 'America/Indiana/Vincennes': 'Eastern Standard Time', + 'America/Indiana/Winamac': 'Eastern Standard Time', + 'America/Indianapolis': 'US Eastern Standard Time', + 'America/Inuvik': 'Mountain Standard Time', + 'America/Iqaluit': 'Eastern Standard Time', + 'America/Jamaica': 'SA Pacific Standard Time', + 'America/Jujuy': 'Argentina Standard Time', + 'America/Juneau': 'Alaskan Standard Time', + 'America/Kentucky/Monticello': 'Eastern Standard Time', + 'America/Kralendijk': 'SA Western Standard Time', + 'America/La_Paz': 'SA Western Standard Time', + 'America/Lima': 'SA Pacific Standard Time', + 'America/Los_Angeles': 'Pacific Standard Time', + 'America/Louisville': 'Eastern Standard Time', + 'America/Lower_Princes': 'SA Western Standard Time', + 'America/Maceio': 'SA Eastern Standard Time', + 'America/Managua': 'Central America Standard Time', + 'America/Manaus': 'SA Western Standard Time', + 'America/Marigot': 'SA Western Standard Time', + 'America/Martinique': 'SA Western Standard Time', + 'America/Matamoros': 'Central Standard Time', + 'America/Mazatlan': 'Mountain Standard Time (Mexico)', + 'America/Mendoza': 'Argentina Standard Time', + 'America/Menominee': 'Central Standard Time', + 'America/Merida': 'Central Standard Time (Mexico)', + 'America/Mexico_City': 'Central Standard Time (Mexico)', + 'America/Moncton': 'Atlantic Standard Time', + 'America/Monterrey': 'Central Standard Time (Mexico)', + 'America/Montevideo': 'Montevideo Standard Time', + 'America/Montreal': 'Eastern Standard Time', + 'America/Montserrat': 'SA Western Standard Time', + 'America/Nassau': 'Eastern Standard Time', + 'America/New_York': 'Eastern Standard Time', + 'America/Nipigon': 'Eastern Standard Time', + 'America/Nome': 'Alaskan Standard Time', + 'America/Noronha': 'UTC-02', + 'America/North_Dakota/Beulah': 'Central Standard Time', + 'America/North_Dakota/Center': 'Central Standard Time', + 'America/North_Dakota/New_Salem': 'Central Standard Time', + 'America/Ojinaga': 'Mountain Standard Time', + 'America/Panama': 'SA Pacific Standard Time', + 'America/Pangnirtung': 'Eastern Standard Time', + 'America/Paramaribo': 'SA Eastern Standard Time', + 'America/Phoenix': 'US Mountain Standard Time', + 'America/Port-au-Prince': 'Eastern Standard Time', + 'America/Port_of_Spain': 'SA Western Standard Time', + 'America/Porto_Velho': 'SA Western Standard Time', + 'America/Puerto_Rico': 'SA Western Standard Time', + 'America/Rainy_River': 'Central Standard Time', + 'America/Rankin_Inlet': 'Central Standard Time', + 'America/Recife': 'SA Eastern Standard Time', + 'America/Regina': 'Canada Central Standard Time', + 'America/Resolute': 'Central Standard Time', + 'America/Rio_Branco': 'SA Pacific Standard Time', + 'America/Santa_Isabel': 'Pacific Standard Time (Mexico)', + 'America/Santarem': 'SA Eastern Standard Time', + 'America/Santiago': 'Pacific SA Standard Time', + 'America/Santo_Domingo': 'SA Western Standard Time', + 'America/Sao_Paulo': 'E. South America Standard Time', + 'America/Scoresbysund': 'Azores Standard Time', + 'America/Sitka': 'Alaskan Standard Time', + 'America/St_Barthelemy': 'SA Western Standard Time', + 'America/St_Johns': 'Newfoundland Standard Time', + 'America/St_Kitts': 'SA Western Standard Time', + 'America/St_Lucia': 'SA Western Standard Time', + 'America/St_Thomas': 'SA Western Standard Time', + 'America/St_Vincent': 'SA Western Standard Time', + 'America/Swift_Current': 'Canada Central Standard Time', + 'America/Tegucigalpa': 'Central America Standard Time', + 'America/Thule': 'Atlantic Standard Time', + 'America/Thunder_Bay': 'Eastern Standard Time', + 'America/Tijuana': 'Pacific Standard Time', + 'America/Toronto': 'Eastern Standard Time', + 'America/Tortola': 'SA Western Standard Time', + 'America/Vancouver': 'Pacific Standard Time', + 'America/Whitehorse': 'Pacific Standard Time', + 'America/Winnipeg': 'Central Standard Time', + 'America/Yakutat': 'Alaskan Standard Time', + 'America/Yellowknife': 'Mountain Standard Time', + 'Antarctica/Casey': 'W. Australia Standard Time', + 'Antarctica/Davis': 'SE Asia Standard Time', + 'Antarctica/DumontDUrville': 'West Pacific Standard Time', + 'Antarctica/Macquarie': 'Central Pacific Standard Time', + 'Antarctica/Mawson': 'West Asia Standard Time', + 'Antarctica/McMurdo': 'New Zealand Standard Time', + 'Antarctica/Palmer': 'Pacific SA Standard Time', + 'Antarctica/Rothera': 'SA Eastern Standard Time', + 'Antarctica/Syowa': 'E. Africa Standard Time', + 'Antarctica/Vostok': 'Central Asia Standard Time', + 'Arctic/Longyearbyen': 'W. Europe Standard Time', + 'Asia/Aden': 'Arab Standard Time', + 'Asia/Almaty': 'Central Asia Standard Time', + 'Asia/Amman': 'Jordan Standard Time', + 'Asia/Anadyr': 'Russia Time Zone 11', + 'Asia/Aqtau': 'West Asia Standard Time', + 'Asia/Aqtobe': 'West Asia Standard Time', + 'Asia/Ashgabat': 'West Asia Standard Time', + 'Asia/Baghdad': 'Arabic Standard Time', + 'Asia/Bahrain': 'Arab Standard Time', + 'Asia/Baku': 'Azerbaijan Standard Time', + 'Asia/Bangkok': 'SE Asia Standard Time', + 'Asia/Beirut': 'Middle East Standard Time', + 'Asia/Bishkek': 'Central Asia Standard Time', + 'Asia/Brunei': 'Singapore Standard Time', + 'Asia/Calcutta': 'India Standard Time', + 'Asia/Chita': 'North Asia East Standard Time', + 'Asia/Choibalsan': 'Ulaanbaatar Standard Time', + 'Asia/Colombo': 'Sri Lanka Standard Time', + 'Asia/Damascus': 'Syria Standard Time', + 'Asia/Dhaka': 'Bangladesh Standard Time', + 'Asia/Dili': 'Tokyo Standard Time', + 'Asia/Dubai': 'Arabian Standard Time', + 'Asia/Dushanbe': 'West Asia Standard Time', + 'Asia/Hong_Kong': 'China Standard Time', + 'Asia/Hovd': 'SE Asia Standard Time', + 'Asia/Irkutsk': 'North Asia East Standard Time', + 'Asia/Jakarta': 'SE Asia Standard Time', + 'Asia/Jayapura': 'Tokyo Standard Time', + 'Asia/Jerusalem': 'Israel Standard Time', + 'Asia/Kabul': 'Afghanistan Standard Time', + 'Asia/Kamchatka': 'Russia Time Zone 11', + 'Asia/Karachi': 'Pakistan Standard Time', + 'Asia/Katmandu': 'Nepal Standard Time', + 'Asia/Khandyga': 'Yakutsk Standard Time', + 'Asia/Krasnoyarsk': 'North Asia Standard Time', + 'Asia/Kuala_Lumpur': 'Singapore Standard Time', + 'Asia/Kuching': 'Singapore Standard Time', + 'Asia/Kuwait': 'Arab Standard Time', + 'Asia/Macau': 'China Standard Time', + 'Asia/Magadan': 'Magadan Standard Time', + 'Asia/Makassar': 'Singapore Standard Time', + 'Asia/Manila': 'Singapore Standard Time', + 'Asia/Muscat': 'Arabian Standard Time', + 'Asia/Nicosia': 'GTB Standard Time', + 'Asia/Novokuznetsk': 'North Asia Standard Time', + 'Asia/Novosibirsk': 'N. Central Asia Standard Time', + 'Asia/Omsk': 'N. Central Asia Standard Time', + 'Asia/Oral': 'West Asia Standard Time', + 'Asia/Phnom_Penh': 'SE Asia Standard Time', + 'Asia/Pontianak': 'SE Asia Standard Time', + 'Asia/Pyongyang': 'Korea Standard Time', + 'Asia/Qatar': 'Arab Standard Time', + 'Asia/Qyzylorda': 'Central Asia Standard Time', + 'Asia/Rangoon': 'Myanmar Standard Time', + 'Asia/Riyadh': 'Arab Standard Time', + 'Asia/Saigon': 'SE Asia Standard Time', + 'Asia/Sakhalin': 'Vladivostok Standard Time', + 'Asia/Samarkand': 'West Asia Standard Time', + 'Asia/Seoul': 'Korea Standard Time', + 'Asia/Shanghai': 'China Standard Time', + 'Asia/Singapore': 'Singapore Standard Time', + 'Asia/Srednekolymsk': 'Russia Time Zone 10', + 'Asia/Taipei': 'Taipei Standard Time', + 'Asia/Tashkent': 'West Asia Standard Time', + 'Asia/Tbilisi': 'Georgian Standard Time', + 'Asia/Tehran': 'Iran Standard Time', + 'Asia/Thimphu': 'Bangladesh Standard Time', + 'Asia/Tokyo': 'Tokyo Standard Time', + 'Asia/Ulaanbaatar': 'Ulaanbaatar Standard Time', + 'Asia/Urumqi': 'Central Asia Standard Time', + 'Asia/Ust-Nera': 'Vladivostok Standard Time', + 'Asia/Vientiane': 'SE Asia Standard Time', + 'Asia/Vladivostok': 'Vladivostok Standard Time', + 'Asia/Yakutsk': 'Yakutsk Standard Time', + 'Asia/Yekaterinburg': 'Ekaterinburg Standard Time', + 'Asia/Yerevan': 'Caucasus Standard Time', + 'Atlantic/Azores': 'Azores Standard Time', + 'Atlantic/Bermuda': 'Atlantic Standard Time', + 'Atlantic/Canary': 'GMT Standard Time', + 'Atlantic/Cape_Verde': 'Cape Verde Standard Time', + 'Atlantic/Faeroe': 'GMT Standard Time', + 'Atlantic/Madeira': 'GMT Standard Time', + 'Atlantic/Reykjavik': 'Greenwich Standard Time', + 'Atlantic/South_Georgia': 'UTC-02', + 'Atlantic/St_Helena': 'Greenwich Standard Time', + 'Atlantic/Stanley': 'SA Eastern Standard Time', + 'Australia/Adelaide': 'Cen. Australia Standard Time', + 'Australia/Brisbane': 'E. Australia Standard Time', + 'Australia/Broken_Hill': 'Cen. Australia Standard Time', + 'Australia/Currie': 'Tasmania Standard Time', + 'Australia/Darwin': 'AUS Central Standard Time', + 'Australia/Hobart': 'Tasmania Standard Time', + 'Australia/Lindeman': 'E. Australia Standard Time', + 'Australia/Melbourne': 'AUS Eastern Standard Time', + 'Australia/Perth': 'W. Australia Standard Time', + 'Australia/Sydney': 'AUS Eastern Standard Time', + 'CST6CDT': 'Central Standard Time', + 'EST5EDT': 'Eastern Standard Time', + 'Etc/GMT': 'UTC', + 'Etc/GMT+1': 'Cape Verde Standard Time', + 'Etc/GMT+10': 'Hawaiian Standard Time', + 'Etc/GMT+11': 'UTC-11', + 'Etc/GMT+12': 'Dateline Standard Time', + 'Etc/GMT+2': 'UTC-02', + 'Etc/GMT+3': 'SA Eastern Standard Time', + 'Etc/GMT+4': 'SA Western Standard Time', + 'Etc/GMT+5': 'SA Pacific Standard Time', + 'Etc/GMT+6': 'Central America Standard Time', + 'Etc/GMT+7': 'US Mountain Standard Time', + 'Etc/GMT-1': 'W. Central Africa Standard Time', + 'Etc/GMT-10': 'West Pacific Standard Time', + 'Etc/GMT-11': 'Central Pacific Standard Time', + 'Etc/GMT-12': 'UTC+12', + 'Etc/GMT-13': 'Tonga Standard Time', + 'Etc/GMT-14': 'Line Islands Standard Time', + 'Etc/GMT-2': 'South Africa Standard Time', + 'Etc/GMT-3': 'E. Africa Standard Time', + 'Etc/GMT-4': 'Arabian Standard Time', + 'Etc/GMT-5': 'West Asia Standard Time', + 'Etc/GMT-6': 'Central Asia Standard Time', + 'Etc/GMT-7': 'SE Asia Standard Time', + 'Etc/GMT-8': 'Singapore Standard Time', + 'Etc/GMT-9': 'Tokyo Standard Time', + 'Etc/UTC': 'UTC', + 'Europe/Amsterdam': 'W. Europe Standard Time', + 'Europe/Andorra': 'W. Europe Standard Time', + 'Europe/Athens': 'GTB Standard Time', + 'Europe/Belgrade': 'Central Europe Standard Time', + 'Europe/Berlin': 'W. Europe Standard Time', + 'Europe/Bratislava': 'Central Europe Standard Time', + 'Europe/Brussels': 'Romance Standard Time', + 'Europe/Bucharest': 'GTB Standard Time', + 'Europe/Budapest': 'Central Europe Standard Time', + 'Europe/Busingen': 'W. Europe Standard Time', + 'Europe/Chisinau': 'GTB Standard Time', + 'Europe/Copenhagen': 'Romance Standard Time', + 'Europe/Dublin': 'GMT Standard Time', + 'Europe/Gibraltar': 'W. Europe Standard Time', + 'Europe/Guernsey': 'GMT Standard Time', + 'Europe/Helsinki': 'FLE Standard Time', + 'Europe/Isle_of_Man': 'GMT Standard Time', + 'Europe/Istanbul': 'Turkey Standard Time', + 'Europe/Jersey': 'GMT Standard Time', + 'Europe/Kaliningrad': 'Kaliningrad Standard Time', + 'Europe/Kiev': 'FLE Standard Time', + 'Europe/Lisbon': 'GMT Standard Time', + 'Europe/Ljubljana': 'Central Europe Standard Time', + 'Europe/London': 'GMT Standard Time', + 'Europe/Luxembourg': 'W. Europe Standard Time', + 'Europe/Madrid': 'Romance Standard Time', + 'Europe/Malta': 'W. Europe Standard Time', + 'Europe/Mariehamn': 'FLE Standard Time', + 'Europe/Minsk': 'Belarus Standard Time', + 'Europe/Monaco': 'W. Europe Standard Time', + 'Europe/Moscow': 'Russian Standard Time', + 'Europe/Oslo': 'W. Europe Standard Time', + 'Europe/Paris': 'Romance Standard Time', + 'Europe/Podgorica': 'Central Europe Standard Time', + 'Europe/Prague': 'Central Europe Standard Time', + 'Europe/Riga': 'FLE Standard Time', + 'Europe/Rome': 'W. Europe Standard Time', + 'Europe/Samara': 'Russia Time Zone 3', + 'Europe/San_Marino': 'W. Europe Standard Time', + 'Europe/Sarajevo': 'Central European Standard Time', + 'Europe/Simferopol': 'Russian Standard Time', + 'Europe/Skopje': 'Central European Standard Time', + 'Europe/Sofia': 'FLE Standard Time', + 'Europe/Stockholm': 'W. Europe Standard Time', + 'Europe/Tallinn': 'FLE Standard Time', + 'Europe/Tirane': 'Central Europe Standard Time', + 'Europe/Uzhgorod': 'FLE Standard Time', + 'Europe/Vaduz': 'W. Europe Standard Time', + 'Europe/Vatican': 'W. Europe Standard Time', + 'Europe/Vienna': 'W. Europe Standard Time', + 'Europe/Vilnius': 'FLE Standard Time', + 'Europe/Volgograd': 'Russian Standard Time', + 'Europe/Warsaw': 'Central European Standard Time', + 'Europe/Zagreb': 'Central European Standard Time', + 'Europe/Zaporozhye': 'FLE Standard Time', + 'Europe/Zurich': 'W. Europe Standard Time', + 'Indian/Antananarivo': 'E. Africa Standard Time', + 'Indian/Chagos': 'Central Asia Standard Time', + 'Indian/Christmas': 'SE Asia Standard Time', + 'Indian/Cocos': 'Myanmar Standard Time', + 'Indian/Comoro': 'E. Africa Standard Time', + 'Indian/Kerguelen': 'West Asia Standard Time', + 'Indian/Mahe': 'Mauritius Standard Time', + 'Indian/Maldives': 'West Asia Standard Time', + 'Indian/Mauritius': 'Mauritius Standard Time', + 'Indian/Mayotte': 'E. Africa Standard Time', + 'Indian/Reunion': 'Mauritius Standard Time', + 'MST7MDT': 'Mountain Standard Time', + 'PST8PDT': 'Pacific Standard Time', + 'Pacific/Apia': 'Samoa Standard Time', + 'Pacific/Auckland': 'New Zealand Standard Time', + 'Pacific/Efate': 'Central Pacific Standard Time', + 'Pacific/Enderbury': 'Tonga Standard Time', + 'Pacific/Fakaofo': 'Tonga Standard Time', + 'Pacific/Fiji': 'Fiji Standard Time', + 'Pacific/Funafuti': 'UTC+12', + 'Pacific/Galapagos': 'Central America Standard Time', + 'Pacific/Guadalcanal': 'Central Pacific Standard Time', + 'Pacific/Guam': 'West Pacific Standard Time', + 'Pacific/Honolulu': 'Hawaiian Standard Time', + 'Pacific/Johnston': 'Hawaiian Standard Time', + 'Pacific/Kiritimati': 'Line Islands Standard Time', + 'Pacific/Kosrae': 'Central Pacific Standard Time', + 'Pacific/Kwajalein': 'UTC+12', + 'Pacific/Majuro': 'UTC+12', + 'Pacific/Midway': 'UTC-11', + 'Pacific/Nauru': 'UTC+12', + 'Pacific/Niue': 'UTC-11', + 'Pacific/Noumea': 'Central Pacific Standard Time', + 'Pacific/Pago_Pago': 'UTC-11', + 'Pacific/Palau': 'Tokyo Standard Time', + 'Pacific/Ponape': 'Central Pacific Standard Time', + 'Pacific/Port_Moresby': 'West Pacific Standard Time', + 'Pacific/Rarotonga': 'Hawaiian Standard Time', + 'Pacific/Saipan': 'West Pacific Standard Time', + 'Pacific/Tahiti': 'Hawaiian Standard Time', + 'Pacific/Tarawa': 'UTC+12', + 'Pacific/Tongatapu': 'Tonga Standard Time', + 'Pacific/Truk': 'West Pacific Standard Time', + 'Pacific/Wake': 'UTC+12', + 'Pacific/Wallis': 'UTC+12'} diff --git a/pylintrc b/pylintrc new file mode 100644 index 00000000..3137b71b --- /dev/null +++ b/pylintrc @@ -0,0 +1,287 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +init-hook=sys.path.insert(0, 'lib/') + +# Profiled execution. +profile=no + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + + +[MESSAGES CONTROL] + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +#C0303 whitespace between the end of a line and the newline. +#C0325 a single item in parentheses follows an if, for, or other keyword +#C0326 wrong number of spaces is used around an operator, bracket or block opener +#I0011 an inline option disables a pylint message or a messages category +#R0801 a set of similar lines has been detected among multiple file +#W0142 a function or method is called using *args or **kwargs to dispatch argument + +# W1201(logging-not-lazy) +# C0330(bad-continuation) +# E1205(logging-too-many-args) + +disable=C0303,C0325,C0326,I0011,R0801,W0142,C0103,C0111,C0301,C0302,C0304,C0321,C1001,E0101,E0203,E0602,E1101,E1123,R0201,R0401,R0911,R0912,R0914,R0915,R0923,W0102,W0109,W0120,W0141,W0201,W0212,W0231,W0232,W0233,W0301,W0311,W0401,W0403,W0404,W0511,W0601,W0602,W0603,W0611,W0612,W0613,W0621,W0622,W0633,W0702,W0703,W1401,W1201,C0330 + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +#output-format=parseable + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=no + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Add a comment according to your evaluation note. This is used by the global +# evaluation report (RP0004). +comment=no + +# Template used to display messages. This is a python new-style format string +# used to format the massage information. See doc for all details +#msg-template= +msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg} + +[BASIC] + +# Required attributes for module, separated by a comma +required-attributes= + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,apply,input + +# Regular expression which should only match correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression which should only match correct module level names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression which should only match correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression which should only match correct function names +function-rgx=[a-z_][a-z0-9_]{2,50}$ + +# Regular expression which should only match correct method names +method-rgx=[a-z_][a-z0-9_]{2,50}$ + +# Regular expression which should only match correct instance attribute names +attr-rgx=[a-z_][a-z0-9_]{2,50}$ + +# Regular expression which should only match correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,50}$ + +# Regular expression which should only match correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,50}$ + +# Regular expression which should only match correct attribute names in class +# bodies +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression which should only match correct list comprehension / +# generator expression variable names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=__.*__ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=150 + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +ignored-classes=SQLObject + +# When zope mode is activated, add a predefined set of Zope acquired attributes +# to generated-members. +zope=no + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. Python regular +# expressions are accepted. +generated-members=REQUEST,acl_users,aq_parent,objects + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the beginning of the name of dummy variables +# (i.e. not used). +dummy-variables-rgx=_$|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + + +[CLASSES] + +# List of interface methods to ignore, separated by a comma. This is used for +# instance to not check methods defines in Zope's Interface base class. +ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=10 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=20 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=0 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=100 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception