PK KKH] mozdownload/parser.py# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """Module to parse directory listings on a remote FTP server.""" from __future__ import absolute_import, unicode_literals import re import urllib from HTMLParser import HTMLParser import requests class DirectoryParser(HTMLParser): """Class to parse directory listings.""" def __init__(self, url, session=None, authentication=None, timeout=None): """Create instance of a directory parser. :param url: url of the directory on the web server. :param session: a requests Session instance used to fetch the directory content. If None, a new session will be created. :param authentication: a tuple (username, password) to authenticate against the web server, or None for no authentication. Note that it will only be used if the given *session* is None. :param timeout: timeout in seconds used when fetching the directory content. """ if not session: session = requests.Session() session.auth = authentication self.session = session self.timeout = timeout self.active_url = None self.entries = [] HTMLParser.__init__(self) # Force the server to not send cached content headers = {'Cache-Control': 'max-age=0'} r = self.session.get(url, headers=headers, timeout=self.timeout) try: r.raise_for_status() self.feed(r.text) finally: r.close() def filter(self, filter): """Filter entries by calling function or applying regex.""" if hasattr(filter, '__call__'): return [entry for entry in self.entries if filter(entry)] else: pattern = re.compile(filter, re.IGNORECASE) return [entry for entry in self.entries if pattern.match(entry)] def handle_starttag(self, tag, attrs): """Callback for when a tag gets opened.""" if not tag == 'a': return for attr in attrs: if attr[0] == 'href': # Links look like: /pub/firefox/nightly/2015/ # We have to trim the fragment down to the last item. Also to ensure we # always get it, we remove a possible final slash first url = urllib.unquote(attr[1]) self.active_url = url.rstrip('/').split('/')[-1] return def handle_endtag(self, tag): """Callback for when a tag gets closed.""" if tag == 'a': self.active_url = None def handle_data(self, data): """Callback when the data of a tag has been collected.""" # Only process the data when we are in an active a tag and have an URL. if not self.active_url: return # The visible text can have a final slash so strip it off if data.strip('/') == self.active_url: self.entries.append(self.active_url) PKMKH{qmozdownload/factory.py# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """Factory to ease the usage of different scrapers.""" from __future__ import absolute_import, unicode_literals from mozdownload import scraper from mozdownload.errors import NotSupportedError # List of known download scrapers scraper_types = {'candidate': scraper.ReleaseCandidateScraper, 'daily': scraper.DailyScraper, 'direct': scraper.DirectScraper, 'release': scraper.ReleaseScraper, 'tinderbox': scraper.TinderboxScraper, 'try': scraper.TryScraper, } class FactoryScraper(scraper.Scraper): """Factory class to instanciate a scraper of a given type.""" def __init__(self, scraper_type, **kwargs): """Create an instance of the expected scraper. :param scraper_type: The type of scraper to use. Scraper: :param application: The name of the application to download. :param base_url: The base url to be used :param branch: Name of the branch. :param build_number: Number of the build (for candidate, daily, and tinderbox builds). :param date: Date of the build. :param debug_build: Download a debug build. :param destination: Directory or file name to download the file to. :param extension: File extension of the build (e.g. ".zip"). :param is_stub_installer: Stub installer (Only applicable to Windows builds). :param locale: Locale of the application. :param logger: Logger instance to use. :param password: Password for basic HTTP authentication. :param platform: Platform of the application :param retry_attempts: Number of times the download will be attempted in the event of a failure :param retry_delay: Amount of time (in seconds) to wait between retry attempts. :param revision: Revision of the build to download. :param timeout: Amount of time (in seconds) until a download times out. :param username: Username for basic HTTP authentication. :param version: Version of the application to be downloaded. Daily builds: :param build_id: ID of the build to download. Direct scraper: :param url: URL to download. """ # Check for valid arguments if scraper_type in ('candidate', 'release') and not kwargs.get('version'): raise ValueError('The version to download has to be specified.') if scraper_type in ('try') and not kwargs.get('revision'): raise ValueError('The revision of the build has to be specified.') if kwargs.get('application') == 'b2g' and scraper_type in ('candidate', 'release'): error_msg = '%s build is not yet supported for B2G' % scraper_type raise NotSupportedError(error_msg) if kwargs.get('application') == 'fennec' and scraper_type not in ('daily'): error_msg = '%s build is not yet supported for fennec' % scraper_type raise NotSupportedError(error_msg) # Instantiate scraper and download the build scraper_keywords = {'application': kwargs.get('application', 'firefox'), 'base_url': kwargs.get('base_url', scraper.BASE_URL), 'destination': kwargs.get('destination'), 'extension': kwargs.get('extension'), 'is_stub_installer': kwargs.get('is_stub_installer'), 'locale': kwargs.get('locale'), 'logger': kwargs.get('logger', None), 'password': kwargs.get('password'), 'platform': kwargs.get('platform'), 'retry_attempts': kwargs.get('retry_attempts', 0), 'retry_delay': kwargs.get('retry_delay', 10), 'timeout': kwargs.get('timeout'), 'username': kwargs.get('username'), } scraper_type_keywords = { 'direct': { 'url': kwargs.get('url'), }, 'release': { 'version': kwargs.get('version'), }, 'candidate': { 'build_number': kwargs.get('build_number'), 'version': kwargs.get('version'), }, 'daily': { 'branch': kwargs.get('branch', 'mozilla-central'), 'build_number': kwargs.get('build_number'), 'build_id': kwargs.get('build_id'), 'date': kwargs.get('date'), 'revision': kwargs.get('revision'), }, 'tinderbox': { 'branch': kwargs.get('branch', 'mozilla-central'), 'build_number': kwargs.get('build_number'), 'date': kwargs.get('date'), 'debug_build': kwargs.get('debug_build', False), 'revision': kwargs.get('revision'), }, 'try': { 'debug_build': kwargs.get('debug_build', False), 'revision': kwargs.get('revision'), }, } kwargs = scraper_keywords.copy() kwargs.update(scraper_type_keywords.get(scraper_type, {})) self.__class__ = scraper_types[scraper_type] scraper_types[scraper_type].__init__(self, **kwargs) PK KKH&&mozdownload/errors.py# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """Exception types in use by mozdownload.""" from __future__ import unicode_literals class NotSupportedError(Exception): """Exception for a build not being supported.""" def __init__(self, message): """Create an instance of an exception.""" Exception.__init__(self, message) class NotFoundError(Exception): """Exception for a resource not being found (e.g. no logs).""" def __init__(self, message, location): """Create an instance of an exception.""" self.location = location Exception.__init__(self, ': '.join([message, location])) class NotImplementedError(Exception): """Exception for a feature which is not implemented yet.""" def __init__(self, message): """Create an instance of an exception.""" Exception.__init__(self, message) class TimeoutError(Exception): """Exception for a download exceeding the allocated timeout.""" def __init__(self): """Create an instance of an exception.""" self.message = 'The download exceeded the allocated timeout' Exception.__init__(self, self.message) PK KKH2՟mozdownload/__init__.py# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import absolute_import, unicode_literals from mozdownload import cli from mozdownload.cli import __version__ from mozdownload.factory import FactoryScraper from mozdownload.scraper import ( Scraper, DailyScraper, DirectScraper, ReleaseScraper, ReleaseCandidateScraper, TinderboxScraper, TryScraper, ) __all__ = [ __version__, cli, FactoryScraper, Scraper, DailyScraper, DirectScraper, ReleaseScraper, ReleaseCandidateScraper, TinderboxScraper, TryScraper, ] PKLKH;ymozdownload/cli.py# !/usr/bin/env python # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """Tool to download different Gecko based applications (version {}).""" from __future__ import absolute_import, unicode_literals import argparse import logging import os import sys from mozdownload import factory, scraper __version__ = '1.20.1' def parse_arguments(argv): """Setup argument parser for command line arguments.""" parser = argparse.ArgumentParser(description=__doc__.format(__version__)) parser.add_argument('--application', '-a', dest='application', choices=scraper.APPLICATIONS, default='firefox', metavar='APPLICATION', help='The name of the application to download, default: "%(default)s"') parser.add_argument('--base_url', dest='base_url', default=scraper.BASE_URL, metavar='BASE_URL', help='The base url to be used, default: "%(default)s"') parser.add_argument('--build-number', dest='build_number', type=int, metavar='BUILD_NUMBER', help='Number of the build (for candidate, daily, and tinderbox builds)') parser.add_argument('--debug-build', dest='debug_build', action='store_true', help='Download a debug build (for tinderbox, and try builds)') parser.add_argument('--destination', '-d', dest='destination', default=os.getcwd(), metavar='DESTINATION', help='Directory or file name to download the ' 'file to, default: current working directory') parser.add_argument('--extension', dest='extension', metavar='EXTENSION', help='File extension of the build (e.g. "zip"), default: ' 'the standard build extension on the platform.') parser.add_argument('--locale', '-l', dest='locale', metavar='LOCALE', help='Locale of the application, default: "en-US" or "multi"') parser.add_argument('--log-level', action='store', dest='log_level', default=logging.INFO, metavar='LOG_LEVEL', help='Threshold for log output (default: INFO') parser.add_argument('--password', dest='password', metavar='PASSWORD', help='Password for basic HTTP authentication.') parser.add_argument('--platform', '-p', dest='platform', choices=scraper.PLATFORM_FRAGMENTS.keys(), metavar='PLATFORM', help='Platform of the application') parser.add_argument('--retry-attempts', dest='retry_attempts', default=0, type=int, metavar='RETRY_ATTEMPTS', help='Number of times the download will be attempted in ' 'the event of a failure, default: %(default)s') parser.add_argument('--retry-delay', dest='retry_delay', default=10., type=float, metavar='RETRY_DELAY', help='Amount of time (in seconds) to wait between retry ' 'attempts, default: %(default)s') parser.add_argument('--revision', dest='revision', help='Revision of the build (for daily, tinderbox, and try builds)') parser.add_argument('--stub', dest='is_stub_installer', action='store_true', help='Stub installer (Only applicable to Windows builds).') parser.add_argument('--timeout', dest='timeout', type=float, metavar='TIMEOUT', help='Amount of time (in seconds) until a download times out.') parser.add_argument('--type', '-t', dest='scraper_type', choices=factory.scraper_types.keys(), default='release', metavar='SCRAPER_TYPE', help='Type of build to download, default: "%(default)s"') parser.add_argument('--url', dest='url', metavar='URL', help='URL to download. Note: Reserved characters (such ' 'as &) must be escaped or put in quotes otherwise ' 'CLI output may be abnormal.') parser.add_argument('--username', dest='username', metavar='USERNAME', help='Username for basic HTTP authentication.') parser.add_argument('--version', '-v', dest='version', metavar='VERSION', help='Version of the application to be downloaded for release ' 'and candidate builds (special values: %s)' % ', '.join( scraper.RELEASE_AND_CANDIDATE_LATEST_VERSIONS.keys())) # Group for daily builds group = parser.add_argument_group('Daily builds', 'Extra options for daily builds.') group.add_argument('--branch', dest='branch', default='mozilla-central', metavar='BRANCH', help='Name of the branch, default: "%(default)s"') group.add_argument('--build-id', dest='build_id', metavar='BUILD_ID', help='ID of the build to download.') group.add_argument('--date', dest='date', metavar='DATE', help='Date of the build, default: latest build') return vars(parser.parse_args(argv)) def cli(argv=None): """CLI entry point for mozdownload.""" kwargs = parse_arguments(argv or sys.argv[1:]) log_level = kwargs.pop('log_level') logging.basicConfig(format='%(levelname)s | %(message)s', level=log_level) logger = logging.getLogger(__name__) # Configure logging levels for sub modules. Set to ERROR by default. sub_log_level = logging.ERROR if log_level == logging.getLevelName(logging.DEBUG): sub_log_level = logging.DEBUG logging.getLogger('redo').setLevel(sub_log_level) logging.getLogger('requests').setLevel(sub_log_level) logging.getLogger('thclient').setLevel(sub_log_level) try: scraper_type = kwargs.pop('scraper_type') # If a URL has been specified use the direct scraper if kwargs.get('url'): scraper_type = 'direct' build = factory.FactoryScraper(scraper_type, **kwargs) build.download() except KeyboardInterrupt: logger.error('Download interrupted by the user') if __name__ == '__main__': sys.exit(cli()) PK KKH$/Xmozdownload/utils.py# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """Module to store various helper functions used in mozdownload.""" from __future__ import absolute_import, unicode_literals import hashlib def urljoin(*fragments): """Concatenate multi part strings into urls.""" # Strip possible already existent final slashes of fragments except for the last one parts = [fragment.rstrip('/') for fragment in fragments[:len(fragments) - 1]] parts.append(fragments[-1]) return '/'.join(parts) def create_md5(path): """Create the md5 hash of a file using the hashlib library.""" m = hashlib.md5() # rb necessary to run correctly in windows. with open(path, "rb") as f: while True: data = f.read(8192) if not data: break m.update(data) return m.hexdigest() PKKH dOmozdownload/scraper.py# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """Scrapers for various kinds of build types.""" from __future__ import absolute_import, unicode_literals import logging import os import re import sys import urllib from datetime import datetime from urlparse import urlparse import mozinfo import progressbar as pb import redo import requests from mozdownload import errors from mozdownload.parser import DirectoryParser from mozdownload.timezones import PacificTimezone from mozdownload import treeherder from mozdownload.utils import urljoin APPLICATIONS = ('b2g', 'firefox', 'fennec', 'thunderbird') # Some applications contain all locales in a single build APPLICATIONS_MULTI_LOCALE = ('b2g', 'fennec') # Used if the application is named differently than the subfolder on the server APPLICATIONS_TO_FTP_DIRECTORY = {'fennec': 'mobile'} # Base URL for the path to all builds BASE_URL = 'https://archive.mozilla.org/pub/' # Chunk size when downloading a file CHUNK_SIZE = 16 * 1024 DEFAULT_FILE_EXTENSIONS = {'android-api-9': 'apk', 'android-api-11': 'apk', 'android-api-15': 'apk', 'android-x86': 'apk', 'linux': 'tar.bz2', 'linux64': 'tar.bz2', 'mac': 'dmg', 'mac64': 'dmg', 'win32': 'exe', 'win64': 'exe'} PLATFORM_FRAGMENTS = {'android-api-9': r'android-arm', 'android-api-11': r'android-arm', 'android-api-15': r'android-arm', 'android-x86': r'android-i386', 'linux': r'linux-i686', 'linux64': r'linux-x86_64', 'mac': r'mac', 'mac64': r'mac(64)?', 'win32': r'win32', 'win64': r'win64(-x86_64)?'} # Special versions for release and candidate builds RELEASE_AND_CANDIDATE_LATEST_VERSIONS = { 'latest': r'^\d+(\.\d+)+(-candidates)?$', 'latest-beta': r'^\d+(\.\d+)+b\d+(-candidates)?$', 'latest-esr': r'^\d+(\.\d+)+esr(-candidates)?$', } class Scraper(object): """Generic class to download a Gecko based application.""" def __init__(self, destination=None, platform=None, application='firefox', locale=None, extension=None, username=None, password=None, retry_attempts=0, retry_delay=10., is_stub_installer=False, timeout=None, logger=None, base_url=BASE_URL): """Create an instance of the generic scraper.""" # Private properties for caching self._filename = None self._binary = None self.logger = logger or logging.getLogger(self.__module__) self.destination = destination or os.getcwd() if not locale: if application in APPLICATIONS_MULTI_LOCALE: self.locale = 'multi' else: self.locale = 'en-US' else: self.locale = locale self.locale_build = self.locale not in ('en-US', 'multi') self.platform = platform or self.detect_platform() self.session = requests.Session() if (username, password) != (None, None): self.session.auth = (username, password) self.retry_attempts = retry_attempts self.retry_delay = retry_delay self.is_stub_installer = is_stub_installer self.timeout_download = timeout # this is the timeout used in requests.get. Unlike "auth", # it does not work if we attach it on the session, so we handle # it independently. self.timeout_network = 60. # build the base URL self.application = application self.base_url = '%s/' % urljoin( base_url, APPLICATIONS_TO_FTP_DIRECTORY.get(self.application, self.application) ) if extension: self.extension = extension else: if self.application in APPLICATIONS_MULTI_LOCALE and \ self.platform in ('win32', 'win64'): # builds for APPLICATIONS_MULTI_LOCALE only exist in zip self.extension = 'zip' else: self.extension = DEFAULT_FILE_EXTENSIONS[self.platform] self._retry_check_404(self.get_build_info) def _retry(self, func, **retry_kwargs): retry_kwargs.setdefault('jitter', 0) retry_kwargs.setdefault('sleeptime', self.retry_delay) retry_kwargs.setdefault('attempts', self.retry_attempts + 1) return redo.retry(func, **retry_kwargs) def _retry_check_404(self, func, err_message="Specified build has not been found", **retry_kwargs): retry_kwargs.setdefault('retry_exceptions', (errors.NotFoundError, requests.exceptions.RequestException)) try: self._retry(func, **retry_kwargs) except requests.exceptions.HTTPError as exc: if exc.response.status_code == 404: raise errors.NotFoundError(err_message, exc.response.url) else: raise def _create_directory_parser(self, url): return DirectoryParser(url, session=self.session, timeout=self.timeout_network) @property def binary(self): """Return the name of the build.""" def _get_binary(): # Retrieve all entries from the remote virtual folder parser = self._create_directory_parser(self.path) if not parser.entries: raise errors.NotFoundError('No entries found', self.path) # Download the first matched directory entry pattern = re.compile(self.binary_regex, re.IGNORECASE) for entry in parser.entries: try: self._binary = pattern.match(entry).group() break except: # No match, continue with next entry continue else: raise errors.NotFoundError("Binary not found in folder", self.path) self._retry_check_404(_get_binary) return self._binary @property def binary_regex(self): """Return the regex for the binary filename.""" raise errors.NotImplementedError(sys._getframe(0).f_code.co_name) @property def url(self): """Return the URL of the build.""" return urllib.quote(urljoin(self.path, self.binary), safe='%/:=&?~#+!$,;\'@()*[]|') @property def path(self): """Return the path to the build folder.""" return urljoin(self.base_url, self.path_regex) @property def path_regex(self): """Return the regex for the path to the build folder.""" raise errors.NotImplementedError(sys._getframe(0).f_code.co_name) @property def platform_regex(self): """Return the platform fragment of the URL.""" return PLATFORM_FRAGMENTS[self.platform] @property def filename(self): """Return the local filename of the build.""" if self._filename is None: if os.path.splitext(self.destination)[1]: # If the filename has been given make use of it target_file = self.destination else: # Otherwise create it from the build details target_file = os.path.join(self.destination, self.build_filename(self.binary)) self._filename = os.path.abspath(target_file) return self._filename def get_build_info(self): """Return additional build information in subclasses if necessary.""" pass def build_filename(self, binary): """Return the proposed filename with extension for the binary.""" raise errors.NotImplementedError(sys._getframe(0).f_code.co_name) def detect_platform(self): """Detect the current platform.""" # For Mac and Linux 32bit we do not need the bits appended if mozinfo.os == 'mac' or \ (mozinfo.os == 'linux' and mozinfo.bits == 32): return mozinfo.os else: return "%s%d" % (mozinfo.os, mozinfo.bits) def download(self): """Download the specified file.""" def total_seconds(td): # Keep backward compatibility with Python 2.6 which doesn't have # this method if hasattr(td, 'total_seconds'): return td.total_seconds() else: return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / 10 ** 6 # Don't re-download the file if os.path.isfile(os.path.abspath(self.filename)): self.logger.info("File has already been downloaded: %s" % (self.filename)) return self.filename directory = os.path.dirname(self.filename) if not os.path.isdir(directory): os.makedirs(directory) self.logger.info('Downloading from: %s' % self.url) self.logger.info('Saving as: %s' % self.filename) tmp_file = self.filename + ".part" def _download(): try: start_time = datetime.now() # Enable streaming mode so we can download content in chunks r = self.session.get(self.url, stream=True) r.raise_for_status() content_length = r.headers.get('Content-length') # ValueError: Value out of range if only total_size given if content_length: total_size = int(content_length.strip()) max_value = ((total_size / CHUNK_SIZE) + 1) * CHUNK_SIZE bytes_downloaded = 0 log_level = self.logger.getEffectiveLevel() if log_level <= logging.INFO and content_length: widgets = [pb.Percentage(), ' ', pb.Bar(), ' ', pb.ETA(), ' ', pb.FileTransferSpeed()] pbar = pb.ProgressBar(widgets=widgets, maxval=max_value).start() with open(tmp_file, 'wb') as f: for chunk in r.iter_content(CHUNK_SIZE): f.write(chunk) bytes_downloaded += CHUNK_SIZE if log_level <= logging.INFO and content_length: pbar.update(bytes_downloaded) t1 = total_seconds(datetime.now() - start_time) if self.timeout_download and \ t1 >= self.timeout_download: raise errors.TimeoutError if log_level <= logging.INFO and content_length: pbar.finish() except: if os.path.isfile(tmp_file): os.remove(tmp_file) raise self._retry(_download, retry_exceptions=(requests.exceptions.RequestException, errors.TimeoutError)) os.rename(tmp_file, self.filename) return self.filename def show_matching_builds(self, builds): """Output the matching builds.""" self.logger.info('Found %s build%s: %s' % ( len(builds), len(builds) > 1 and 's' or '', len(builds) > 10 and ' ... '.join([', '.join(builds[:5]), ', '.join(builds[-5:])]) or ', '.join(builds))) class DailyScraper(Scraper): """Class to download a daily build from the Mozilla server.""" def __init__(self, branch='mozilla-central', build_id=None, date=None, build_number=None, revision=None, *args, **kwargs): """Create an instance of the daily scraper.""" self.branch = branch self.build_id = build_id self.date = date self.build_number = build_number self.revision = revision Scraper.__init__(self, *args, **kwargs) def get_build_info(self): """Define additional build information.""" # Retrieve build by revision if self.revision: th = treeherder.Treeherder( APPLICATIONS_TO_FTP_DIRECTORY.get(self.application, self.application), self.branch, self.platform) builds = th.query_builds_by_revision( self.revision, job_type_name='L10n Nightly' if self.locale_build else 'Nightly') if not builds: raise errors.NotFoundError('No builds have been found for revision', self.revision) # Extract the build folders which are prefixed with the buildid self.builds = [build.rsplit('/', 2)[1] for build in builds] self.show_matching_builds(self.builds) # There is only a single build per revision and platform self.build_index = 0 self.logger.info('Selected build: %s' % self.builds[self.build_index]) # Retrieve the date from the build folder which is always 19 chars long self.date = datetime.strptime(self.builds[self.build_index][:19], '%Y-%m-%d-%H-%M-%S') return # Internally we access builds via index if self.build_number is not None: self.build_index = int(self.build_number) - 1 else: self.build_index = None if self.build_id: # A build id has been specified. Split up its components so the # date and time can be extracted: # '20111212042025' -> '2011-12-12 04:20:25' self.date = datetime.strptime(self.build_id, '%Y%m%d%H%M%S') elif self.date: # A date (without time) has been specified. Use its value and the # build index to find the requested build for that day. try: self.date = datetime.strptime(self.date, '%Y-%m-%d') except: raise ValueError('%s is not a valid date' % self.date) else: # If no querying option has been specified the latest available # build of the given branch has to be identified. We also have to # retrieve the date of the build via its build id. self.date = self.get_latest_build_date() self.builds, self.build_index = self.get_build_info_for_date( self.date, self.build_index) def get_latest_build_date(self): """Return date of latest available nightly build.""" if self.application not in ('fennec'): url = urljoin(self.base_url, 'nightly', 'latest-%s/' % self.branch) else: url = urljoin(self.base_url, 'nightly', 'latest-%s-%s/' % (self.branch, self.platform)) self.logger.info('Retrieving the build status file from %s' % url) parser = self._create_directory_parser(url) parser.entries = parser.filter(r'.*%s\.txt' % self.platform_regex) if not parser.entries: message = 'Status file for %s build cannot be found' % \ self.platform_regex raise errors.NotFoundError(message, url) # Read status file for the platform, retrieve build id, # and convert to a date headers = {'Cache-Control': 'max-age=0'} r = self.session.get(url + parser.entries[-1], headers=headers) try: r.raise_for_status() return datetime.strptime(r.text.split('\n')[0], '%Y%m%d%H%M%S') finally: r.close() def is_build_dir(self, folder_name): """Return whether or not the given dir contains a build.""" # Cannot move up to base scraper due to parser.entries call in # get_build_info_for_date (see below) url = '%s/' % urljoin(self.base_url, self.monthly_build_list_regex, folder_name) if self.application in APPLICATIONS_MULTI_LOCALE \ and self.locale != 'multi': url = '%s/' % urljoin(url, self.locale) parser = self._create_directory_parser(url) pattern = re.compile(self.binary_regex, re.IGNORECASE) for entry in parser.entries: try: pattern.match(entry).group() return True except: # No match, continue with next entry continue return False def get_build_info_for_date(self, date, build_index=None): """Return the build information for a given date.""" url = urljoin(self.base_url, self.monthly_build_list_regex) has_time = date and date.time() self.logger.info('Retrieving list of builds from %s' % url) parser = self._create_directory_parser(url) regex = r'%(DATE)s-(\d+-)+%(BRANCH)s%(L10N)s%(PLATFORM)s$' % { 'DATE': date.strftime('%Y-%m-%d'), 'BRANCH': self.branch, # ensure to select the correct subfolder for localized builds 'L10N': '(-l10n)?' if self.locale_build else '', 'PLATFORM': '' if self.application not in ( 'fennec') else '-' + self.platform } parser.entries = parser.filter(regex) parser.entries = parser.filter(self.is_build_dir) if has_time: # If a time is included in the date, use it to determine the # build's index regex = r'.*%s.*' % date.strftime('%H-%M-%S') parser.entries = parser.filter(regex) if not parser.entries: date_format = '%Y-%m-%d-%H-%M-%S' if has_time else '%Y-%m-%d' message = 'Folder for builds on %s has not been found' % \ self.date.strftime(date_format) raise errors.NotFoundError(message, url) # If no index has been given, set it to the last build of the day. self.show_matching_builds(parser.entries) # If no index has been given, set it to the last build of the day. if build_index is None: # Find the most recent non-empty entry. build_index = len(parser.entries) for build in reversed(parser.entries): build_index -= 1 if not build_index or self.is_build_dir(build): break self.logger.info('Selected build: %s' % parser.entries[build_index]) return (parser.entries, build_index) @property def binary_regex(self): """Return the regex for the binary.""" regex_base_name = r'^%(APP)s-.*\.%(LOCALE)s\.%(PLATFORM)s' regex_suffix = {'android-api-9': r'\.%(EXT)s$', 'android-api-11': r'\.%(EXT)s$', 'android-api-15': r'\.%(EXT)s$', 'android-x86': r'\.%(EXT)s$', 'linux': r'\.%(EXT)s$', 'linux64': r'\.%(EXT)s$', 'mac': r'\.%(EXT)s$', 'mac64': r'\.%(EXT)s$', 'win32': r'(\.installer%(STUB)s)?\.%(EXT)s$', 'win64': r'(\.installer%(STUB)s)?\.%(EXT)s$'} regex = regex_base_name + regex_suffix[self.platform] return regex % {'APP': self.application, 'LOCALE': self.locale, 'PLATFORM': self.platform_regex, 'EXT': self.extension, 'STUB': '-stub' if self.is_stub_installer else ''} def build_filename(self, binary): """Return the proposed filename with extension for the binary.""" try: # Get exact timestamp of the build to build the local file name folder = self.builds[self.build_index] timestamp = re.search('([\d\-]+)-\D.*', folder).group(1) except: # If it's not available use the build's date timestamp = self.date.strftime('%Y-%m-%d') return '%(TIMESTAMP)s-%(BRANCH)s-%(NAME)s' % { 'TIMESTAMP': timestamp, 'BRANCH': self.branch, 'NAME': binary} @property def monthly_build_list_regex(self): """Return the regex for the folder containing builds of a month.""" # Regex for possible builds for the given date return r'nightly/%(YEAR)s/%(MONTH)s/' % { 'YEAR': self.date.year, 'MONTH': str(self.date.month).zfill(2)} @property def path_regex(self): """Return the regex for the path to the build folder.""" try: path = '%s/' % urljoin(self.monthly_build_list_regex, self.builds[self.build_index]) if self.application in APPLICATIONS_MULTI_LOCALE \ and self.locale != 'multi': path = '%s/' % urljoin(path, self.locale) return path except: folder = urljoin(self.base_url, self.monthly_build_list_regex) raise errors.NotFoundError("Specified sub folder cannot be found", folder) class DirectScraper(Scraper): """Class to download a file from a specified URL.""" def __init__(self, url, *args, **kwargs): """Create an instance of the direct scraper.""" self._url = url Scraper.__init__(self, *args, **kwargs) @property def filename(self): """File name of the downloaded file.""" if os.path.splitext(self.destination)[1]: # If the filename has been given make use of it target_file = self.destination else: # Otherwise determine it from the url. parsed_url = urlparse(self.url) source_filename = (parsed_url.path.rpartition('/')[-1] or parsed_url.hostname) target_file = os.path.join(self.destination, source_filename) return os.path.abspath(target_file) @property def url(self): """Location of the file to be downloaded.""" return self._url class ReleaseScraper(Scraper): """Class to download a release build of a Gecko based application.""" def __init__(self, version, *args, **kwargs): """Create instance of a release scraper.""" self.version = version Scraper.__init__(self, *args, **kwargs) @property def binary_regex(self): """Return the regex for the binary.""" regex = {'linux': r'^%(APP)s-%(VERSION)s\.%(EXT)s$', 'linux64': r'^%(APP)s-%(VERSION)s\.%(EXT)s$', 'mac': r'^%(APP)s %(VERSION)s\.%(EXT)s$', 'mac64': r'^%(APP)s %(VERSION)s\.%(EXT)s$', 'win32': r'^%(APP)s Setup %(STUB)s%(VERSION)s\.%(EXT)s$', 'win64': r'^%(APP)s Setup %(STUB)s%(VERSION)s\.%(EXT)s$', } return regex[self.platform] % { 'APP': self.application, 'EXT': self.extension, 'STUB': 'Stub ' if self.is_stub_installer else '', 'VERSION': self.version, } @property def path_regex(self): """Return the regex for the path to the build folder.""" regex = r'releases/%(VERSION)s/%(PLATFORM)s/%(LOCALE)s/' return regex % {'LOCALE': self.locale, 'PLATFORM': self.platform_regex, 'VERSION': self.version} @property def platform_regex(self): """Return the platform fragment of the URL.""" if self.platform == 'win64': return self.platform return PLATFORM_FRAGMENTS[self.platform] def build_filename(self, binary): """Return the proposed filename with extension for the binary.""" template = '%(APP)s-%(VERSION)s.%(LOCALE)s.%(PLATFORM)s%(STUB)s' \ '.%(EXT)s' return template % {'APP': self.application, 'VERSION': self.version, 'LOCALE': self.locale, 'PLATFORM': self.platform, 'STUB': '-stub' if self.is_stub_installer else '', 'EXT': self.extension} def get_build_info(self): """Define additional build information.""" self.version = self.query_versions(self.version)[0] def query_versions(self, version=None): """Check specified version and resolve special values.""" if version not in RELEASE_AND_CANDIDATE_LATEST_VERSIONS: return [version] url = urljoin(self.base_url, 'releases/') parser = self._create_directory_parser(url) if version: versions = parser.filter(RELEASE_AND_CANDIDATE_LATEST_VERSIONS[version]) from distutils.version import LooseVersion versions.sort(key=LooseVersion) return [versions[-1]] else: return parser.entries class ReleaseCandidateScraper(ReleaseScraper): """Class to download a release candidate build of a Gecko based application.""" def __init__(self, build_number=None, *args, **kwargs): """Create an instance of a release candidate scraper.""" self.build_number = build_number ReleaseScraper.__init__(self, *args, **kwargs) def get_build_info(self): """Define additional build information.""" ReleaseScraper.get_build_info(self) # Internally we access builds via index url = urljoin(self.base_url, self.candidate_build_list_regex) self.logger.info('Retrieving list of candidate builds from %s' % url) parser = self._create_directory_parser(url) if not parser.entries: message = 'Folder for specific candidate builds at %s has not' \ 'been found' % url raise errors.NotFoundError(message, url) self.show_matching_builds(parser.entries) self.builds = parser.entries self.build_index = len(parser.entries) - 1 if self.build_number and \ ('build%s' % self.build_number) in self.builds: self.builds = ['build%s' % self.build_number] self.build_index = 0 self.logger.info('Selected build: build%s' % self.build_number) else: self.logger.info('Selected build: build%d' % (self.build_index + 1)) @property def candidate_build_list_regex(self): """Return the regex for the folder with the list of candidate builds.""" # Regex for possible builds for the given date return r'candidates/%(VERSION)s-candidates/' % { 'VERSION': self.version} @property def path_regex(self): """Return the regex for the path to the build folder.""" regex = r'%(PREFIX)s%(BUILD)s/%(PLATFORM)s/%(LOCALE)s/' return regex % {'PREFIX': self.candidate_build_list_regex, 'BUILD': self.builds[self.build_index], 'LOCALE': self.locale, 'PLATFORM': self.platform_regex} @property def platform_regex(self): """Return the platform fragment of the URL.""" if self.platform == 'win64': return self.platform return PLATFORM_FRAGMENTS[self.platform] def build_filename(self, binary): """Return the proposed filename with extension for the binary.""" template = '%(APP)s-%(VERSION)s-%(BUILD)s.%(LOCALE)s.' \ '%(PLATFORM)s%(STUB)s.%(EXT)s' return template % {'APP': self.application, 'VERSION': self.version, 'BUILD': self.builds[self.build_index], 'LOCALE': self.locale, 'PLATFORM': self.platform, 'STUB': '-stub' if self.is_stub_installer else '', 'EXT': self.extension} class TinderboxScraper(Scraper): """Class to download a tinderbox build of a Gecko based application.""" def __init__(self, branch='mozilla-central', build_number=None, date=None, debug_build=False, revision=None, *args, **kwargs): """Create instance of a tinderbox scraper.""" self.branch = branch self.build_number = build_number self.debug_build = debug_build self.date = date self.revision = revision self.timestamp = None # Currently any time in RelEng is based on the Pacific time zone. self.timezone = PacificTimezone() Scraper.__init__(self, *args, **kwargs) def get_build_info(self): """Define additional build information.""" # Retrieve build by revision if self.revision: th = treeherder.Treeherder( APPLICATIONS_TO_FTP_DIRECTORY.get(self.application, self.application), self.branch, self.platform) builds = th.query_builds_by_revision( self.revision, job_type_name='Build', debug_build=self.debug_build) if not builds: raise errors.NotFoundError('No builds have been found for revision', self.revision) # Extract timestamp from each build folder self.builds = [build.rsplit('/', 2)[1] for build in builds] self.show_matching_builds(self.builds) # There is only a single build self.build_index = 0 self.logger.info('Selected build: %s' % self.builds[self.build_index]) return # Internally we access builds via index if self.build_number is not None: self.build_index = int(self.build_number) - 1 else: self.build_index = None if self.date is not None: try: # date is provided in the format 2013-07-23 self.date = datetime.strptime(self.date, '%Y-%m-%d') except: try: # date is provided as a unix timestamp datetime.fromtimestamp(float(self.date)) self.timestamp = self.date except: raise ValueError('%s is not a valid date' % self.date) # For localized builds we do not have to retrieve the list of builds # because only the last build is available if not self.locale_build: self.builds, self.build_index = self.get_build_info_for_index( self.build_index) @property def binary_regex(self): """Return the regex for the binary.""" regex_base_name = r'^%(APP)s-.*\.%(LOCALE)s\.%(PLATFORM)s' regex_suffix = {'linux': r'.*\.%(EXT)s$', 'linux64': r'.*\.%(EXT)s$', 'mac': r'.*\.%(EXT)s$', 'mac64': r'.*\.%(EXT)s$', 'win32': r'(\.installer%(STUB)s)?\.%(EXT)s$', 'win64': r'(\.installer%(STUB)s)?\.%(EXT)s$'} regex = regex_base_name + regex_suffix[self.platform] return regex % {'APP': self.application, 'LOCALE': self.locale, 'PLATFORM': PLATFORM_FRAGMENTS[self.platform], 'STUB': '-stub' if self.is_stub_installer else '', 'EXT': self.extension} def build_filename(self, binary): """Return the proposed filename with extension for the binary.""" return '%(TIMESTAMP)s%(BRANCH)s%(DEBUG)s-%(NAME)s' % { 'TIMESTAMP': self.timestamp + '-' if self.timestamp else '', 'BRANCH': self.branch, 'DEBUG': '-debug' if self.debug_build else '', 'NAME': binary} @property def build_list_regex(self): """Return the regex for the folder which contains the list of builds.""" regex = 'tinderbox-builds/%(BRANCH)s-%(PLATFORM)s%(L10N)s%(DEBUG)s/' return regex % { 'BRANCH': self.branch, 'PLATFORM': '' if self.locale_build else self.platform_regex, 'L10N': 'l10n' if self.locale_build else '', 'DEBUG': '-debug' if self.debug_build else ''} def date_matches(self, timestamp): """Determine whether the timestamp date is equal to the argument date.""" if self.date is None: return False timestamp = datetime.fromtimestamp(float(timestamp), self.timezone) if self.date.date() == timestamp.date(): return True return False def detect_platform(self): """Detect the current platform.""" platform = Scraper.detect_platform(self) # On OS X we have to special case the platform detection code and # fallback to 64 bit builds for the en-US locale if mozinfo.os == 'mac' and self.locale == 'en-US' and \ mozinfo.bits == 64: platform = "%s%d" % (mozinfo.os, mozinfo.bits) return platform def is_build_dir(self, folder_name): """Return whether or not the given dir contains a build.""" # Cannot move up to base scraper due to parser.entries call in # get_build_info_for_index (see below) url = '%s/' % urljoin(self.base_url, self.build_list_regex, folder_name) if self.application in APPLICATIONS_MULTI_LOCALE \ and self.locale != 'multi': url = '%s/' % urljoin(url, self.locale) parser = self._create_directory_parser(url) pattern = re.compile(self.binary_regex, re.IGNORECASE) for entry in parser.entries: try: pattern.match(entry).group() return True except: # No match, continue with next entry continue return False def get_build_info_for_index(self, build_index=None): """Get additional information for the build at the given index.""" url = urljoin(self.base_url, self.build_list_regex) self.logger.info('Retrieving list of builds from %s' % url) parser = self._create_directory_parser(url) parser.entries = parser.filter(r'^\d+$') if self.timestamp: # If a timestamp is given, retrieve the folder with the timestamp # as name parser.entries = self.timestamp in parser.entries and \ [self.timestamp] elif self.date: # If date is given, retrieve the subset of builds on that date parser.entries = filter(self.date_matches, parser.entries) if not parser.entries: message = 'No builds have been found' raise errors.NotFoundError(message, url) self.show_matching_builds(parser.entries) # If no index has been given, set it to the last build of the day. if build_index is None: # Find the most recent non-empty entry. build_index = len(parser.entries) for build in reversed(parser.entries): build_index -= 1 if not build_index or self.is_build_dir(build): break self.logger.info('Selected build: %s' % parser.entries[build_index]) return (parser.entries, build_index) @property def path_regex(self): """Return the regex for the path to the build folder.""" if self.locale_build: return self.build_list_regex return '%s/' % urljoin(self.build_list_regex, self.builds[self.build_index]) @property def platform_regex(self): """Return the platform fragment of the URL.""" platform_fragments = {'linux': 'linux', 'linux64': 'linux64', 'mac': 'macosx64', 'mac64': 'macosx64', 'win32': 'win32', 'win64': 'win64'} return platform_fragments[self.platform] class TryScraper(Scraper): """Class to download a try build of a Gecko based application.""" def __init__(self, revision=None, debug_build=False, *args, **kwargs): """Create an instance of a try scraper.""" self.debug_build = debug_build self.revision = revision Scraper.__init__(self, *args, **kwargs) def get_build_info(self): """Define additional build information.""" # Retrieve build by revision th = treeherder.Treeherder( APPLICATIONS_TO_FTP_DIRECTORY.get(self.application, self.application), 'try', self.platform) builds = th.query_builds_by_revision( self.revision, job_type_name='Build', debug_build=self.debug_build) if not builds: raise errors.NotFoundError('No builds have been found for revision', self.revision) # Extract username and revision from build folders self.builds = [build.rsplit('/', 3)[1] for build in builds] self.show_matching_builds(self.builds) # There is only a single build per revision and platform self.build_index = 0 self.logger.info('Selected build: %s' % self.builds[self.build_index]) @property def binary_regex(self): """Return the regex for the binary.""" regex_base_name = r'^%(APP)s-.*\.%(LOCALE)s\.%(PLATFORM)s' regex_suffix = {'linux': r'.*\.%(EXT)s$', 'linux64': r'.*\.%(EXT)s$', 'mac': r'.*\.%(EXT)s$', 'mac64': r'.*\.%(EXT)s$', 'win32': r'.*(\.installer%(STUB)s)\.%(EXT)s$', 'win64': r'.*(\.installer%(STUB)s)\.%(EXT)s$'} regex = regex_base_name + regex_suffix[self.platform] return regex % {'APP': self.application, 'LOCALE': self.locale, 'PLATFORM': PLATFORM_FRAGMENTS[self.platform], 'STUB': '-stub' if self.is_stub_installer else '', 'EXT': self.extension} def build_filename(self, binary): """Return the proposed filename with extension for the binary.""" return '%(REVISION)s%(DEBUG)s-%(NAME)s' % { 'REVISION': self.revision, 'DEBUG': '-debug' if self.debug_build else '', 'NAME': binary} @property def build_list_regex(self): """Return the regex for the folder which contains the list of builds.""" return 'try-builds/' def detect_platform(self): """Detect the current platform.""" platform = Scraper.detect_platform(self) # On OS X we have to special case the platform detection code and # fallback to 64 bit builds for the en-US locale if mozinfo.os == 'mac' and self.locale == 'en-US' and \ mozinfo.bits == 64: platform = "%s%d" % (mozinfo.os, mozinfo.bits) return platform @property def path_regex(self): """Return the regex for the path to the build folder.""" build_dir = 'try-%(PLATFORM)s%(DEBUG)s/' % { 'PLATFORM': self.platform_regex, 'DEBUG': '-debug' if self.debug_build else ''} return urljoin(self.build_list_regex, self.builds[self.build_index], build_dir) @property def platform_regex(self): """Return the platform fragment of the URL.""" platform_fragments = {'linux': 'linux', 'linux64': 'linux64', 'mac': 'macosx64', 'mac64': 'macosx64', 'win32': 'win32', 'win64': 'win64'} return platform_fragments[self.platform] PKMKHlmozdownload/treeherder.py# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """Module to retrieve builds via revision.""" from __future__ import absolute_import, unicode_literals import logging from thclient import TreeherderClient PLATFORM_MAP = { 'android-api-9': {'build_platform': 'android-2-3-armv7-api9'}, 'android-api-11': {'build_platform': 'android-4-0-armv7-api11'}, 'android-api-15': {'build_platform': 'android-4-0-armv7-api15'}, 'android-x86': {'build_platform': 'android-4-2-x86'}, 'linux': {'build_platform': 'linux32'}, 'linux64': {'build_platform': 'linux64'}, 'mac': {'build_os': 'mac', 'build_architecture': 'x86_64'}, 'mac64': {'build_os': 'mac', 'build_architecture': 'x86_64'}, 'win32': {'build_os': 'win', 'build_architecture': 'x86'}, 'win64': {'build_os': 'win', 'build_architecture': 'x86_64'}, } TREEHERDER_HOST = 'treeherder.mozilla.org' class Treeherder(object): """Wrapper class for TreeherderClient to ease the use of its API.""" def __init__(self, application, branch, platform, host=TREEHERDER_HOST, protocol='https'): """Create a new instance of the Treeherder class. :param application: The name of the application to download. :param branch: Name of the branch. :param platform: Platform of the application. :param host: The Treeherder host to make use of. :param protocol: The protocol for the Treeherder host. """ self.logger = logging.getLogger(__name__) self.client = TreeherderClient(host=host, protocol=protocol) self.application = application self.branch = branch self.platform = self.get_treeherder_platform(platform) def get_treeherder_platform(self, platform): """Return the internal Treeherder platform identifier. :param platform: Platform of the application. """ return PLATFORM_MAP.get(platform, platform) def query_builds_by_revision(self, revision, job_type_name='Build', debug_build=False): """Retrieve build folders for a given revision with the help of Treeherder. :param revision: Revision of the build to download. :param job_type_name: Name of the job to look for. For builds it should be 'Build', 'Nightly', and 'L10n Nightly'. Defaults to `Build`. :param debug_build: Download a debug build. """ builds = set() try: self.logger.info('Querying {host} for list of builds for revision: {revision}'.format( host=self.client.host, revision=revision)) # Retrieve the option hash to filter for type of build (opt, and debug for now) option_hash = None for key, values in self.client.get_option_collection_hash().iteritems(): for value in values: if value['name'] == ('debug' if debug_build else 'opt'): option_hash = key break if option_hash: break resultsets = self.client.get_resultsets(self.branch, revision=revision) # Set filters to speed-up querying jobs kwargs = { 'platform': self.platform, 'option_collection_hash': option_hash, 'job_type_name': job_type_name, 'exclusion_profile': False, } for resultset in resultsets: kwargs.update({'result_set_id': resultset['id']}) jobs = self.client.get_jobs(self.branch, **kwargs) for job in jobs: log_urls = self.client.get_job_log_url(self.branch, job_id=job['id']) for log_url in log_urls: if self.application in log_url['url']: self.logger.debug('Found build folder: {}'.format(log_url['url'])) builds.update([log_url['url']]) except Exception: self.logger.exception('Failure occurred when querying Treeherder for builds') return list(builds) PK KKHܩmozdownload/timezones.py# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. """Module for providing specific timezones.""" from __future__ import absolute_import, unicode_literals from datetime import datetime, timedelta, tzinfo class PacificTimezone(tzinfo): """Class to set the timezone to PST/PDT, and adjusts for daylight saving.""" def utcoffset(self, dt): """Offset from PST/PDT to UTC.""" return timedelta(hours=-8) + self.dst(dt) def tzname(self, dt): """Name of the timezone.""" return "Pacific" def dst(self, dt): """Calculate delta for daylight saving.""" # Daylight saving starts on the second Sunday of March at 2AM standard dst_start_date = self.first_sunday(dt.year, 3) + timedelta(days=7) \ + timedelta(hours=2) # Daylight saving ends on the first Sunday of November at 2AM standard dst_end_date = self.first_sunday(dt.year, 11) + timedelta(hours=2) if dst_start_date <= dt.replace(tzinfo=None) < dst_end_date: return timedelta(hours=1) else: return timedelta(0) def first_sunday(self, year, month): """Get the first sunday of a month.""" date = datetime(year, month, 1, 0) days_until_sunday = 6 - date.weekday() return date + timedelta(days=days_until_sunday) PKKH;y,mozdownload-1.20.1.dist-info/DESCRIPTION.rst[![PyPI version](https://badge.fury.io/py/mozdownload.svg)](http://badge.fury.io/py/mozdownload) [![Build Status](https://travis-ci.org/mozilla/mozdownload.svg?branch=master)](https://travis-ci.org/mozilla/mozdownload) [![Stories in Ready](https://badge.waffle.io/mozilla/mozdownload.png?label=ready&title=Ready)](https://waffle.io/mozilla/mozdownload) # mozdownload [mozdownload](https://github.com/mozilla/mozdownload) is a [python package](http://pypi.python.org/pypi/mozdownload) which handles downloading of Mozilla applications. ## Installation If the tool should only be used for downloading applications we propose to install it via pip. The following command will install the latest release: pip install mozdownload Otherwise follow the steps below to setup a development environment. It is recommended that [virtualenv](http://virtualenv.readthedocs.org/en/latest/installation.html) and [virtualenvwrapper](http://virtualenvwrapper.readthedocs.org/en/latest/) be used in conjunction with mozdownload. Start by installing these. Then first fork our repository into your own github account, and run: git clone https://github.com/%your_account%/mozdownload.git cd mozdownload python setup.py develop More detailed developer documentation can be found in the [wiki](https://github.com/mozilla/mozdownload/wiki). ## Command Line Usage The `mozdownload` command will download the application based on the provided command line options. ### Examples Download the latest official Firefox release for your platform: mozdownload --version=latest Download the latest official Firefox beta release for your platform: mozdownload --version=latest-beta Download the latest official Firefox esr release for your platform: mozdownload --version=latest-esr Download the latest Firefox release candidate for your platform: mozdownload --type candidate --version=latest Download the latest Firefox Aurora build for Windows (32bit): mozdownload --type=daily --branch=mozilla-aurora --platform=win32 Download the latest official Thunderbird release for your platform: mozdownload --application=thunderbird --version=latest Download the latest Earlybird build for Linux (64bit): mozdownload --application=thunderbird --type=daily --branch=comm-aurora --platform=linux64 Download this README file: mozdownload --url=https://raw.github.com/mozilla/mozdownload/master/README.md Download a file from a URL protected with basic authentication: mozdownload --url=http://example.com/secrets.txt --username=admin --password=password Run `mozdownload --help` for detailed information on the command line options. ### Command Line Options To see the full list of command line options, execute the command below and check the list of options for the build type to download: mozdownload --help ## API Beside the CLI mozdownload also offers an API to be used. To create specific instances of scrapers the FactoryScraper class can be used. Here some examples: # Create a release scraper for the German locale of Firefox 40.0.3 from mozdownload import FactoryScraper scraper = mozdownload.FactoryScraper('release', version='40.0.3', locale='de') # Create a candidate scraper for Windows 32bit of Firefox 41.0b9 from mozdownload import FactoryScraper scraper = mozdownload.FactoryScraper('candidate', version='41.0b9', platform='win32') # Create a daily scraper for the latest Dev Edition build on the current platform from mozdownload import FactoryScraper scraper = mozdownload.FactoryScraper('daily', branch='mozilla-aurora') All those scraper instances allow you to retrieve the url which is used to download the files, and the filename for the local destination: from mozdownload import FactoryScraper scraper = mozdownload.FactoryScraper('daily') print scraper.url print scraper.filename To actually download the remote file the download() method has to be called: from mozdownload import FactoryScraper scraper = mozdownload.FactoryScraper('daily') filename = scraper.download() PKKHU3yee-mozdownload-1.20.1.dist-info/entry_points.txt # -*- Entry points: -*- [console_scripts] mozdownload = mozdownload.cli:cli PKKH |bb*mozdownload-1.20.1.dist-info/metadata.json{"extensions": {"python.commands": {"wrap_console": {"mozdownload": "mozdownload.cli:cli"}}, "python.details": {"contacts": [{"email": "tools@lists.mozilla.com", "name": "Mozilla Automation and Testing Team", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}, "project_urls": {"Home": "http://github.com/mozilla/mozdownload"}}, "python.exports": {"console_scripts": {"mozdownload": "mozdownload.cli:cli"}}}, "extras": [], "generator": "bdist_wheel (0.26.0)", "keywords": ["mozilla"], "license": "Mozilla Public License 2.0 (MPL 2.0)", "metadata_version": "2.0", "name": "mozdownload", "run_requires": [{"requires": ["mozinfo (>=0.9)", "progressbar (==2.2)", "redo (==1.5)", "requests (==2.9.1)", "treeherder-client (==2.0.1)"]}], "summary": "Script to download builds for Firefox and Thunderbird from the Mozilla server.", "version": "1.20.1"}PKKHk *mozdownload-1.20.1.dist-info/top_level.txtmozdownload PKKH''\\"mozdownload-1.20.1.dist-info/WHEELWheel-Version: 1.0 Generator: bdist_wheel (0.26.0) Root-Is-Purelib: true Tag: py2-none-any PKKHC++%mozdownload-1.20.1.dist-info/METADATAMetadata-Version: 2.0 Name: mozdownload Version: 1.20.1 Summary: Script to download builds for Firefox and Thunderbird from the Mozilla server. Home-page: http://github.com/mozilla/mozdownload Author: Mozilla Automation and Testing Team Author-email: tools@lists.mozilla.com License: Mozilla Public License 2.0 (MPL 2.0) Keywords: mozilla Platform: UNKNOWN Requires-Dist: mozinfo (>=0.9) Requires-Dist: progressbar (==2.2) Requires-Dist: redo (==1.5) Requires-Dist: requests (==2.9.1) Requires-Dist: treeherder-client (==2.0.1) [![PyPI version](https://badge.fury.io/py/mozdownload.svg)](http://badge.fury.io/py/mozdownload) [![Build Status](https://travis-ci.org/mozilla/mozdownload.svg?branch=master)](https://travis-ci.org/mozilla/mozdownload) [![Stories in Ready](https://badge.waffle.io/mozilla/mozdownload.png?label=ready&title=Ready)](https://waffle.io/mozilla/mozdownload) # mozdownload [mozdownload](https://github.com/mozilla/mozdownload) is a [python package](http://pypi.python.org/pypi/mozdownload) which handles downloading of Mozilla applications. ## Installation If the tool should only be used for downloading applications we propose to install it via pip. The following command will install the latest release: pip install mozdownload Otherwise follow the steps below to setup a development environment. It is recommended that [virtualenv](http://virtualenv.readthedocs.org/en/latest/installation.html) and [virtualenvwrapper](http://virtualenvwrapper.readthedocs.org/en/latest/) be used in conjunction with mozdownload. Start by installing these. Then first fork our repository into your own github account, and run: git clone https://github.com/%your_account%/mozdownload.git cd mozdownload python setup.py develop More detailed developer documentation can be found in the [wiki](https://github.com/mozilla/mozdownload/wiki). ## Command Line Usage The `mozdownload` command will download the application based on the provided command line options. ### Examples Download the latest official Firefox release for your platform: mozdownload --version=latest Download the latest official Firefox beta release for your platform: mozdownload --version=latest-beta Download the latest official Firefox esr release for your platform: mozdownload --version=latest-esr Download the latest Firefox release candidate for your platform: mozdownload --type candidate --version=latest Download the latest Firefox Aurora build for Windows (32bit): mozdownload --type=daily --branch=mozilla-aurora --platform=win32 Download the latest official Thunderbird release for your platform: mozdownload --application=thunderbird --version=latest Download the latest Earlybird build for Linux (64bit): mozdownload --application=thunderbird --type=daily --branch=comm-aurora --platform=linux64 Download this README file: mozdownload --url=https://raw.github.com/mozilla/mozdownload/master/README.md Download a file from a URL protected with basic authentication: mozdownload --url=http://example.com/secrets.txt --username=admin --password=password Run `mozdownload --help` for detailed information on the command line options. ### Command Line Options To see the full list of command line options, execute the command below and check the list of options for the build type to download: mozdownload --help ## API Beside the CLI mozdownload also offers an API to be used. To create specific instances of scrapers the FactoryScraper class can be used. Here some examples: # Create a release scraper for the German locale of Firefox 40.0.3 from mozdownload import FactoryScraper scraper = mozdownload.FactoryScraper('release', version='40.0.3', locale='de') # Create a candidate scraper for Windows 32bit of Firefox 41.0b9 from mozdownload import FactoryScraper scraper = mozdownload.FactoryScraper('candidate', version='41.0b9', platform='win32') # Create a daily scraper for the latest Dev Edition build on the current platform from mozdownload import FactoryScraper scraper = mozdownload.FactoryScraper('daily', branch='mozilla-aurora') All those scraper instances allow you to retrieve the url which is used to download the files, and the filename for the local destination: from mozdownload import FactoryScraper scraper = mozdownload.FactoryScraper('daily') print scraper.url print scraper.filename To actually download the remote file the download() method has to be called: from mozdownload import FactoryScraper scraper = mozdownload.FactoryScraper('daily') filename = scraper.download() PKKH"o>>#mozdownload-1.20.1.dist-info/RECORDmozdownload/__init__.py,sha256=1VfM3odMTEAlORCozAaScRrf0kZ4UWeSOeN6X0hIUFE,744 mozdownload/cli.py,sha256=lqD4dUjzn8cak_gfYfSVHO_AricbBX7MgcamoWivhts,7671 mozdownload/errors.py,sha256=7nVzuGZdb0gDw5YckVPdw4cruMbrmVv98AM3MqPb6_U,1318 mozdownload/factory.py,sha256=-Wxj2KpLbBiDC1s0busUl79KvLbkzJVBedOVp0cOJOA,5635 mozdownload/parser.py,sha256=5YFOCD7lapzoFBU4ExlT-PB5ar7pjMYxfLkbV-Y7F1M,3241 mozdownload/scraper.py,sha256=4ay2zRMfD2QjKRtl3wYclMgaNT7Q5WPMzB0_I0VgBeA,40383 mozdownload/timezones.py,sha256=fnIU5JH50nUkpjvaCPK3QNuQdo5fTd8JxUPUpq_6Zgo,1530 mozdownload/treeherder.py,sha256=v0qrtbiNBrqiupTHkd5gk3CRb3QAfBn432rf5v-xORk,4281 mozdownload/utils.py,sha256=fVwgIL6vQsXW-JyrI9PUb8GCkk9JcsI_wq7G1E1JnNo,997 mozdownload-1.20.1.dist-info/DESCRIPTION.rst,sha256=lqILDswqxodwN0yb9rjbGXtXyKccavJOv6JN-FG9sCU,4122 mozdownload-1.20.1.dist-info/METADATA,sha256=Q46seFfEOz_tCHiQxWYHsF_0hCAa19cX-dMMGlqGPs4,4651 mozdownload-1.20.1.dist-info/RECORD,, mozdownload-1.20.1.dist-info/WHEEL,sha256=JTb7YztR8fkPg6aSjc571Q4eiVHCwmUDlX8PhuuqIIE,92 mozdownload-1.20.1.dist-info/entry_points.txt,sha256=8YyzRrAqZuTPuOjzz4N-stS0HXg7KylvRsIosxHL_js,101 mozdownload-1.20.1.dist-info/metadata.json,sha256=NUjPncaUoVzX8FWnoQ86OCehmAKxF9z2XlohVyWJJDA,866 mozdownload-1.20.1.dist-info/top_level.txt,sha256=u5cYVPkGd0Yt_-HCT2nO8q4dkP5SDawH7bJ7OP3tcD8,12 PK KKH] mozdownload/parser.pyPKMKH{q mozdownload/factory.pyPK KKH&&#mozdownload/errors.pyPK KKH2՟l(mozdownload/__init__.pyPKLKH;y+mozdownload/cli.pyPK KKH$/XImozdownload/utils.pyPKKH dOMmozdownload/scraper.pyPKMKHlmozdownload/treeherder.pyPK KKHܩmozdownload/timezones.pyPKKH;y,mozdownload-1.20.1.dist-info/DESCRIPTION.rstPKKHU3yee->mozdownload-1.20.1.dist-info/entry_points.txtPKKH |bb*mozdownload-1.20.1.dist-info/metadata.jsonPKKHk *mozdownload-1.20.1.dist-info/top_level.txtPKKH''\\"mozdownload-1.20.1.dist-info/WHEELPKKHC++%mozdownload-1.20.1.dist-info/METADATAPKKH"o>>#*mozdownload-1.20.1.dist-info/RECORDPKu0