PKFH4W dcos/__init__.py# Version is set for releases by our build system. # Be extremely careful when modifying. version = '0.3.1' """DCOS version""" PKցFH~YVVdcos/marathon.pyimport json from distutils.version import LooseVersion from dcos import http, util from dcos.errors import DCOSException, DCOSHTTPException from six.moves import urllib logger = util.get_logger(__name__) def create_client(config=None): """Creates a Marathon client with the supplied configuration. :param config: configuration dictionary :type config: config.Toml :returns: Marathon client :rtype: dcos.marathon.Client """ if config is None: config = util.get_config() marathon_url = _get_marathon_url(config) timeout = config.get('core.timeout', http.DEFAULT_TIMEOUT) logger.info('Creating marathon client with: %r', marathon_url) return Client(marathon_url, timeout=timeout) def _get_marathon_url(config): """ :param config: configuration dictionary :type config: config.Toml :returns: marathon base url :rtype: str """ marathon_url = config.get('marathon.url') if marathon_url is None: dcos_url = util.get_config_vals(['core.dcos_url'], config)[0] marathon_url = urllib.parse.urljoin(dcos_url, 'marathon/') return marathon_url def _to_exception(response): """ :param response: HTTP response object or Exception :type response: requests.Response | Exception :returns: An exception with the message from the response JSON :rtype: Exception """ if response.status_code == 400: msg = 'Error on request [{0} {1}]: HTTP {2}: {3}'.format( response.request.method, response.request.url, response.status_code, response.reason) # Marathon is buggy and sometimes return JSON, and sometimes # HTML. We only include the error message if it's JSON. try: json_msg = response.json() msg += ':\n' + json.dumps(json_msg, indent=2, sort_keys=True, separators=(',', ': ')) except ValueError: pass return DCOSException(msg) elif response.status_code == 409: return DCOSException( 'App or group is locked by one or more deployments. ' 'Override with --force.') try: response_json = response.json() except Exception: logger.exception( 'Unable to decode response body as a JSON value: %r', response) return DCOSException( 'Error decoding response from [{0}]: HTTP {1}: {2}'.format( response.request.url, response.status_code, response.reason)) message = response_json.get('message') if message is None: errs = response_json.get('errors') if errs is None: logger.error( 'Marathon server did not return a message: %s', response_json) return DCOSException(_default_marathon_error()) msg = '\n'.join(error['error'] for error in errs) return DCOSException(_default_marathon_error(msg)) return DCOSException('Error: {}'.format(message)) def _http_req(fn, *args, **kwargs): """Make an HTTP request, and raise a marathon-specific exception for HTTP error codes. :param fn: function to call :type fn: function :param args: args to pass to `fn` :type args: [object] :param kwargs: kwargs to pass to `fn` :type kwargs: dict :returns: `fn` return value :rtype: object """ try: return fn(*args, **kwargs) except DCOSHTTPException as e: raise _to_exception(e.response) class Client(object): """Class for talking to the Marathon server. :param marathon_url: the base URL for the Marathon server :type marathon_url: str """ def __init__(self, marathon_url, timeout=http.DEFAULT_TIMEOUT): self._base_url = marathon_url self._timeout = timeout min_version = "0.8.1" version = LooseVersion(self.get_about()["version"]) self._version = version if version < LooseVersion(min_version): msg = ("The configured Marathon with version {0} is outdated. " + "Please use version {1} or later.").format( version, min_version) raise DCOSException(msg) def _create_url(self, path): """Creates the url from the provided path. :param path: url path :type path: str :returns: constructed url :rtype: str """ return urllib.parse.urljoin(self._base_url, path) def get_version(self): """Get marathon version :returns: marathon version rtype: LooseVersion """ return self._version def get_about(self): """Returns info about Marathon instance :returns Marathon information :rtype: dict """ url = self._create_url('v2/info') response = _http_req(http.get, url, timeout=self._timeout) return response.json() def get_app(self, app_id, version=None): """Returns a representation of the requested application version. If version is None the return the latest version. :param app_id: the ID of the application :type app_id: str :param version: application version as a ISO8601 datetime :type version: str :returns: the requested Marathon application :rtype: dict """ app_id = self.normalize_app_id(app_id) if version is None: url = self._create_url('v2/apps{}'.format(app_id)) else: url = self._create_url( 'v2/apps{}/versions/{}'.format(app_id, version)) response = _http_req(http.get, url, timeout=self._timeout) # Looks like Marathon return different JSON for versions if version is None: return response.json()['app'] else: return response.json() def get_groups(self): """Get a list of known groups. :returns: list of known groups :rtype: list of dict """ url = self._create_url('v2/groups') response = _http_req(http.get, url, timeout=self._timeout) return response.json()['groups'] def get_group(self, group_id, version=None): """Returns a representation of the requested group version. If version is None the return the latest version. :param group_id: the ID of the application :type group_id: str :param version: application version as a ISO8601 datetime :type version: str :returns: the requested Marathon application :rtype: dict """ group_id = self.normalize_app_id(group_id) if version is None: url = self._create_url('v2/groups{}'.format(group_id)) else: url = self._create_url( 'v2/groups{}/versions/{}'.format(group_id, version)) response = _http_req(http.get, url, timeout=self._timeout) return response.json() def get_app_versions(self, app_id, max_count=None): """Asks Marathon for all the versions of the Application up to a maximum count. :param app_id: the ID of the application or group :type app_id: str :param id_type: type of the id ("apps" or "groups") :type app_id: str :param max_count: the maximum number of version to fetch :type max_count: int :returns: a list of all the version of the application :rtype: [str] """ if max_count is not None and max_count <= 0: raise DCOSException( 'Maximum count must be a positive number: {}'.format(max_count) ) app_id = self.normalize_app_id(app_id) url = self._create_url('v2/apps{}/versions'.format(app_id)) response = _http_req(http.get, url, timeout=self._timeout) if max_count is None: return response.json()['versions'] else: return response.json()['versions'][:max_count] def get_apps(self): """Get a list of known applications. :returns: list of known applications :rtype: [dict] """ url = self._create_url('v2/apps') response = _http_req(http.get, url, timeout=self._timeout) return response.json()['apps'] def add_app(self, app_resource): """Add a new application. :param app_resource: application resource :type app_resource: dict, bytes or file :returns: the application description :rtype: dict """ url = self._create_url('v2/apps') # The file type exists only in Python 2, preventing type(...) is file. if hasattr(app_resource, 'read'): app_json = json.load(app_resource) else: app_json = app_resource response = _http_req(http.post, url, json=app_json, timeout=self._timeout) return response.json() def _update(self, resource_id, payload, force=None, url_endpoint="apps"): """Update an application or group. :param resource_id: the app or group id :type resource_id: str :param payload: the json payload :type payload: dict :param force: whether to override running deployments :type force: bool :param url_endpoint: resource type to update ("apps" or "groups") :type url_endpoint: str :returns: the resulting deployment ID :rtype: str """ resource_id = self.normalize_app_id(resource_id) if not force: params = None else: params = {'force': 'true'} url = self._create_url('v2/{}{}'.format(url_endpoint, resource_id)) response = _http_req(http.put, url, params=params, json=payload, timeout=self._timeout) return response.json().get('deploymentId') def update_app(self, app_id, payload, force=None): """Update an application. :param app_id: the application id :type app_id: str :param payload: the json payload :type payload: dict :param force: whether to override running deployments :type force: bool :returns: the resulting deployment ID :rtype: str """ return self._update(app_id, payload, force) def update_group(self, group_id, payload, force=None): """Update a group. :param group_id: the group id :type group_id: str :param payload: the json payload :type payload: dict :param force: whether to override running deployments :type force: bool :returns: the resulting deployment ID :rtype: str """ return self._update(group_id, payload, force, "groups") def scale_app(self, app_id, instances, force=None): """Scales an application to the requested number of instances. :param app_id: the ID of the application to scale :type app_id: str :param instances: the requested number of instances :type instances: int :param force: whether to override running deployments :type force: bool :returns: the resulting deployment ID :rtype: str """ app_id = self.normalize_app_id(app_id) if not force: params = None else: params = {'force': 'true'} url = self._create_url('v2/apps{}'.format(app_id)) response = _http_req(http.put, url, params=params, json={'instances': int(instances)}, timeout=self._timeout) deployment = response.json()['deploymentId'] return deployment def scale_group(self, group_id, scale_factor, force=None): """Scales a group with the requested scale-factor. :param group_id: the ID of the group to scale :type group_id: str :param scale_factor: the requested value of scale-factor :type scale_factor: float :param force: whether to override running deployments :type force: bool :returns: the resulting deployment ID :rtype: bool """ group_id = self.normalize_app_id(group_id) if not force: params = None else: params = {'force': 'true'} url = self._create_url('v2/groups{}'.format(group_id)) response = http.put(url, params=params, json={'scaleBy': scale_factor}, timeout=self._timeout) deployment = response.json()['deploymentId'] return deployment def stop_app(self, app_id, force=None): """Scales an application to zero instances. :param app_id: the ID of the application to stop :type app_id: str :param force: whether to override running deployments :type force: bool :returns: the resulting deployment ID :rtype: bool """ return self.scale_app(app_id, 0, force) def remove_app(self, app_id, force=None): """Completely removes the requested application. :param app_id: the ID of the application to remove :type app_id: str :param force: whether to override running deployments :type force: bool :rtype: None """ app_id = self.normalize_app_id(app_id) if not force: params = None else: params = {'force': 'true'} url = self._create_url('v2/apps{}'.format(app_id)) _http_req(http.delete, url, params=params, timeout=self._timeout) def remove_group(self, group_id, force=None): """Completely removes the requested application. :param group_id: the ID of the application to remove :type group_id: str :param force: whether to override running deployments :type force: bool :rtype: None """ group_id = self.normalize_app_id(group_id) if not force: params = None else: params = {'force': 'true'} url = self._create_url('v2/groups{}'.format(group_id)) _http_req(http.delete, url, params=params, timeout=self._timeout) def kill_tasks(self, app_id, scale=None, host=None): """Kills the tasks for a given application, and can target a given agent, with a future target scale :param app_id: the id of the application to restart :type app_id: str :param scale: Scale the app down after killing the specified tasks :type scale: bool :param host: host to target restarts on :type host: string """ params = {} app_id = self.normalize_app_id(app_id) if host: params['host'] = host if scale: params['scale'] = scale url = self._create_url('v2/apps{}/tasks'.format(app_id)) response = _http_req(http.delete, url, params=params, timeout=self._timeout) return response.json() def restart_app(self, app_id, force=None): """Performs a rolling restart of all of the tasks. :param app_id: the id of the application to restart :type app_id: str :param force: whether to override running deployments :type force: bool :returns: the deployment id and version :rtype: dict """ app_id = self.normalize_app_id(app_id) if not force: params = None else: params = {'force': 'true'} url = self._create_url('v2/apps{}/restart'.format(app_id)) response = _http_req(http.post, url, params=params, timeout=self._timeout) return response.json() def get_deployment(self, deployment_id): """Returns a deployment. :param deployment_id: the deployment id :type deployment_id: str :returns: a deployment :rtype: dict """ url = self._create_url('v2/deployments') response = _http_req(http.get, url, timeout=self._timeout) deployment = next( (deployment for deployment in response.json() if deployment_id == deployment['id']), None) return deployment def get_deployments(self, app_id=None): """Returns a list of deployments, optionally limited to an app. :param app_id: the id of the application :type app_id: str :returns: a list of deployments :rtype: list of dict """ url = self._create_url('v2/deployments') response = _http_req(http.get, url, timeout=self._timeout) if app_id is not None: app_id = self.normalize_app_id(app_id) deployments = [ deployment for deployment in response.json() if app_id in deployment['affectedApps'] ] else: deployments = response.json() return deployments def _cancel_deployment(self, deployment_id, force): """Cancels an application deployment. :param deployment_id: the deployment id :type deployment_id: str :param force: if set to `False`, stop the deployment and create a new rollback deployment to reinstate the previous configuration. If set to `True`, simply stop the deployment. :type force: bool :returns: cancelation deployment :rtype: dict """ if not force: params = None else: params = {'force': 'true'} url = self._create_url('v2/deployments/{}'.format(deployment_id)) response = _http_req(http.delete, url, params=params, timeout=self._timeout) if force: return None else: return response.json() def rollback_deployment(self, deployment_id): """Rolls back an application deployment. :param deployment_id: the deployment id :type deployment_id: str :returns: cancelation deployment :rtype: dict """ return self._cancel_deployment(deployment_id, False) def stop_deployment(self, deployment_id): """Stops an application deployment. :param deployment_id: the deployment id :type deployment_id: str :rtype: None """ self._cancel_deployment(deployment_id, True) def get_tasks(self, app_id): """Returns a list of tasks, optionally limited to an app. :param app_id: the id of the application to restart :type app_id: str :returns: a list of tasks :rtype: [dict] """ url = self._create_url('v2/tasks') response = _http_req(http.get, url, timeout=self._timeout) if app_id is not None: app_id = self.normalize_app_id(app_id) tasks = [ task for task in response.json()['tasks'] if app_id == task['appId'] ] else: tasks = response.json()['tasks'] return tasks def get_task(self, task_id): """Returns a task :param task_id: the id of the task :type task_id: str :returns: a tasks :rtype: dict """ url = self._create_url('v2/tasks') response = _http_req(http.get, url, timeout=self._timeout) task = next( (task for task in response.json()['tasks'] if task_id == task['id']), None) return task def get_app_schema(self): """Returns app json schema :returns: application json schema :rtype: json schema or None if endpoint doesn't exist """ version = self.get_version() schema_version = LooseVersion("0.9.0") if version < schema_version: return None url = self._create_url('v2/schemas/app') response = _http_req(http.get, url, timeout=self._timeout) return response.json() def normalize_app_id(self, app_id): """Normalizes the application id. :param app_id: raw application ID :type app_id: str :returns: normalized application ID :rtype: str """ return urllib.parse.quote('/' + app_id.strip('/')) def create_group(self, group_resource): """Add a new group. :param group_resource: grouplication resource :type group_resource: dict, bytes or file :returns: the group description :rtype: dict """ url = self._create_url('v2/groups') # The file type exists only in Python 2, preventing type(...) is file. if hasattr(group_resource, 'read'): group_json = json.load(group_resource) else: group_json = group_resource response = _http_req(http.post, url, json=group_json, timeout=self._timeout) return response.json() def get_leader(self): """ Get the leading marathon instance. :returns: string of the form : :rtype: str """ url = self._create_url('v2/leader') response = _http_req(http.get, url, timeout=self._timeout) return response.json()['leader'] def _default_marathon_error(message=""): """ :param message: additional message :type message: str :returns: marathon specific error message :rtype: str """ return ("Marathon likely misconfigured. Please check your proxy or " "Marathon URL settings. See dcos config --help. {}").format( message) PKցFHԻudcos/jsonitem.pyimport collections import json import re from dcos import util from dcos.errors import DCOSException logger = util.get_logger(__name__) def parse_json_item(json_item, schema): """Parse the json item (optionally based on a schema). :param json_item: A JSON item in the form 'key=value' :type json_item: str :param schema: The JSON schema to use for parsing :type schema: dict :returns: A tuple for the parsed JSON item :rtype: (str, any) where any is one of str, int, float, bool, list or dict """ terms = json_item.split('=', 1) if len(terms) != 2: raise DCOSException('{!r} is not a valid json-item'.format(json_item)) # Check that it is a valid key in our jsonschema key = terms[0] # Use the schema if we have it else, guess the type if schema: value = parse_json_value(key, terms[1], schema) else: value = _find_type(clean_value(terms[1])) return (json.dumps(key), value) def parse_json_value(key, value, schema): """Parse the json value based on a schema. :param key: the key property :type key: str :param value: the value of property :type value: str :param schema: The JSON schema to use for parsing :type schema: dict :returns: parsed value :rtype: str | int | float | bool | list | dict """ value_type = find_parser(key, schema) return value_type(value) def find_parser(key, schema): """ :param key: JSON field :type key: str :param schema: The JSON schema to use :type schema: dict :returns: A callable capable of parsing a string to its type :rtype: ValueTypeParser """ key_schema = schema['properties'].get(key) if key_schema is None: keys = ', '.join(schema['properties'].keys()) raise DCOSException( 'Error: {!r} is not a valid property. ' 'Possible properties are: {}'.format(key, keys)) else: return ValueTypeParser(key_schema) class ValueTypeParser(object): """Callable for parsing a string against a known JSON type. :param schema: The JSON type as a schema :type schema: dict """ def __init__(self, schema): self.schema = schema def __call__(self, value): """ :param value: String to try and parse :type value: str :returns: The parse value :rtype: str | int | float | bool | list | dict """ value = clean_value(value) if self.schema['type'] == 'string': if self.schema.get('format') == 'uri': return _parse_url(value) else: return _parse_string(value) elif self.schema['type'] == 'object': return _parse_object(value) elif self.schema['type'] == 'number': return _parse_number(value) elif self.schema['type'] == 'integer': return _parse_integer(value) elif self.schema['type'] == 'boolean': return _parse_boolean(value) elif self.schema['type'] == 'array': return _parse_array(value) else: raise DCOSException('Unknown type {!r}'.format(self._value_type)) def clean_value(value): """ :param value: String to try and clean :type value: str :returns: The cleaned string :rtype: str """ if len(value) > 1 and value.startswith('"') and value.endswith('"'): return value[1:-1] elif len(value) > 1 and value.startswith("'") and value.endswith("'"): return value[1:-1] else: return value def _find_type(value): """Find the correct type of the value :param value: value to parse :type value: str :returns: The parsed value :rtype: int|float| """ to_try = [_parse_integer, _parse_number, _parse_boolean, _parse_array, _parse_url, _parse_object, _parse_string] while len(to_try) > 0: try: return to_try.pop(0)(value) except DCOSException: pass raise DCOSException( 'Unable to parse {!r} as a JSON object'.format(value)) def _parse_string(value): """ :param value: The string to parse :type value: str :returns: The parsed value :rtype: str """ return None if value == 'null' else value def _parse_object(value): """ :param value: The string to parse :type value: str :returns: The parsed value :rtype: dict """ try: json_object = json.loads(value) if json_object is None or isinstance(json_object, collections.Mapping): return json_object else: raise DCOSException( 'Unable to parse {!r} as a JSON object'.format(value)) except ValueError as error: logger.exception('Error parsing value as a JSON object') msg = 'Unable to parse {!r} as a JSON object: {}'.format(value, error) raise DCOSException(msg) def _parse_number(value): """ :param value: The string to parse :type value: str :returns: The parsed value :rtype: float """ try: return None if value == 'null' else float(value) except ValueError as error: logger.exception('Error parsing value as a JSON number') msg = 'Unable to parse {!r} as a float: {}'.format(value, error) raise DCOSException(msg) def _parse_integer(value): """ :param value: The string to parse :type value: str :returns: The parsed value :rtype: int """ try: return None if value == 'null' else int(value) except ValueError as error: logger.exception('Error parsing value as a JSON integer') msg = 'Unable to parse {!r} as an int: {}'.format(value, error) raise DCOSException(msg) def _parse_boolean(value): """ :param value: The string to parse :type value: str :returns: The parsed value :rtype: bool """ try: boolean = json.loads(value) if boolean is None or isinstance(boolean, bool): return boolean else: raise DCOSException( 'Unable to parse {!r} as a boolean'.format(value)) except ValueError as error: logger.exception('Error parsing value as a JSON boolean') msg = 'Unable to parse {!r} as a boolean: {}'.format(value, error) raise DCOSException(msg) def _parse_array(value): """ :param value: The string to parse :type value: str :returns: The parsed value :rtype: list """ try: array = json.loads(value) if array is None or isinstance(array, collections.Sequence): return array else: raise DCOSException( 'Unable to parse {!r} as an array'.format(value)) except ValueError as error: logger.exception('Error parsing value as a JSON array') msg = 'Unable to parse {!r} as an array: {}'.format(value, error) raise DCOSException(msg) def _parse_url(value): """ :param value: The url to parse :type url: str :returns: The parsed value :rtype: str """ scheme_pattern = r'^(?P(?:(?:https?)://))' domain_pattern = ( r'(?P(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.?)+' '(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)?|') # domain, value_regex = re.match( scheme_pattern + # http:// or https:// r'(([^:])+(:[^:]+)?@){0,1}' + # auth credentials domain_pattern + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}))' # or ip r'(?P(?::\d+))?' # port r'(?P(?:/?|[/?]\S+))$', # resource path value, re.IGNORECASE) if value_regex is None: scheme_match = re.match(scheme_pattern, value, re.IGNORECASE) if scheme_match is None: msg = 'Please check url {!r}. Missing http(s)://'.format(value) raise DCOSException(msg) else: raise DCOSException( 'Unable to parse {!r} as a url'.format(value)) else: return value PKցFHD`kk dcos/mesos.pyimport fnmatch import itertools import os from dcos import http, util from dcos.errors import DCOSException, DCOSHTTPException from six.moves import urllib logger = util.get_logger(__name__) def get_master(dcos_client=None): """Create a Master object using the url stored in the 'core.mesos_master_url' property if it exists. Otherwise, we use the `core.dcos_url` property :param dcos_client: DCOSClient :type dcos_client: DCOSClient | None :returns: master state object :rtype: Master """ dcos_client = dcos_client or DCOSClient() return Master(dcos_client.get_master_state()) class DCOSClient(object): """Client for communicating with DCOS""" def __init__(self): config = util.get_config() self._dcos_url = None self._mesos_master_url = None mesos_master_url = config.get('core.mesos_master_url') if mesos_master_url is None: self._dcos_url = util.get_config_vals(['core.dcos_url'], config)[0] else: self._mesos_master_url = mesos_master_url self._timeout = config.get('core.timeout') def get_dcos_url(self, path): """ Create a DCOS URL :param path: the path suffix of the URL :type path: str :returns: DCOS URL :rtype: str """ if self._dcos_url: return urllib.parse.urljoin(self._dcos_url, path) else: raise util.missing_config_exception('core.dcos_url') def master_url(self, path): """ Create a master URL :param path: the path suffix of the desired URL :type path: str :returns: URL that hits the master :rtype: str """ base_url = (self._mesos_master_url or urllib.parse.urljoin(self._dcos_url, 'mesos/')) return urllib.parse.urljoin(base_url, path) def slave_url(self, slave_id, private_url, path): """Create a slave URL :param slave_id: slave ID :type slave_id: str :param private_url: The slave's private URL derived from its pid. Used when we're accessing mesos directly, rather than through DCOS. :type private_url: str :param path: the path suffix of the desired URL :type path: str :returns: URL that hits the master :rtype: str """ if self._dcos_url: return urllib.parse.urljoin(self._dcos_url, 'slave/{}/{}'.format(slave_id, path)) else: return urllib.parse.urljoin(private_url, path) def get_master_state(self): """Get the Mesos master state json object :returns: Mesos' master state json object :rtype: dict """ url = self.master_url('master/state.json') return http.get(url, timeout=self._timeout).json() def get_slave_state(self, slave_id, private_url): """Get the Mesos slave state json object :param slave_id: slave ID :type slave_id: str :param private_url: The slave's private URL derived from its pid. Used when we're accessing mesos directly, rather than through DCOS. :type private_url: str :returns: Mesos' master state json object :rtype: dict """ url = self.slave_url(slave_id, private_url, 'state.json') return http.get(url, timeout=self._timeout).json() def get_state_summary(self): """Get the Mesos master state summary json object :returns: Mesos' master state summary json object :rtype: dict """ url = self.master_url('master/state-summary') return http.get(url, timeout=self._timeout).json() def slave_file_read(self, slave_id, private_url, path, offset, length): """See the master_file_read() docs :param slave_id: slave ID :type slave_id: str :param path: absolute path to read :type path: str :param private_url: The slave's private URL derived from its pid. Used when we're accessing mesos directly, rather than through DCOS. :type private_url: str :param offset: start byte location, or -1. -1 means read no data, and is used to fetch the size of the file in the response's 'offset' parameter. :type offset: int :param length: number of bytes to read, or -1. -1 means read the whole file :type length: int :returns: files/read.json response :rtype: dict """ url = self.slave_url(slave_id, private_url, 'files/read.json') params = {'path': path, 'length': length, 'offset': offset} return http.get(url, params=params, timeout=self._timeout).json() def master_file_read(self, path, length, offset): """This endpoint isn't well documented anywhere, so here is the spec derived from the mesos source code: request format: { path: absolute path to read offset: start byte location, or -1. -1 means read no data, and is used to fetch the size of the file in the response's 'offset' parameter. length: number of bytes to read, or -1. -1 means read the whole file. } response format: { data: file data. Empty if a request.offset=-1. Could be smaller than request.length if EOF was reached, or if (I believe) request.length is larger than the length supported by the server (16 pages I believe). offset: the offset value from the request, or the size of the file if the request offset was -1 or >= the file size. } :param path: absolute path to read :type path: str :param offset: start byte location, or -1. -1 means read no data, and is used to fetch the size of the file in the response's 'offset' parameter. :type offset: int :param length: number of bytes to read, or -1. -1 means read the whole file :type length: int :returns: files/read.json response :rtype: dict """ url = self.master_url('files/read.json') params = {'path': path, 'length': length, 'offset': offset} return http.get(url, params=params, timeout=self._timeout).json() def shutdown_framework(self, framework_id): """Shuts down a Mesos framework :param framework_id: ID of the framework to shutdown :type framework_id: str :returns: None """ logger.info('Shutting down framework {}'.format(framework_id)) data = 'frameworkId={}'.format(framework_id) url = self.master_url('master/teardown') # In Mesos 0.24, /shutdown was removed. # If /teardown doesn't exist, we try /shutdown. try: http.post(url, data=data, timeout=self._timeout) except DCOSHTTPException as e: if e.response.status_code == 404: url = self.master_url('master/shutdown') http.post(url, data=data, timeout=self._timeout) else: raise def metadata(self): """ GET /metadata :returns: /metadata content :rtype: dict """ url = self.get_dcos_url('metadata') return http.get(url, timeout=self._timeout).json() def browse(self, slave, path): """ GET /files/browse.json Request path:... # path to run ls on Response [ { path: # full path to file nlink: size: mtime: mode: uid: gid: } ] :param slave: slave to issue the request on :type slave: Slave :returns: /files/browse.json response :rtype: dict """ url = self.slave_url(slave['id'], slave.http_url(), 'files/browse.json') return http.get(url, params={'path': path}).json() class MesosDNSClient(object): """ Mesos-DNS client :param url: mesos-dns URL :type url: str """ def __init__(self, url=None): self.url = url or urllib.parse.urljoin( util.get_config_vals(['core.dcos_url'])[0], '/mesos_dns/') def _path(self, path): """ Construct a full path :param path: path suffix :type path: str :returns: full path :rtype: str """ return urllib.parse.urljoin(self.url, path) def hosts(self, host): """ GET v1/hosts/ :param host: host :type host: str :returns: {'ip', 'host'} dictionary :rtype: dict(str, str) """ url = self._path('v1/hosts/{}'.format(host)) return http.get(url, headers={}).json() class Master(object): """Mesos Master Model :param state: Mesos master's state.json :type state: dict """ def __init__(self, state): self._state = state self._frameworks = {} self._slaves = {} def state(self): """Returns master's master/state.json. :returns: state.json :rtype: dict """ return self._state def slave_base_url(self, slave): """Returns the base url of the provided slave object. :param slave: slave to create a url for :type slave: Slave :returns: slave's base url :rtype: str """ if self._mesos_master_url is not None: slave_ip = slave['pid'].split('@')[1] return 'http://{}'.format(slave_ip) else: return urllib.parse.urljoin(self._dcos_url, 'slave/{}/'.format(slave['id'])) def slave(self, fltr): """Returns the slave that has `fltr` in its ID. Raises a DCOSException if there is not exactly one such slave. :param fltr: filter string :type fltr: str :returns: the slave that has `fltr` in its ID :rtype: Slave """ slaves = self.slaves(fltr) if len(slaves) == 0: raise DCOSException('No slave found with ID "{}".'.format(fltr)) elif len(slaves) > 1: matches = ['\t{0}'.format(slave['id']) for slave in slaves] raise DCOSException( "There are multiple slaves with that ID. " + "Please choose one: {}".format('\n'.join(matches))) else: return slaves[0] def task(self, fltr): """Returns the task with `fltr` in its ID. Raises a DCOSException if there is not exactly one such task. :param fltr: filter string :type fltr: str :returns: the task that has `fltr` in its ID :rtype: Task """ tasks = self.tasks(fltr) if len(tasks) == 0: raise DCOSException( 'Cannot find a task with ID containing "{}"'.format(fltr)) elif len(tasks) > 1: msg = [("There are multiple tasks with ID matching [{}]. " + "Please choose one:").format(fltr)] msg += ["\t{0}".format(t["id"]) for t in tasks] raise DCOSException('\n'.join(msg)) else: return tasks[0] def framework(self, framework_id): """Returns a framework by ID :param framework_id: the framework's ID :type framework_id: str :returns: the framework :rtype: Framework """ for f in self._framework_dicts(True, True): if f['id'] == framework_id: return self._framework_obj(f) return None def slaves(self, fltr=""): """Returns those slaves that have `fltr` in their 'id' :param fltr: filter string :type fltr: str :returns: Those slaves that have `fltr` in their 'id' :rtype: [Slave] """ return [self._slave_obj(slave) for slave in self.state()['slaves'] if fltr in slave['id']] def tasks(self, fltr="", completed=False): """Returns tasks running under the master :param fltr: May be a substring or regex. Only return tasks whose 'id' matches `fltr`. :type fltr: str :param completed: also include completed tasks :type completed: bool :returns: a list of tasks :rtype: [Task] """ keys = ['tasks'] if completed: keys = ['completed_tasks'] tasks = [] for framework in self._framework_dicts(completed, completed): for task in _merge(framework, keys): if fltr in task['id'] or fnmatch.fnmatchcase(task['id'], fltr): task = self._framework_obj(framework).task(task['id']) tasks.append(task) return tasks def frameworks(self, inactive=False, completed=False): """Returns a list of all frameworks :param inactive: also include inactive frameworks :type inactive: bool :param completed: also include completed frameworks :type completed: bool :returns: a list of frameworks :rtype: [Framework] """ return [self._framework_obj(framework) for framework in self._framework_dicts(inactive, completed)] @util.duration def fetch(self, path, **kwargs): """GET the resource located at `path` :param path: the URL path :type path: str :param **kwargs: http.get kwargs :type **kwargs: dict :returns: the response object :rtype: Response """ url = urllib.parse.urljoin(self._base_url(), path) return http.get(url, **kwargs) def _slave_obj(self, slave): """Returns the Slave object corresponding to the provided `slave` dict. Creates it if it doesn't exist already. :param slave: slave :type slave: dict :returns: Slave :rtype: Slave """ if slave['id'] not in self._slaves: self._slaves[slave['id']] = Slave(slave, None, self) return self._slaves[slave['id']] def _framework_obj(self, framework): """Returns the Framework object corresponding to the provided `framework` dict. Creates it if it doesn't exist already. :param framework: framework :type framework: dict :returns: Framework :rtype: Framework """ if framework['id'] not in self._frameworks: self._frameworks[framework['id']] = Framework(framework, self) return self._frameworks[framework['id']] def _framework_dicts(self, inactive=False, completed=False): """Returns a list of all frameworks as their raw dictionaries :param inactive: also include inactive frameworks :type inactive: bool :param completed: also include completed frameworks :type completed: bool :returns: a list of frameworks """ if completed: for framework in self.state()['completed_frameworks']: yield framework for framework in self.state()['frameworks']: if inactive or framework['active']: yield framework class Slave(object): """Mesos Slave Model :param short_state: slave's entry from the master's state.json :type short_state: dict :param state: slave's state.json :type state: dict | None :param master: slave's master :type master: Master """ def __init__(self, short_state, state, master): self._short_state = short_state self._state = state self._master = master def state(self): """Get the slave's state.json object. Fetch it if it's not already an instance variable. :returns: This slave's state.json object :rtype: dict """ if not self._state: self._state = DCOSClient().get_slave_state(self['id'], self.http_url()) return self._state def http_url(self): """ :returns: The private HTTP URL of the slave. Derived from the `pid` property. :rtype: str """ parsed_pid = parse_pid(self['pid']) return 'http://{}:{}'.format(parsed_pid[1], parsed_pid[2]) def _framework_dicts(self): """Returns the framework dictionaries from the state.json dict :returns: frameworks :rtype: [dict] """ return _merge(self.state(), ['frameworks', 'completed_frameworks']) def executor_dicts(self): """Returns the executor dictionaries from the state.json :returns: executors :rtype: [dict] """ iters = [_merge(framework, ['executors', 'completed_executors']) for framework in self._framework_dicts()] return itertools.chain(*iters) def __getitem__(self, name): """Support the slave[attr] syntax :param name: attribute to get :type name: str :returns: the value for this attribute in the underlying slave dictionary :rtype: object """ return self._short_state[name] class Framework(object): """ Mesos Framework Model :param framework: framework properties :type framework: dict :param master: framework's master :type master: Master """ def __init__(self, framework, master): self._framework = framework self._master = master self._tasks = {} # id->Task map def task(self, task_id): """Returns a task by id :param task_id: the task's id :type task_id: str :returns: the task :rtype: Task """ for task in _merge(self._framework, ['tasks', 'completed_tasks']): if task['id'] == task_id: return self._task_obj(task) return None def _task_obj(self, task): """Returns the Task object corresponding to the provided `task` dict. Creates it if it doesn't exist already. :param task: task :type task: dict :returns: Task :rtype: Task """ if task['id'] not in self._tasks: self._tasks[task['id']] = Task(task, self._master) return self._tasks[task['id']] def dict(self): return self._framework def __getitem__(self, name): """Support the framework[attr] syntax :param name: attribute to get :type name: str :returns: the value for this attribute in the underlying framework dictionary :rtype: object """ return self._framework[name] class Task(object): """Mesos Task Model. :param task: task properties :type task: dict :param master: mesos master :type master: Master """ def __init__(self, task, master): self._task = task self._master = master def dict(self): """ :returns: dictionary representation of this Task :rtype: dict """ return self._task def framework(self): """Returns this task's framework :returns: task's framework :rtype: Framework """ return self._master.framework(self["framework_id"]) def slave(self): """Returns the task's slave :returns: task's slave :rtype: Slave """ return self._master.slave(self["slave_id"]) def user(self): """Task owner :returns: task owner :rtype: str """ return self.framework()['user'] def executor(self): """ Returns this tasks' executor :returns: task's executor :rtype: dict """ for executor in self.slave().executor_dicts(): tasks = _merge(executor, ['completed_tasks', 'tasks', 'queued_tasks']) if any(task['id'] == self['id'] for task in tasks): return executor return None def directory(self): """ Sandbox directory for this task :returns: path to task's sandbox :rtype: str """ return self.executor()['directory'] def __getitem__(self, name): """Support the task[attr] syntax :param name: attribute to get :type name: str :returns: the value for this attribute in the underlying task dictionary :rtype: object """ return self._task[name] class MesosFile(object): """File-like object that is backed by a remote slave or master file. Uses the files/read.json endpoint. If `task` is provided, the file host is `task.slave()`. If `slave` is provided, the file host is `slave`. It is invalid to provide both. If neither is provided, the file host is the leading master. :param path: file's path, relative to the sandbox if `task` is given :type path: str :param task: file's task :type task: Task | None :param slave: slave where the file lives :type slave: Slave | None :param dcos_client: client to use for network requests :type dcos_client: DCOSClient | None """ def __init__(self, path, task=None, slave=None, dcos_client=None): if task and slave: raise ValueError( "You cannot provide both `task` and `slave` " + "arguments. `slave` is understood to be `task.slave()`") if slave: self._slave = slave elif task: self._slave = task.slave() else: self._slave = None self._task = task self._path = path self._dcos_client = dcos_client or DCOSClient() self._cursor = 0 def size(self): """Size of the file :returns: size of the file :rtype: int """ params = self._params(0, offset=-1) return self._fetch(params)["offset"] def seek(self, offset, whence=os.SEEK_SET): """Seek to the provided location in the file. :param offset: location to seek to :type offset: int :param whence: determines whether `offset` represents a location that is absolute, relative to the beginning of the file, or relative to the end of the file :type whence: os.SEEK_SET | os.SEEK_CUR | os.SEEK_END :returns: None :rtype: None """ if whence == os.SEEK_SET: self._cursor = 0 + offset elif whence == os.SEEK_CUR: self._cursor += offset elif whence == os.SEEK_END: self._cursor = self.size() + offset else: raise ValueError( "Unexpected value for `whence`: {}".format(whence)) def tell(self): """ The current cursor position. :returns: the current cursor position :rtype: int """ return self._cursor def read(self, length=None): """Reads up to `length` bytes, or the entire file if `length` is None. :param length: number of bytes to read :type length: int | None :returns: data read :rtype: str """ data = '' while length is None or length - len(data) > 0: chunk_length = -1 if length is None else length - len(data) chunk = self._fetch_chunk(chunk_length) if chunk == '': break data += chunk return data def _host_path(self): """ The absolute path to the file on slave. :returns: the absolute path to the file on slave :rtype: str """ if self._task: directory = self._task.directory() if directory[-1] == '/': return directory + self._path else: return directory + '/' + self._path else: return self._path def _params(self, length, offset=None): """GET parameters to send to files/read.json. See the MesosFile docstring for full information. :param length: number of bytes to read :type length: int :param offset: start location. if None, will use the location of the current file cursor :type offset: int :returns: GET parameters :rtype: dict """ if offset is None: offset = self._cursor return { 'path': self._host_path(), 'offset': offset, 'length': length } def _fetch_chunk(self, length, offset=None): """Fetch data from files/read.json :param length: number of bytes to fetch :type length: int :param offset: start location. If not None, this file's cursor is set to `offset` :type offset: int :returns: data read :rtype: str """ if offset is not None: self.seek(offset, os.SEEK_SET) params = self._params(length) data = self._fetch(params)["data"] self.seek(len(data), os.SEEK_CUR) return data def _fetch(self, params): """Fetch data from files/read.json :param params: GET parameters :type params: dict :returns: response dict :rtype: dict """ if self._slave: return self._dcos_client.slave_file_read(self._slave['id'], self._slave.http_url(), **params) else: return self._dcos_client.master_file_read(**params) def __str__(self): """String representation of the file: :returns: string representation of the file :rtype: str """ if self._task: return "task:{0}:{1}".format(self._task['id'], self._path) elif self._slave: return "slave:{0}:{1}".format(self._slave['id'], self._path) else: return "master:{0}".format(self._path) def parse_pid(pid): """ Parse the mesos pid string, :param pid: pid of the form "id@ip:port" :type pid: str :returns: (id, ip, port) :rtype: (str, str, str) """ id_, second = pid.split('@') ip, port = second.split(':') return id_, ip, port def _merge(d, keys): """ Merge multiple lists from a dictionary into one iterator. e.g. _merge({'a': [1, 2], 'b': [3]}, ['a', 'b']) -> iter(1, 2, 3) :param d: dictionary :type d: dict :param keys: keys to merge :type keys: [hashable] :returns: iterator :rtype: iter """ return itertools.chain(*[d[k] for k in keys]) PKցFH0~~dcos/config.pyimport collections import toml from dcos import util from dcos.errors import DCOSException def load_from_path(path, mutable=False): """Loads a TOML file from the path :param path: Path to the TOML file :type path: str :param mutable: True if the returned Toml object should be mutable :type mutable: boolean :returns: Map for the configuration file :rtype: Toml | MutableToml """ util.ensure_file_exists(path) with util.open_file(path, 'r') as config_file: try: toml_obj = toml.loads(config_file.read()) except Exception as e: raise DCOSException( 'Error parsing config file at [{}]: {}'.format(path, e)) return (MutableToml if mutable else Toml)(toml_obj) def save(toml_config): """ :param toml_config: TOML configuration object :type toml_config: MutableToml or Toml """ serial = toml.dumps(toml_config._dictionary) path = util.get_config_path() with util.open_file(path, 'w') as config_file: config_file.write(serial) def _get_path(config, path): """ :param config: Dict with the configuration values :type config: dict :param path: Path to the value. E.g. 'path.to.value' :type path: str :returns: Value stored at the given path :rtype: double, int, str, list or dict """ for section in path.split('.'): config = config[section] return config def _iterator(parent, dictionary): """ :param parent: Path to the value parameter :type parent: str :param dictionary: Value of the key :type dictionary: collection.Mapping :returns: An iterator of tuples for each property and value :rtype: iterator of (str, any) where any can be str, int, double, list """ for key, value in dictionary.items(): new_key = key if parent is not None: new_key = "{}.{}".format(parent, key) if not isinstance(value, collections.Mapping): yield (new_key, value) else: for x in _iterator(new_key, value): yield x class Toml(collections.Mapping): """Class for getting value from TOML. :param dictionary: configuration dictionary :type dictionary: dict """ def __init__(self, dictionary): self._dictionary = dictionary def __getitem__(self, path): """ :param path: Path to the value. E.g. 'path.to.value' :type path: str :returns: Value stored at the given path :rtype: double, int, str, list or dict """ config = _get_path(self._dictionary, path) if isinstance(config, collections.Mapping): return Toml(config) else: return config def __iter__(self): """ :returns: Dictionary iterator :rtype: iterator """ return iter(self._dictionary) def property_items(self): """Iterator for full-path keys and values :returns: Iterator for pull-path keys and values :rtype: iterator of tuples """ return _iterator(None, self._dictionary) def __len__(self): """ :returns: The length of the dictionary :rtype: int """ return len(self._dictionary) class MutableToml(collections.MutableMapping): """Class for managing CLI configuration through TOML. :param dictionary: configuration dictionary :type dictionary: dict """ def __init__(self, dictionary): self._dictionary = dictionary def __getitem__(self, path): """ :param path: Path to the value. E.g. 'path.to.value' :type path: str :returns: Value stored at the given path :rtype: double, int, str, list or dict """ config = _get_path(self._dictionary, path) if isinstance(config, collections.MutableMapping): return MutableToml(config) else: return config def __iter__(self): """ :returns: Dictionary iterator :rtype: iterator """ return iter(self._dictionary) def property_items(self): """Iterator for full-path keys and values :returns: Iterator for pull-path keys and values :rtype: iterator of tuples """ return _iterator(None, self._dictionary) def __len__(self): """ :returns: The length of the dictionary :rtype: int """ return len(self._dictionary) def __setitem__(self, path, value): """ :param path: Path to set :type path: str :param value: Value to store :type value: double, int, str, list or dict """ config = self._dictionary sections = path.split('.') for section in sections[:-1]: config = config.setdefault(section, {}) config[sections[-1]] = value def __delitem__(self, path): """ :param path: Path to delete :type path: str """ config = self._dictionary sections = path.split('.') for section in sections[:-1]: config = config[section] del config[sections[-1]] PKցFHqZZdcos/emitting.pyfrom __future__ import print_function import abc import collections import json import os import pydoc import re import sys import pager import pygments import six from dcos import constants, errors, util from pygments.formatters import Terminal256Formatter from pygments.lexers import JsonLexer logger = util.get_logger(__name__) class Emitter(object): """Abstract class for emitting events.""" @abc.abstractmethod def publish(self, event): """Publishes an event. :param event: event to publish :type event: any """ raise NotImplementedError class FlatEmitter(Emitter): """Simple emitter that sends all publish events to the provided handler. If no handler is provider then use :py:const:`DEFAULT_HANDLER`. :param handler: event handler to call when publish is called :type handler: func(event) where event is defined in :py:func:`FlatEmitter.publish` """ def __init__(self, handler=None): if handler is None: self._handler = DEFAULT_HANDLER else: self._handler = handler def publish(self, event): """Publishes an event. :param event: event to publish :type event: any """ self._handler(event) def print_handler(event): """Default handler for printing event to stdout. :param event: event to emit to stdout :type event: str, dict, list, or dcos.errors.Error """ pager_command = os.environ.get(constants.DCOS_PAGER_COMMAND_ENV) if event is None: # Do nothing pass elif isinstance(event, six.string_types): _page(event, pager_command) elif isinstance(event, errors.Error): print(event.error(), file=sys.stderr) sys.stderr.flush() elif (isinstance(event, collections.Mapping) or isinstance(event, collections.Sequence) or isinstance(event, bool) or isinstance(event, six.integer_types) or isinstance(event, float)): # These are all valid JSON types let's treat them different processed_json = _process_json(event, pager_command) _page(processed_json, pager_command) elif isinstance(event, errors.DCOSException): print(event, file=sys.stderr) else: logger.debug('Printing unknown type: %s, %r.', type(event), event) _page(event, pager_command) def publish_table(emitter, objs, table_fn, json_): """Publishes a json representation of `objs` if `json_` is True, otherwise, publishes a table representation. :param emitter: emitter to use for publishing :type emitter: Emitter :param objs: objects to print :type objs: [object] :param table_fn: function used to generate a PrettyTable from `objs` :type table_fn: objs -> PrettyTable :param json_: whether or not to publish a json representation :type json_: bool :rtype: None """ if json_: emitter.publish(objs) else: table = table_fn(objs) output = str(table) if output: emitter.publish(output) def _process_json(event, pager_command): """Conditionally highlights the supplied JSON value. :param event: event to emit to stdout :type event: str, dict, list, or dcos.errors.Error :returns: String representation of the supplied JSON value, possibly syntax-highlighted. :rtype: str """ json_output = json.dumps(event, sort_keys=True, indent=2) # Strip trailing whitespace json_output = re.sub(r'\s+$', '', json_output, 0, re.M) force_colors = False # TODO(CD): Introduce a --colors flag if not sys.stdout.isatty(): if force_colors: return _highlight_json(json_output) else: return json_output supports_colors = not util.is_windows_platform() pager_is_set = pager_command is not None should_highlight = force_colors or supports_colors and not pager_is_set if should_highlight: json_output = _highlight_json(json_output) return json_output def _page(output, pager_command=None): """Conditionally pipes the supplied output through a pager. :param output: :type output: object :param pager_command: :type pager_command: str """ output = str(output) if pager_command is None: pager_command = 'less -R' if not sys.stdout.isatty() or util.is_windows_platform(): print(output) return num_lines = output.count('\n') exceeds_tty_height = pager.getheight() - 1 < num_lines if exceeds_tty_height: pydoc.pipepager(output, cmd=pager_command) else: print(output) def _highlight_json(json_value): """ :param json_value: JSON value to syntax-highlight :type json_value: dict, list, number, string, boolean, or None :returns: A string representation of the supplied JSON value, highlighted for a terminal that supports ANSI colors. :rtype: str """ return pygments.highlight( json_value, JsonLexer(), Terminal256Formatter()).strip() DEFAULT_HANDLER = print_handler """The default handler for an emitter: :py:func:`print_handler`.""" PKցFHHiQQdcos/package.pyimport abc import base64 import collections import copy import hashlib import json import os import re import shutil import stat import subprocess import zipfile from distutils.version import LooseVersion import git import portalocker import pystache import six from dcos import (constants, emitting, errors, http, marathon, mesos, subcommand, util) from dcos.errors import DCOSException, DefaultError from six.moves import urllib logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() PACKAGE_METADATA_KEY = 'DCOS_PACKAGE_METADATA' PACKAGE_NAME_KEY = 'DCOS_PACKAGE_NAME' PACKAGE_VERSION_KEY = 'DCOS_PACKAGE_VERSION' PACKAGE_SOURCE_KEY = 'DCOS_PACKAGE_SOURCE' PACKAGE_FRAMEWORK_KEY = 'DCOS_PACKAGE_IS_FRAMEWORK' PACKAGE_RELEASE_KEY = 'DCOS_PACKAGE_RELEASE' PACKAGE_COMMAND_KEY = 'DCOS_PACKAGE_COMMAND' PACKAGE_REGISTRY_VERSION_KEY = 'DCOS_PACKAGE_REGISTRY_VERSION' PACKAGE_FRAMEWORK_NAME_KEY = 'DCOS_PACKAGE_FRAMEWORK_NAME' def install_app(pkg, revision, init_client, options, app_id): """Installs a package's application :param pkg: the package to install :type pkg: Package :param revision: the package revision to install :type revision: str :param init_client: the program to use to run the package :type init_client: object :param options: package parameters :type options: dict :param app_id: app ID for installation of this package :type app_id: str :rtype: None """ # Insert option parameters into the init template init_desc = pkg.marathon_json(revision, options) if app_id is not None: logger.debug('Setting app ID to "%s" (was "%s")', app_id, init_desc['id']) init_desc['id'] = app_id # Send the descriptor to init init_client.add_app(init_desc) def _make_package_labels(pkg, revision, options): """Returns Marathon app labels for a package. :param pkg: The package to install :type pkg: Package :param revision: The package revision to install :type revision: str :param options: package parameters :type options: dict :returns: Marathon app labels :rtype: dict """ metadata = pkg.package_json(revision) # add images to package.json metadata for backwards compatability in the UI if pkg._has_resource_definition(revision): images = {"images": pkg._resource_json(revision)["images"]} metadata.update(images) encoded_metadata = _base64_encode(metadata) is_framework = metadata.get('framework') if not is_framework: is_framework = False package_registry_version = pkg.registry.get_version() package_labels = { PACKAGE_METADATA_KEY: encoded_metadata, PACKAGE_NAME_KEY: metadata['name'], PACKAGE_VERSION_KEY: metadata['version'], PACKAGE_SOURCE_KEY: pkg.registry.source.url, PACKAGE_FRAMEWORK_KEY: json.dumps(is_framework), PACKAGE_REGISTRY_VERSION_KEY: package_registry_version, PACKAGE_RELEASE_KEY: revision } if pkg.has_command_definition(revision): command = pkg.command_json(revision, options) package_labels[PACKAGE_COMMAND_KEY] = _base64_encode(command) # Run a heuristic that determines the hint for the framework name framework_name = _find_framework_name(pkg.name(), options) if framework_name: package_labels[PACKAGE_FRAMEWORK_NAME_KEY] = framework_name return package_labels def _find_framework_name(package_name, options): """ :param package_name: the name of the package :type package_name: str :param options: the options object :type options: dict :returns: the name of framework if found; None otherwise :rtype: str """ return options.get(package_name, {}).get('framework-name', None) def _base64_encode(dictionary): """Returns base64(json(dictionary)). :param dictionary: dict to encode :type dictionary: dict :returns: base64 encoding :rtype: str """ json_str = json.dumps(dictionary, sort_keys=True) str_bytes = six.b(json_str) return base64.b64encode(str_bytes).decode('utf-8') def uninstall(package_name, remove_all, app_id, cli, app): """Uninstalls a package. :param package_name: The package to uninstall :type package_name: str :param remove_all: Whether to remove all instances of the named app :type remove_all: boolean :param app_id: App ID of the app instance to uninstall :type app_id: str :param init_client: The program to use to run the app :type init_client: object :rtype: None """ if cli is False and app is False: cli = app = True uninstalled = False if cli: if subcommand.uninstall(package_name): uninstalled = True if app: num_apps = uninstall_app( package_name, remove_all, app_id, marathon.create_client(), mesos.DCOSClient()) if num_apps > 0: uninstalled = True if uninstalled: return None else: msg = 'Package [{}]'.format(package_name) if app_id is not None: msg += " with id [{}]".format(app_id) msg += " is not installed." raise DCOSException(msg) def uninstall_subcommand(distribution_name): """Uninstalls a subcommand. :param distribution_name: the name of the package :type distribution_name: str :returns: True if the subcommand was uninstalled :rtype: bool """ return subcommand.uninstall(distribution_name) def uninstall_app(app_name, remove_all, app_id, init_client, dcos_client): """Uninstalls an app. :param app_name: The app to uninstall :type app_name: str :param remove_all: Whether to remove all instances of the named app :type remove_all: boolean :param app_id: App ID of the app instance to uninstall :type app_id: str :param init_client: The program to use to run the app :type init_client: object :param dcos_client: the DCOS client :type dcos_client: dcos.mesos.DCOSClient :returns: number of apps uninstalled :rtype: int """ apps = init_client.get_apps() def is_match(app): encoding = 'utf-8' # We normalize encoding for byte-wise comparison name_label = app.get('labels', {}).get(PACKAGE_NAME_KEY, u'') name_label_enc = name_label.encode(encoding) app_name_enc = app_name.encode(encoding) name_matches = name_label_enc == app_name_enc if app_id is not None: pkg_app_id = app.get('id', '') normalized_app_id = init_client.normalize_app_id(app_id) return name_matches and pkg_app_id == normalized_app_id else: return name_matches matching_apps = [a for a in apps if is_match(a)] if not remove_all and len(matching_apps) > 1: app_ids = [a.get('id') for a in matching_apps] raise DCOSException( ("Multiple apps named [{}] are installed: [{}].\n" + "Please use --app-id to specify the ID of the app to uninstall," + " or use --all to uninstall all apps.").format( app_name, ', '.join(app_ids))) for app in matching_apps: package_json = _decode_and_add_context( app['id'], app.get('labels', {})) # First, remove the app from Marathon init_client.remove_app(app['id'], force=True) # Second, shutdown the framework with Mesos framework_name = app.get('labels', {}).get(PACKAGE_FRAMEWORK_NAME_KEY) if framework_name is not None: logger.info( 'Trying to shutdown framework {}'.format(framework_name)) frameworks = mesos.Master(dcos_client.get_master_state()) \ .frameworks(inactive=True) # Look up all the framework names framework_ids = [ framework['id'] for framework in frameworks if framework['name'] == framework_name ] logger.info( 'Found the following frameworks: {}'.format(framework_ids)) # Emit post uninstall notes emitter.publish( DefaultError( 'Uninstalled package [{}] version [{}]'.format( package_json['name'], package_json['version']))) if 'postUninstallNotes' in package_json: emitter.publish( DefaultError(package_json['postUninstallNotes'])) if len(framework_ids) == 1: dcos_client.shutdown_framework(framework_ids[0]) elif len(framework_ids) > 1: raise DCOSException( "Unable to shutdown the framework for [{}] because there " "are multiple frameworks with the same name: [{}]. " "Manually shut them down using 'dcos service " "shutdown'.".format( framework_name, ', '.join(framework_ids))) return len(matching_apps) class InstalledPackage(object): """Represents an intalled DCOS package. One of `app` and `subcommand` must be supplied. :param apps: A dictionary representing a marathon app. Of the format returned by `installed_apps()` :type apps: [dict] :param subcommand: Installed subcommand :type subcommand: subcommand.InstalledSubcommand """ def __init__(self, apps=[], subcommand=None): assert apps or subcommand self.apps = apps self.subcommand = subcommand def name(self): """ :returns: The name of the package :rtype: str """ if self.subcommand: return self.subcommand.name else: return self.apps[0]['name'] def dict(self): """ A dictionary representation of the package. Used by `dcos package list`. :returns: A dictionary representation of the package. :rtype: dict """ ret = {} if self.subcommand: ret['command'] = {'name': self.subcommand.name} if self.apps: ret['apps'] = [app['appId'] for app in self.apps] if self.subcommand: package_json = self.subcommand.package_json() ret.update(package_json) ret['packageSource'] = self.subcommand.package_source() ret['releaseVersion'] = self.subcommand.package_revision() else: ret.update(self.apps[0]) ret.pop('appId') return ret def installed_packages(init_client, endpoints): """Returns all installed packages in the format: [{ 'apps': [], 'command': { 'name': } ...... }] :param init_client: The program to use to list packages :type init_client: object :param endpoints: Whether to include a list of endpoints as port-host pairs :type endpoints: boolean :returns: A list of installed packages :rtype: [InstalledPackage] """ apps = installed_apps(init_client, endpoints) subcommands = installed_subcommands() dicts = collections.defaultdict(lambda: {'apps': [], 'command': None}) for app in apps: key = (app['name'], app['releaseVersion'], app['packageSource']) dicts[key]['apps'].append(app) for subcmd in subcommands: package_revision = subcmd.package_revision() package_source = subcmd.package_source() key = (subcmd.name, package_revision, package_source) dicts[key]['command'] = subcmd return [ InstalledPackage(pkg['apps'], pkg['command']) for pkg in dicts.values() ] def installed_subcommands(): """Returns all installed subcommands. :returns: all installed subcommands :rtype: [InstalledSubcommand] """ return [subcommand.InstalledSubcommand(name) for name in subcommand.distributions()] def installed_apps(init_client, endpoints=False): """ Returns all installed apps. An app is of the format: { 'appId': , 'packageSource': , 'registryVersion': , 'releaseVersion': 'endpoints' (optional): [{ 'host': , 'ports': , }] .... } :param init_client: The program to use to list packages :type init_client: object :param endpoints: Whether to include a list of endpoints as port-host pairs :type endpoints: boolean :returns: all installed apps :rtype: [dict] """ apps = init_client.get_apps() encoded_apps = [(a['id'], a['labels']) for a in apps if a.get('labels', {}).get(PACKAGE_METADATA_KEY)] # Filter elements that failed to parse correctly as JSON valid_apps = [] for app_id, labels in encoded_apps: try: decoded = _decode_and_add_context(app_id, labels) except Exception: logger.exception( 'Unable to decode package metadata during install: %s', app_id) valid_apps.append(decoded) if endpoints: for app in valid_apps: tasks = init_client.get_tasks(app["appId"]) app['endpoints'] = [{"host": t["host"], "ports": t["ports"]} for t in tasks] return valid_apps def _decode_and_add_context(app_id, labels): """ Create an enhanced package JSON from Marathon labels { 'appId': , 'packageSource': , 'registryVersion': , 'releaseVersion': , .... } :param app_id: Marathon application id :type app_id: str :param labels: Marathon label dictionary :type labels: dict :rtype: dict """ encoded = labels.get(PACKAGE_METADATA_KEY, {}) decoded = base64.b64decode(six.b(encoded)).decode() decoded_json = util.load_jsons(decoded) decoded_json['appId'] = app_id decoded_json['packageSource'] = labels.get(PACKAGE_SOURCE_KEY) decoded_json['releaseVersion'] = labels.get(PACKAGE_RELEASE_KEY) return decoded_json def search(query, cfg): """Returns a list of index entry collections, one for each registry in the supplied config. :param query: The search term :type query: str :param cfg: Configuration dictionary :type cfg: dcos.config.Toml :rtype: [IndexEntries] """ threshold = 0.5 # Minimum rank required to appear in results results = [] def clean_package_entry(entry): result = entry.copy() result.update({ 'versions': list(entry['versions'].keys()) }) return result for registry in registries(cfg): source_results = [] index = registry.get_index() for pkg in index['packages']: rank = _search_rank(pkg, query) if rank >= threshold: source_results.append(clean_package_entry(pkg)) entries = IndexEntries(registry.source, source_results) results.append(entries) return results def _search_rank(pkg, query): """ :param pkg: Index entry to rank for affinity with the search term :type pkg: object :param query: Search term :type query: str :rtype: float """ result = 0.0 wildcard_symbol = '*' regex_pattern = '.*' q = query.lower() if wildcard_symbol in q: q = q.replace(wildcard_symbol, regex_pattern) if q.endswith(wildcard_symbol): q = '^{}'.format(q) else: q = '{}$'.format(q) if re.match(q, pkg['name'].lower()): result += 2.0 return result if q in pkg['name'].lower(): result += 2.0 for tag in pkg['tags']: if q in tag.lower(): result += 1.0 if q in pkg['description'].lower(): result += 0.5 return result def _extract_default_values(config_schema): """ :param config_schema: A json-schema describing configuration options. :type config_schema: dict :returns: a dictionary with the default specified by the schema :rtype: dict | None """ defaults = {} if 'properties' not in config_schema: return None for key, value in config_schema['properties'].items(): if isinstance(value, dict) and 'default' in value: defaults[key] = value['default'] elif isinstance(value, dict) and value.get('type', '') == 'object': # Generate the default value from the embedded schema defaults[key] = _extract_default_values(value) return defaults def _merge_options(first, second, overrides=True): """Merges the :code:`second` dictionary into the :code:`first` dictionary. If both dictionaries have the same key and both values are dictionaries then it recursively merges those two dictionaries. :param first: first dictionary :type first: dict :param second: second dictionary :type second: dict :param overrides: allow second to override first if both have same key :type overrides: bool :returns: merged dictionary :rtype: dict """ result = copy.deepcopy(first) for key, second_value in second.items(): if key in first: first_value = first[key] if (isinstance(first_value, collections.Mapping) and isinstance(second_value, collections.Mapping)): result[key] = _merge_options(first_value, second_value) elif not overrides and first_value != second_value: raise DCOSException( "Trying to override package.json's key {} to {}".format( key, second_value)) else: result[key] = second_value else: result[key] = second_value return result def resolve_package(package_name, config=None): """Returns the first package with the supplied name found by looking at the configured sources in the order they are defined. :param package_name: The name of the package to resolve :type package_name: str :param config: dcos config :type config: dcos.config.Toml | None :returns: The named package, if found :rtype: Package """ if not config: config = util.get_config() for registry in registries(config): package = registry.get_package(package_name) if package: return package return None def registries(config): """Returns configured cached package registries. :param config: Configuration dictionary :type config: dcos.config.Toml :returns: The list of registries, in resolution order :rtype: [Registry] """ sources = list_sources(config) return [Registry(source, source.local_cache(config)) for source in sources] def list_sources(config): """List configured package sources. :param config: Configuration dictionary :type config: dcos.config.Toml :returns: The list of sources, in resolution order :rtype: [Source] """ source_uris = util.get_config_vals(['package.sources'], config)[0] sources = [url_to_source(s) for s in source_uris] errs = [source for source in sources if isinstance(source, Error)] if errs: raise DCOSException('\n'.join(err.error() for err in errs)) return sources def url_to_source(url): """Creates a package source from the supplied URL. :param url: Location of the package source :type url: str :returns: A Source backed by the supplied URL :rtype: Source | Error """ parse_result = urllib.parse.urlparse(url) scheme = parse_result.scheme if scheme == 'file': return FileSource(url) elif scheme == 'http' or scheme == 'https': return HttpSource(url) elif scheme == 'git': return GitSource(url) else: return Error("Source URL uses unsupported protocol [{}]".format(url)) def _acquire_file_lock(lock_file_path): """Acquires an exclusive lock on the supplied file. :param lock_file_path: Path to the lock file :type lock_file_path: str :returns: Lock file :rtype: File """ try: lock_file = open(lock_file_path, 'w') except IOError as e: logger.exception('Failed to open lock file: %s', lock_file_path) raise util.io_exception(lock_file_path, e.errno) acquire_mode = portalocker.LOCK_EX | portalocker.LOCK_NB try: portalocker.lock(lock_file, acquire_mode) return lock_file except portalocker.LockException: logger.exception( 'Failure while tring to aquire file lock: %s', lock_file_path) lock_file.close() raise DCOSException('Unable to acquire the package cache lock') def update_sources(config, validate=False): """Overwrites the local package cache with the latest source data. :param config: Configuration dictionary :type config: dcos.config.Toml :rtype: None """ errors = [] # ensure the cache directory is properly configured cache_dir = os.path.expanduser( util.get_config_vals(['package.cache'], config)[0]) # ensure the cache directory exists if not os.path.exists(cache_dir): os.makedirs(cache_dir) if not os.path.isdir(cache_dir): raise DCOSException( 'Cache directory does not exist! [{}]'.format(cache_dir)) # obtain an exclusive file lock on $CACHE/.lock lock_path = os.path.join(cache_dir, '.lock') with _acquire_file_lock(lock_path): # list sources sources = list_sources(config) for source in sources: emitter.publish('Updating source [{}]'.format(source)) # create a temporary staging directory with util.tempdir() as tmp_dir: stage_dir = os.path.join(tmp_dir, source.hash()) # copy to the staging directory try: source.copy_to_cache(stage_dir) except DCOSException as e: logger.exception( 'Failed to copy universe source %s to cache %s', source.url, stage_dir) errors.append(str(e)) continue # check version # TODO(jsancio): move this to the validation when it is forced Registry(source, stage_dir).check_version( LooseVersion('1.0'), LooseVersion('3.0')) # validate content if validate: validation_errors = Registry(source, stage_dir).validate() if len(validation_errors) > 0: errors += validation_errors continue # keep updating the other sources # remove the $CACHE/source.hash() directory target_dir = os.path.join(cache_dir, source.hash()) try: if os.path.exists(target_dir): shutil.rmtree(target_dir, onerror=_rmtree_on_error, ignore_errors=False) except OSError: logger.exception( 'Error removing target directory before move: %s', target_dir) err = "Could not remove directory [{}]".format(target_dir) errors.append(err) continue # keep updating the other sources # move the staging directory to $CACHE/source.hash() shutil.move(stage_dir, target_dir) if errors: raise DCOSException(util.list_to_err(errors)) class Source: """A source of DCOS packages.""" @property @abc.abstractmethod def url(self): """ :returns: Location of the package source :rtype: str """ raise NotImplementedError def hash(self): """Returns a cryptographically secure hash derived from this source. :returns: a hexadecimal string :rtype: str """ return hashlib.sha1(self.url.encode('utf-8')).hexdigest() def local_cache(self, config): """Returns the file system path to this source's local cache. :param config: Configuration dictionary :type config: dcos.config.Toml :returns: Path to this source's local cache on disk :rtype: str or None """ cache_dir = os.path.expanduser( util.get_config_vals(['package.cache'], config)[0]) return os.path.join(cache_dir, self.hash()) def copy_to_cache(self, target_dir): """Copies the source content to the supplied local directory. :param target_dir: Path to the destination directory. :type target_dir: str :rtype: None """ raise NotImplementedError def __repr__(self): return self.url class FileSource(Source): """A registry of DCOS packages. :param url: Location of the package source :type url: str """ def __init__(self, url): self._url = url @property def url(self): """ :returns: Location of the package source :rtype: str """ return self._url def copy_to_cache(self, target_dir): """Copies the source content to the supplied local directory. :param target_dir: Path to the destination directory. :type target_dir: str :rtype: None """ # copy the source to the target_directory parse_result = urllib.parse.urlparse(self._url) source_dir = parse_result.path try: shutil.copytree(source_dir, target_dir) return None except OSError: logger.exception( 'Error copying source director [%s] to target directory [%s].', source_dir, target_dir) raise DCOSException( 'Unable to fetch packages from [{}]'.format(self.url)) class HttpSource(Source): """A registry of DCOS packages. :param url: Location of the package source :type url: str """ def __init__(self, url): self._url = url @property def url(self): """ :returns: Location of the package source :rtype: str """ return self._url def copy_to_cache(self, target_dir): """Copies the source content to the supplied local directory. :param target_dir: Path to the destination directory. :type target_dir: str :returns: The error, if one occurred :rtype: None """ try: with util.tempdir() as tmp_dir: tmp_file = os.path.join(tmp_dir, 'packages.zip') # Download the zip file. req = http.get(self.url) if req.status_code == 200: with open(tmp_file, 'wb') as f: for chunk in req.iter_content(1024): f.write(chunk) else: raise Exception( 'HTTP GET for {} did not return 200: {}'.format( self.url, req.status_code)) # Unzip the downloaded file. packages_zip = zipfile.ZipFile(tmp_file, 'r') packages_zip.extractall(tmp_dir) # Move the enclosing directory to the target directory enclosing_dirs = [item for item in os.listdir(tmp_dir) if os.path.isdir( os.path.join(tmp_dir, item))] # There should only be one directory present after extracting. assert(len(enclosing_dirs) is 1) enclosing_dir = os.path.join(tmp_dir, enclosing_dirs[0]) shutil.copytree(enclosing_dir, target_dir) # Set appropriate file permissions on the scripts. x_mode = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP) scripts_dir = os.path.join(target_dir, 'scripts') scripts = os.listdir(scripts_dir) for script in scripts: script_path = os.path.join(scripts_dir, script) if os.path.isfile(script_path): os.chmod(script_path, x_mode) return None except Exception: logger.exception('Unable to fetch packages from URL: %s', self.url) raise DCOSException( 'Unable to fetch packages from [{}]'.format(self.url)) class GitSource(Source): """A registry of DCOS packages. :param url: Location of the package source :type url: str """ def __init__(self, url): self._url = url @property def url(self): """ :returns: Location of the package source :rtype: str """ return self._url def copy_to_cache(self, target_dir): """Copies the source content to the supplied local directory. :param target_dir: Path to the destination directory. :type target_dir: str :returns: The error, if one occurred :rtype: None """ try: # TODO(SS): add better url parsing # Ensure git is installed properly. git_program = util.which('git') if git_program is None: raise DCOSException("""Could not locate the git program. Make sure \ it is installed and on the system search path. PATH = {}""".format(os.environ[constants.PATH_ENV])) # Clone git repo into the supplied target directory. git.Repo.clone_from(self._url, to_path=target_dir, progress=None, branch='master') # Remove .git directory to save space. shutil.rmtree(os.path.join(target_dir, ".git"), onerror=_rmtree_on_error) return None except git.exc.GitCommandError: logger.exception('Unable to fetch packages from git: %s', self.url) raise DCOSException( 'Unable to fetch packages from [{}]'.format(self.url)) def _rmtree_on_error(func, path, exc_info): """Error handler for ``shutil.rmtree``. If the error is due to an access error (read only file) it attempts to add write permission and then retries. If the error is for another reason it re-raises the error. Usage : ``shutil.rmtree(path, onerror=onerror)``. :param func: Function which raised the exception. :type func: function :param path: The path name passed to ``shutil.rmtree`` function. :type path: str :param exc_info: Information about the last raised exception. :type exc_info: tuple :rtype: None """ import stat if not os.access(path, os.W_OK): os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) func(path) else: raise class Error(errors.Error): """Class for describing errors during packaging operations. :param message: Error message :type message: str """ def __init__(self, message): self._message = message def error(self): """Return error message :returns: The error message :rtype: str """ return self._message class Registry(): """Represents a package registry on disk. :param base_path: Path to the registry :type base_path: str :param source: The associated package source :type source: Source """ def __init__(self, source, base_path): self._base_path = base_path self._source = source def validate(self): """Validates a package registry. :returns: Validation errors :rtype: [str] """ # TODO(CD): implement these checks in pure Python? scripts_dir = os.path.join(self._base_path, 'scripts') if util.is_windows_platform(): validate_script = os.path.join(scripts_dir, '1-validate-packages.ps1') cmd = ['powershell', '-ExecutionPolicy', 'ByPass', '-File', validate_script] result = subprocess.call(cmd) else: validate_script = os.path.join(scripts_dir, '1-validate-packages.sh') result = subprocess.call(validate_script) if result is not 0: return ["Source tree is not valid [{}]".format(self._base_path)] else: return [] @property def source(self): """Returns the associated upstream package source for this registry. :rtype: Source """ return self._source def check_version(self, min_version, max_version): """Checks that the version is [min_version, max_version) :param min_version: the min version inclusive :type min_version: LooseVersion :param max_version: the max version exclusive :type max_version: LooseVersion :returns: None """ version = LooseVersion(self.get_version()) if not (version >= min_version and version < max_version): raise DCOSException(( 'Unable to update source [{}] because version {} is ' 'not supported. Supported versions are between {} and ' '{}. Please update your DCOS CLI.').format( self._source.url, version, min_version, max_version)) def get_version(self): """Returns the version of this registry. :rtype: str """ # The package version is found in $BASE/repo/meta/version.json index_path = os.path.join( self._base_path, 'repo', 'meta', 'version.json') if not os.path.isfile(index_path): raise DCOSException('Path [{}] is not a file'.format(index_path)) try: with util.open_file(index_path) as fd: version_json = json.load(fd) return version_json.get('version') except ValueError: logger.exception('Unable to parse JSON: %s', index_path) raise DCOSException('Unable to parse [{}]'.format(index_path)) def get_index(self): """Returns the index of packages in this registry. :rtype: dict """ # The package index is found in $BASE/repo/meta/index.json index_path = os.path.join( self._base_path, 'repo', 'meta', 'index.json') if not os.path.isfile(index_path): raise DCOSException('Path [{}] is not a file'.format(index_path)) try: with util.open_file(index_path) as fd: return json.load(fd) except ValueError: logger.exception('Unable to parse JSON: %s', index_path) raise DCOSException('Unable to parse [{}]'.format(index_path)) def get_package(self, package_name): """Returns the named package, if it exists. :param package_name: The name of the package to fetch :type package_name: str :returns: The requested package :rtype: Package """ if len(package_name) is 0: raise DCOSException('Package name must not be empty.') # Packages are found in $BASE/repo/package// first_character = package_name[0].title() package_path = os.path.join( self._base_path, 'repo', 'packages', first_character, package_name) if not os.path.isdir(package_path): return None try: return Package(self, package_path) except: logger.exception('Unable to read package: %s', package_path) raise DCOSException( 'Could not read package [{}]'.format(package_name)) class Package(): """Interface to a package on disk. :param registry: The containing registry for this package. :type registry: Registry :param path: Path to the package description on disk :type path: str """ def __init__(self, registry, path): assert os.path.isdir(path) self._registry = registry self.path = path def name(self): """Returns the package name. :returns: The name of this package :rtype: str """ return os.path.basename(self.path) def options(self, revision, user_options): """Merges package options with user supplied options, validates, and returns the result. :param revision: the package revision to install :type revision: str :param user_options: package parameters :type user_options: dict :returns: a dictionary with the user supplied options :rtype: dict """ if user_options is None: user_options = {} config_schema = self.config_json(revision) default_options = _extract_default_values(config_schema) if default_options is None: pkg = self.package_json(revision) msg = ("An object in the package's config.json is missing the " "required 'properties' feature:\n {}".format(config_schema)) if 'maintainer' in pkg: msg += "\nPlease contact the project maintainer: {}".format( pkg['maintainer']) raise DCOSException(msg) logger.info('Generated default options: %r', default_options) # Merge option overrides, second argument takes precedence options = _merge_options(default_options, user_options) logger.info('Merged options: %r', options) # Validate options with the config schema errs = util.validate_json(options, config_schema) if len(errs) != 0: raise DCOSException( "{}\n\n{}".format( util.list_to_err(errs), 'Please create a JSON file with the appropriate options, ' 'and pass the /path/to/file as an --options argument.')) return options @property def registry(self): """Returns the containing registry for this package. :rtype: Registry """ return self._registry def has_definition(self, revision, filename): """Returns true if the package defines filename; false otherwise. :param revision: package revision :type revision: str :param filename: file in package definition :type filename: str :returns: whether filename is defined :rtype: bool """ return os.path.isfile( os.path.join( self.path, os.path.join(revision, filename))) def has_command_definition(self, revision): """Returns true if the package defines a command; false otherwise. :param revision: package revision :type revision: str :rtype: bool """ return self.has_definition(revision, 'command.json') def _has_resource_definition(self, revision): """Returns true if the package defines a resource; false otherwise. :param revision: package revision :type revision: str :rtype: bool """ return self.has_definition(revision, 'resource.json') def has_marathon_definition(self, revision): """Returns true if the package defines a Marathon json. false otherwise. :param revision: package revision :type revision: str :rtype: bool """ return self.has_definition(revision, 'marathon.json') def has_marathon_mustache_definition(self, revision): """Returns true if the package defines a Marathon.json.mustache false otherwise. :param revision: package revision :type revision: str :rtype: bool """ return self.has_definition(revision, 'marathon.json.mustache') def _get_marathon_json_file(self, revision): """Returns the file name of Marathon json :param revision: package revision :type revision: str :returns: Marathon file name :rtype: str """ if self.has_marathon_definition(revision): return 'marathon.json' elif self.has_marathon_mustache_definition(revision): return 'marathon.json.mustache' else: raise DCOSException("Missing Marathon json definition of package") def config_json(self, revision): """Returns the JSON content of the config.json file. :param revision: package revision :type revision: str :returns: Package config schema :rtype: dict """ return self._json(revision, 'config.json') def package_json(self, revision): """Returns the JSON content of the package.json file. :param revision: the package revision :type revision: str :returns: Package data :rtype: dict """ return self._json(revision, 'package.json') def _resource_json(self, revision): """Returns the JSON content of the resource.json file. :param revision: the package revision :type revision: str :returns: Package data :rtype: dict """ return self._json(revision, 'resource.json') def marathon_json(self, revision, options): """Returns the JSON content of the marathon.json template, after rendering it with options. :param revision: the package revision :type revision: str :param options: the template options to use in rendering :type options: dict :rtype: dict """ marathon_file = self._get_marathon_json_file(revision) if self.has_marathon_mustache_definition(revision) and \ self._has_resource_definition(revision): resources = {"resource": self._resource_json(revision)} options = _merge_options(options, resources, False) init_desc = self._render_template( marathon_file, revision, options) # Add package metadata package_labels = _make_package_labels(self, revision, options) # Preserve existing labels labels = init_desc.get('labels', {}) labels.update(package_labels) init_desc['labels'] = labels return init_desc def command_json(self, revision, options): """Returns the JSON content of the comand.json template, after rendering it with options. :param revision: the package revision :type revision: str :param options: the template options to use in rendering :type options: dict :returns: Package data :rtype: dict """ template = self._data(revision, 'command.json') rendered = pystache.render(template, options) return json.loads(rendered) def marathon_template(self, revision): """ Returns raw data from marathon.json :param revision: the package revision :type revision: str :returns: raw data from marathon.json :rtype: str """ return self._data(revision, self._get_marathon_json_file(revision)) def command_template(self, revision): """ Returns raw data from command.json :param revision: the package revision :type revision: str :returns: raw data from command.json :rtype: str """ return self._data(revision, 'command.json') def _render_template(self, name, revision, options): """Render a template. :param name: the file name of the template :type name: str :param revision: the package revision :type revision: str :param options: the template options to use in rendering :type options: dict :rtype: dict """ template = self._data(revision, name) return util.render_mustache_json(template, options) def _json(self, revision, name): """Returns the json content of the file named `name` in the directory named `revision` :param revision: the package revision :type revision: str :param name: file name :type name: str :rtype: dict """ data = self._data(revision, name) return util.load_jsons(data) def _data(self, revision, name): """Returns the content of the file named `name` in the directory named `revision` :param revision: the package revision :type revision: str :param name: file name :type name: str :returns: File content of the supplied path :rtype: str """ path = os.path.join(revision, name) full_path = os.path.join(self.path, path) return util.read_file(full_path) def package_revisions(self): """Returns all of the available package revisions, most recent first. :returns: Available revisions of this package :rtype: [str] """ vs = sorted((f for f in os.listdir(self.path) if not f.startswith('.')), key=int, reverse=True) return vs def package_revisions_map(self): """Returns an ordered mapping from the package revision to the package version, sorted by package revision. :returns: Map from package revision to package version :rtype: OrderedDict """ package_version_map = collections.OrderedDict() for rev in self.package_revisions(): pkg_json = self.package_json(rev) package_version_map[rev] = pkg_json['version'] return package_version_map def latest_package_revision(self, package_version=None): """Returns the most recent package revision, for a given package version if specified. :param package_version: a given package version :type package_version: str :returns: package revision :rtype: str | None """ if package_version: pkg_rev_map = self.package_revisions_map() # depends on package_revisions() returning an OrderedDict if package_version in pkg_rev_map.values(): return next(pkg_rev for pkg_rev in reversed(pkg_rev_map) if pkg_rev_map[pkg_rev] == package_version) else: return None else: pkg_revisions = self.package_revisions() revision = pkg_revisions[0] return revision def __repr__(self): rev = self.latest_package_revision() pkg_json = self.package_json(rev) return json.dumps(pkg_json) class IndexEntries(): """A collection of package index entries from a single source. Each entry is a dict as described by the JSON schema for the package index: https://github.com/mesosphere/universe/blob/master/repo/meta/schema/index-schema.json :param source: The source of these index entries :type source: Source :param packages: The index entries :type packages: [dict] """ def __init__(self, source, packages): self._source = source self._packages = packages @property def source(self): """Returns the source of these index entries. :rtype: Source """ return self._source @property def packages(self): """Returns the package index entries. :rtype: list of dict """ return self._packages def as_dict(self): """ :rtype: dict """ return {'source': self.source.url, 'packages': self.packages} def get_apps_for_framework(framework_name, client): """ Return all apps running the given framework. :param framework_name: framework name :type framework_name: str :param client: marathon client :type client: marathon.Client :rtype: [dict] """ return [app for app in client.get_apps() if app.get('labels', {}).get( PACKAGE_FRAMEWORK_NAME_KEY) == framework_name] PKցFH4F00dcos/subcommand.pyfrom __future__ import print_function import json import os import shutil import subprocess import tempfile from dcos import constants, util from dcos.errors import DCOSException logger = util.get_logger(__name__) def command_executables(subcommand): """List the real path to executable dcos program for specified subcommand. :param subcommand: name of subcommand. E.g. marathon :type subcommand: str :returns: the dcos program path :rtype: str """ executables = [ command_path for command_path in list_paths() if noun(command_path) == subcommand ] if len(executables) > 1: msg = 'Found more than one executable for command {!r}.' raise DCOSException(msg.format(subcommand)) if len(executables) == 0: msg = "{!r} is not a dcos command." raise DCOSException(msg.format(subcommand)) return executables[0] def get_package_commands(package_name): """List the real path(s) to executables for a specific dcos subcommand :param package_name: package name :type package_name: str :returns: list of all the dcos program paths in package :rtype: [str] """ bin_dir = os.path.join(package_dir(package_name), constants.DCOS_SUBCOMMAND_VIRTUALENV_SUBDIR, BIN_DIRECTORY) executables = [] for filename in os.listdir(bin_dir): path = os.path.join(bin_dir, filename) if (filename.startswith(constants.DCOS_COMMAND_PREFIX) and _is_executable(path)): executables.append(path) return executables def list_paths(): """List the real path to executable dcos subcommand programs. :returns: list of all the dcos program paths :rtype: [str] """ # Let's get all the default subcommands binpath = util.dcos_bin_path() commands = [ os.path.join(binpath, filename) for filename in os.listdir(binpath) if (filename.startswith(constants.DCOS_COMMAND_PREFIX) and _is_executable(os.path.join(binpath, filename))) ] subcommands = [] for package in distributions(): subcommands += get_package_commands(package) return commands + subcommands def _is_executable(path): """ :param path: the path to a program :type path: str :returns: True if the path is an executable; False otherwise :rtype: bool """ return os.access(path, os.X_OK) and ( not util.is_windows_platform() or path.endswith('.exe')) def distributions(): """List all of the installed subcommand packages :returns: a list of packages :rtype: list of str """ subcommand_dir = _subcommand_dir() if os.path.isdir(subcommand_dir): return [ subdir for subdir in os.listdir(subcommand_dir) if os.path.isdir( os.path.join( subcommand_dir, subdir, constants.DCOS_SUBCOMMAND_VIRTUALENV_SUBDIR)) ] else: return [] def documentation(executable_path): """Gather subcommand summary :param executable_path: real path to the dcos subcommands :type executable_path: str :returns: subcommand and its summary :rtype: (str, str) """ path_noun = noun(executable_path) return (path_noun, info(executable_path, path_noun)) def info(executable_path, path_noun): """Collects subcommand information :param executable_path: real path to the dcos subcommand :type executable_path: str :param path_noun: subcommand :type path_noun: str :returns: the subcommand information :rtype: str """ out = subprocess.check_output( [executable_path, path_noun, '--info']) return out.decode('utf-8').strip() def config_schema(executable_path): """Collects subcommand config schema :param executable_path: real path to the dcos subcommand :type executable_path: str :returns: the subcommand config schema :rtype: dict """ out = subprocess.check_output( [executable_path, noun(executable_path), '--config-schema']) return json.loads(out.decode('utf-8')) def noun(executable_path): """Extracts the subcommand single noun from the path to the executable. E.g for :code:`bin/dcos-subcommand` this method returns :code:`subcommand`. :param executable_path: real pth to the dcos subcommand :type executable_path: str :returns: the subcommand :rtype: str """ basename = os.path.basename(executable_path) noun = basename[len(constants.DCOS_COMMAND_PREFIX):].replace('.exe', '') return noun def _write_package_json(pkg, revision): """ Write package.json locally. :param pkg: the package being installed :type pkg: Package :param revision: the package revision to install :type revision: str :rtype: None """ pkg_dir = package_dir(pkg.name()) package_path = os.path.join(pkg_dir, 'package.json') package_json = pkg.package_json(revision) with util.open_file(package_path, 'w') as package_file: json.dump(package_json, package_file) def _write_package_revision(pkg, revision): """ Write package revision locally. :param pkg: the package being installed :type pkg: Package :param revision: the package revision to install :type revision: str :rtype: None """ pkg_dir = package_dir(pkg.name()) revision_path = os.path.join(pkg_dir, 'version') with util.open_file(revision_path, 'w') as revision_file: revision_file.write(revision) def _write_package_source(pkg): """ Write package source locally. :param pkg: the package being installed :type pkg: Package :rtype: None """ pkg_dir = package_dir(pkg.name()) source_path = os.path.join(pkg_dir, 'source') with util.open_file(source_path, 'w') as source_file: source_file.write(pkg.registry.source.url) def _install_env(pkg, revision, options): """ Install subcommand virtual env. :param pkg: the package to install :type pkg: Package :param revision: the package revision to install :type revision: str :param options: package parameters :type options: dict :rtype: None """ pkg_dir = package_dir(pkg.name()) install_operation = pkg.command_json(revision, options) env_dir = os.path.join(pkg_dir, constants.DCOS_SUBCOMMAND_VIRTUALENV_SUBDIR) if 'pip' in install_operation: _install_with_pip( pkg.name(), env_dir, install_operation['pip']) else: raise DCOSException("Installation methods '{}' not supported".format( install_operation.keys())) def install(pkg, revision, options): """Installs the dcos cli subcommand :param pkg: the package to install :type pkg: Package :param revision: the package revision to install :type revision: str :param options: package parameters :type options: dict :rtype: None """ pkg_dir = package_dir(pkg.name()) util.ensure_dir_exists(pkg_dir) _write_package_json(pkg, revision) _write_package_revision(pkg, revision) _write_package_source(pkg) _install_env(pkg, revision, options) def _subcommand_dir(): """ Returns ~/.dcos/subcommands """ return os.path.expanduser(os.path.join("~", constants.DCOS_DIR, constants.DCOS_SUBCOMMAND_SUBDIR)) # TODO(mgummelt): should be made private after "dcos subcommand" is removed def package_dir(name): """ Returns ~/.dcos/subcommands/ :param name: package name :type name: str :rtype: str """ return os.path.join(_subcommand_dir(), name) def uninstall(package_name): """Uninstall the dcos cli subcommand :param package_name: the name of the package :type package_name: str :returns: True if the subcommand was uninstalled :rtype: bool """ pkg_dir = package_dir(package_name) if os.path.isdir(pkg_dir): shutil.rmtree(pkg_dir) return True return False BIN_DIRECTORY = 'Scripts' if util.is_windows_platform() else 'bin' def _find_virtualenv(bin_directory): """ :param bin_directory: directory to first use to find virtualenv :type bin_directory: str :returns: Absolute path to virutalenv program :rtype: str """ virtualenv_path = os.path.join(bin_directory, 'virtualenv') if not os.path.exists(virtualenv_path): virtualenv_path = util.which('virtualenv') if virtualenv_path is None: raise DCOSException('Unable to find the virtualenv program') return virtualenv_path def _install_with_pip( package_name, env_directory, requirements): """ :param package_name: the name of the package :type package_name: str :param env_directory: the path to the directory in which to install the package's virtual env :type env_directory: str :param requirements: the list of pip requirements :type requirements: list of str :rtype: None """ bin_directory = util.dcos_bin_path() new_package_dir = not os.path.exists(env_directory) pip_path = os.path.join(env_directory, BIN_DIRECTORY, 'pip') if not os.path.exists(pip_path): cmd = [_find_virtualenv(bin_directory), env_directory] if _execute_install(cmd) != 0: raise _generic_error(package_name) with tempfile.NamedTemporaryFile() as temp_file: # Write the requirements to the file for line in requirements: temp_file.write((line + os.linesep).encode('utf-8')) # Make sure that we flush the file before passing it to pip temp_file.flush() cmd = [ os.path.join(env_directory, BIN_DIRECTORY, 'pip'), 'install', '--requirement', temp_file.name, ] if _execute_install(cmd) != 0: # We should remove the directory that we just created if new_package_dir: shutil.rmtree(env_directory) raise _generic_error(package_name) return None def _execute_install(command): """ :param command: the install command to execute :type command: list of str :returns: the process return code :rtype: int """ logger.info('Calling: %r', command) process = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.returncode != 0: logger.error("Install script's stdout: %s", stdout) logger.error("Install script's stderr: %s", stderr) else: logger.info("Install script's stdout: %s", stdout) logger.info("Install script's stderr: %s", stderr) return process.returncode def _generic_error(package_name): """ :param package: package name :type: str :returns: generic error when installing package :rtype: DCOSException """ return DCOSException( ('Error installing {!r} package.\n' 'Run with `dcos --log-level=ERROR` to see the full output.').format( package_name)) class InstalledSubcommand(object): """ Represents an installed subcommand. :param name: The name of the subcommand :type name: str """ def __init__(self, name): self.name = name def _dir(self): """ :returns: path to this subcommand's directory. :rtype: str """ return package_dir(self.name) def package_revision(self): """ :returns: this subcommand's revision. :rtype: str """ revision_path = os.path.join(self._dir(), 'version') return util.read_file(revision_path) def package_source(self): """ :returns: this subcommand's source. :rtype: str """ source_path = os.path.join(self._dir(), 'source') return util.read_file(source_path) def package_json(self): """ :returns: contents of this subcommand's package.json file. :rtype: dict """ package_json_path = os.path.join(self._dir(), 'package.json') with util.open_file(package_json_path) as package_json_file: return util.load_json(package_json_file) PKցFHPLldcos/constants.pyDCOS_DIR = ".dcos" """DCOS data directory. Can store subcommands and the config file.""" DCOS_SUBCOMMAND_VIRTUALENV_SUBDIR = 'env' """In a package's directory, this is the virtualenv subdirectory.""" DCOS_SUBCOMMAND_SUBDIR = 'subcommands' """Name of the subdirectory that contains all of the subcommands. This is relative to the location of the executable.""" DCOS_CONFIG_ENV = 'DCOS_CONFIG' """Name of the environment variable pointing to the DCOS config.""" DCOS_LOG_LEVEL_ENV = 'DCOS_LOG_LEVEL' """Name of the environment variable for the DCOS log level""" DCOS_DEBUG_ENV = 'DCOS_DEBUG' """Name of the environment variable to enable DCOS debug messages""" DCOS_PAGER_COMMAND_ENV = 'PAGER' """Command to use to page long command output (e.g. 'less -R')""" DCOS_SSL_VERIFY_ENV = 'DCOS_SSL_VERIFY' """Whether or not ot verify SSL certs for HTTPS or path to certificate(s)""" PATH_ENV = 'PATH' """Name of the environment variable pointing to the executable directories.""" DCOS_COMMAND_PREFIX = 'dcos-' """Prefix for all the DCOS CLI commands.""" VALID_LOG_LEVEL_VALUES = ['debug', 'info', 'warning', 'error', 'critical'] """List of all the supported log level values for the CLIs""" PKցFH Z`` dcos/cmds.pyimport collections from dcos.errors import DCOSException Command = collections.namedtuple( 'Command', ['hierarchy', 'arg_keys', 'function']) """Describe a CLI command. :param hierarchy: the noun and verbs that need to be set for the command to execute :type hierarchy: list of str :param arg_keys: the arguments that must get passed to the function; the order of the keys determines the order in which they get passed to the function :type arg_keys: list of str :param function: the function to execute :type function: func(args) -> int """ def execute(cmds, args): """Executes one of the commands based on the arguments passed. :param cmds: commands to try to execute; the order determines the order of evaluation :type cmds: list of Command :param args: command line arguments :type args: dict :returns: the process status :rtype: int """ for hierarchy, arg_keys, function in cmds: # Let's find if the function matches the command match = True for positional in hierarchy: if not args[positional]: match = False if match: params = [args[name] for name in arg_keys] return function(*params) raise DCOSException('Could not find a command with the passed arguments') PKցFH$$dcos/errors.pyimport abc class DCOSException(Exception): pass class DCOSHTTPException(DCOSException): """ A wrapper around Response objects for HTTP error codes. :param response: requests Response object :type response: Response """ def __init__(self, response): self.response = response def __str__(self): return 'Error while fetching [{0}]: HTTP {1}: {2}'.format( self.response.request.url, self.response.status_code, self.response.reason) class Error(object): """Abstract class for describing errors.""" @abc.abstractmethod def error(self): """Creates an error message :returns: The error message :rtype: str """ raise NotImplementedError class DefaultError(Error): """Construct a basic Error class based on a string :param message: String to use for the error message :type message: str """ def __init__(self, message): self._message = message def error(self): return self._message PKցFHtdcos/options.pydef make_command_summary_string(command_summaries): """Construct subcommand summaries :param command_summaries: Commands and their summaries :type command_summaries: list of (str, str) :returns: The subcommand summaries :rtype: str """ doc = '' for command, summary in command_summaries: doc += '\n\t{:15}\t{}'.format(command, summary.strip()) return doc def make_generic_usage_message(doc): """Construct generic usage error :param doc: Usage documentation for program :type doc: str :returns: Generic usage error :rtype: str """ return 'Unknown option\n{}'.format(doc) PKցFHBB dcos/util.pyimport collections import contextlib import functools import json import logging import os import platform import re import shutil import sys import tempfile import time import concurrent.futures import jsonschema import png import pystache import six from dcos import constants from dcos.errors import DCOSException def get_logger(name): """Get a logger :param name: The name of the logger. E.g. __name__ :type name: str :returns: The logger for the specified name :rtype: logging.Logger """ return logging.getLogger(name) @contextlib.contextmanager def tempdir(): """A context manager for temporary directories. The lifetime of the returned temporary directory corresponds to the lexical scope of the returned file descriptor. :return: Reference to a temporary directory :rtype: str """ tmpdir = tempfile.mkdtemp() try: yield tmpdir finally: shutil.rmtree(tmpdir, ignore_errors=True) def sh_copy(src, dst): """Copy file src to the file or directory dst. :param src: source file :type src: str :param dst: destination file or directory :type dst: str :rtype: None """ try: shutil.copy(src, dst) except EnvironmentError as e: logger.exception('Unable to copy [%s] to [%s]', src, dst) if e.strerror: if e.filename: raise DCOSException("{}: {}".format(e.strerror, e.filename)) else: raise DCOSException(e.strerror) else: raise DCOSException(e) except Exception as e: logger.exception('Unknown error while coping [%s] to [%s]', src, dst) raise DCOSException(e) def ensure_dir_exists(directory): """If `directory` does not exist, create it. :param directory: path to the directory :type directory: string :rtype: None """ if not os.path.exists(directory): logger.info('Creating directory: %r', directory) try: os.makedirs(directory, 0o775) except os.error as e: raise DCOSException( 'Cannot create directory [{}]: {}'.format(directory, e)) def ensure_file_exists(path): """ Create file if it doesn't exist :param path: path of file to create :type path: str :rtype: None """ if not os.path.exists(path): try: open(path, 'w').close() except IOError as e: raise DCOSException( 'Cannot create file [{}]: {}'.format(path, e)) def read_file(path): """ :param path: path to file :type path: str :returns: contents of file :rtype: str """ if not os.path.isfile(path): raise DCOSException('Path [{}] is not a file'.format(path)) with open_file(path) as file_: return file_.read() def get_config_path(): """ Returns the path to the DCOS config file. :returns: path to the DCOS config file :rtype: str """ default = os.path.expanduser( os.path.join("~", constants.DCOS_DIR, 'dcos.toml')) return os.environ.get(constants.DCOS_CONFIG_ENV, default) def get_config(mutable=False): """ Returns the DCOS configuration object :param mutable: True if the returned Toml object should be mutable :type mutable: boolean :returns: Configuration object :rtype: Toml | MutableToml """ # avoid circular import from dcos import config path = get_config_path() return config.load_from_path(path, mutable) def get_config_vals(keys, config=None): """Gets config values for each of the keys. Raises a DCOSException if any of the keys don't exist. :param config: config :type config: Toml :param keys: keys in the config dict :type keys: [str] :returns: values for each of the keys :rtype: [object] """ config = config or get_config() missing = [key for key in keys if key not in config] if missing: raise missing_config_exception(keys) return [config[key] for key in keys] def missing_config_exception(keys): """ DCOSException for a missing config value :param keys: keys in the config dict :type keys: [str] :returns: DCOSException :rtype: DCOSException """ msg = '\n'.join( 'Missing required config parameter: "{0}".'.format(key) + ' Please run `dcos config set {0} `.'.format(key) for key in keys) return DCOSException(msg) def which(program): """Returns the path to the named executable program. :param program: The program to locate: :type program: str :rtype: str """ def is_exe(file_path): return os.path.isfile(file_path) and os.access(file_path, os.X_OK) file_path, filename = os.path.split(program) if file_path: if is_exe(program): return program elif constants.PATH_ENV in os.environ: for path in os.environ[constants.PATH_ENV].split(os.pathsep): path = path.strip('"') exe_file = os.path.join(path, program) if is_exe(exe_file): return exe_file if is_windows_platform() and not program.endswith('.exe'): return which(program + '.exe') return None def dcos_bin_path(): """Returns the real DCOS path based on the current executable :returns: the real path to the DCOS path :rtype: str """ return os.path.dirname(os.path.realpath(sys.argv[0])) def configure_process_from_environ(): """Configure the program's logger and debug messages using the environment variable :rtype: None """ configure_logger(os.environ.get(constants.DCOS_LOG_LEVEL_ENV)) configure_debug(os.environ.get(constants.DCOS_DEBUG_ENV)) def configure_debug(is_debug): """Configure debug messages for the program :param is_debug: Enable debug message if true; otherwise disable debug messages :type is_debug: bool :rtype: None """ if is_debug: six.moves.http_client.HTTPConnection.debuglevel = 1 def configure_logger(log_level): """Configure the program's logger. :param log_level: Log level for configuring logging :type log_level: str :rtype: None """ if log_level is None: logging.disable(logging.CRITICAL) return None if log_level in constants.VALID_LOG_LEVEL_VALUES: logging.basicConfig( format=('%(asctime)s ' '%(pathname)s:%(funcName)s:%(lineno)d - ' '%(message)s'), stream=sys.stderr, level=log_level.upper()) return None msg = 'Log level set to an unknown value {!r}. Valid values are {!r}' raise DCOSException( msg.format(log_level, constants.VALID_LOG_LEVEL_VALUES)) def load_json(reader): """Deserialize a reader into a python object :param reader: the json reader :type reader: a :code:`.read()`-supporting object :returns: the deserialized JSON object :rtype: dict | list | str | int | float | bool """ try: return json.load(reader) except Exception as error: logger.error( 'Unhandled exception while loading JSON: %r', error) raise DCOSException('Error loading JSON: {}'.format(error)) def load_jsons(value): """Deserialize a string to a python object :param value: The JSON string :type value: str :returns: The deserialized JSON object :rtype: dict | list | str | int | float | bool """ try: return json.loads(value) except: logger.exception( 'Unhandled exception while loading JSON: %r', value) raise DCOSException('Error loading JSON.') def validate_json(instance, schema): """Validate an instance under the given schema. :param instance: the instance to validate :type instance: dict :param schema: the schema to validate with :type schema: dict :returns: list of errors as strings :rtype: [str] """ def sort_key(ve): return six.u(_hack_error_message_fix(ve.message)) validator = jsonschema.Draft4Validator(schema) validation_errors = list(validator.iter_errors(instance)) validation_errors = sorted(validation_errors, key=sort_key) return [_format_validation_error(e) for e in validation_errors] # TODO(jsancio): clean up this hack # The error string from jsonschema already contains improperly formatted # JSON values, so we have to resort to removing the unicode prefix using # a regular expression. def _hack_error_message_fix(message): """ :param message: message to fix by removing u'...' :type message: str :returns: the cleaned up message :rtype: str """ # This regular expression matches the character 'u' followed by the # single-quote character, all optionally preceded by a left square # bracket, parenthesis, curly brace, or whitespace character. return re.compile("([\[\(\{\s])u'").sub( "\g<1>'", re.compile("^u'").sub("'", message)) def _format_validation_error(error): """ :param error: validation error to format :type error: jsonchema.exceptions.ValidationError :returns: string representation of the validation error :rtype: str """ error_message = _hack_error_message_fix(error.message) match = re.search("(.+) is a required property", error_message) if match: message = 'Error: missing required property {}.'.format( match.group(1)) else: message = 'Error: {}\n'.format(error_message) if len(error.absolute_path) > 0: message += 'Path: {}\n'.format( '.'.join([str(path) for path in error.absolute_path])) message += 'Value: {}'.format(json.dumps(error.instance)) return message def create_schema(obj): """ Creates a basic json schema derived from `obj`. :param obj: object for which to derive a schema :type obj: str | int | float | dict | list :returns: json schema :rtype: dict """ if isinstance(obj, bool): return {'type': 'boolean'} elif isinstance(obj, float): return {'type': 'number'} elif isinstance(obj, six.integer_types): return {'type': 'integer'} elif isinstance(obj, six.string_types): return {'type': 'string'} elif isinstance(obj, collections.Mapping): schema = {'type': 'object', 'properties': {}, 'additionalProperties': False, 'required': list(obj.keys())} for key, val in obj.items(): schema['properties'][key] = create_schema(val) return schema elif isinstance(obj, collections.Sequence): schema = {'type': 'array'} if obj: schema['items'] = create_schema(obj[0]) return schema else: raise ValueError( 'Cannot create schema with object {} of unrecognized type' .format(str(obj))) def list_to_err(errs): """convert list of error strings to a single string :param errs: list of string errors :type errs: [str] :returns: error message :rtype: str """ return str.join('\n\n', errs) def parse_int(string): """Parse string and an integer :param string: string to parse as an integer :type string: str :returns: the interger value of the string :rtype: int """ try: return int(string) except: logger.error( 'Unhandled exception while parsing string as int: %r', string) raise DCOSException('Error parsing string as int') def parse_float(string): """Parse string and an float :param string: string to parse as an float :type string: str :returns: the float value of the string :rtype: float """ try: return float(string) except: logger.error( 'Unhandled exception while parsing string as float: %r', string) raise DCOSException('Error parsing string as float') def render_mustache_json(template, data): """Render the supplied mustache template and data as a JSON value :param template: the mustache template to render :type template: str :param data: the data to use as a rendering context :type data: dict :returns: the rendered template :rtype: dict | list | str | int | float | bool """ try: r = CustomJsonRenderer() rendered = r.render(template, data) except Exception as e: logger.exception( 'Error rendering mustache template [%r] [%r]', template, data) raise DCOSException(e) logger.debug('Rendered mustache template: %s', rendered) return load_jsons(rendered) def is_windows_platform(): """ :returns: True is program is running on Windows platform, False in other case :rtype: boolean """ return platform.system() == "Windows" class CustomJsonRenderer(pystache.Renderer): def str_coerce(self, val): """ Coerce a non-string value to a string. This method is called whenever a non-string is encountered during the rendering process when a string is needed (e.g. if a context value for string interpolation is not a string). :param val: the mustache template to render :type val: any :returns: a string containing a JSON representation of the value :rtype: str """ return json.dumps(val) def duration(fn): """ Decorator to log the duration of a function. :param fn: function to measure :type fn: function :returns: wrapper function :rtype: function """ @functools.wraps(fn) def timer(*args, **kwargs): start = time.time() try: return fn(*args, **kwargs) finally: logger.debug("duration: {0}.{1}: {2:2.2f}s".format( fn.__module__, fn.__name__, time.time() - start)) return timer def humanize_bytes(b): """ Return a human representation of a number of bytes. :param b: number of bytes :type b: number :returns: human representation of a number of bytes :rtype: str """ abbrevs = ( (1 << 30, 'GB'), (1 << 20, 'MB'), (1 << 10, 'kB'), (1, 'B') ) for factor, suffix in abbrevs: if b >= factor: break return "{0:.2f} {1}".format(b/float(factor), suffix) @contextlib.contextmanager def open_file(path, *args): """Context manager that opens a file, and raises a DCOSException if it fails. :param path: file path :type path: str :param *args: other arguments to pass to `open` :type *args: [str] :returns: a context manager :rtype: context manager """ try: file_ = open(path, *args) yield file_ except IOError as e: logger.exception('Unable to open file: %s', path) raise io_exception(path, e.errno) file_.close() def io_exception(path, errno): """Returns a DCOSException for when there is an error opening the file at `path` :param path: file path :type path: str :param errno: IO error number :type errno: int :returns: DCOSException :rtype: DCOSException """ return DCOSException('Error opening file [{}]: {}'.format( path, os.strerror(errno))) STREAM_CONCURRENCY = 20 def stream(fn, objs): """Apply `fn` to `objs` in parallel, yielding the (Future, obj) for each as it completes. :param fn: function :type fn: function :param objs: objs :type objs: objs :returns: iterator over (Future, typeof(obj)) :rtype: iterator over (Future, typeof(obj)) """ with concurrent.futures.ThreadPoolExecutor(STREAM_CONCURRENCY) as pool: jobs = {pool.submit(fn, obj): obj for obj in objs} for job in concurrent.futures.as_completed(jobs): yield job, jobs[job] def get_ssh_options(config_file, options): """Returns the SSH arguments for the given parameters. Used by commands that wrap SSH. :param config_file: SSH config file. :type config_file: str | None :param options: SSH options :type options: [str] :rtype: str """ ssh_options = ' '.join('-o {}'.format(opt) for opt in options) if config_file: ssh_options += ' -F {}'.format(config_file) if ssh_options: ssh_options += ' ' return ssh_options def validate_png(filename): """Validate file as a png image. Throws a DCOSException if it is not an PNG :param filename: path to the image :type filename: str :rtype: None """ try: png.Reader(filename=filename).validate_signature() except Exception as e: logger.exception(e) raise DCOSException( 'Unable to validate [{}] as a PNG file'.format(filename)) logger = get_logger(__name__) PKցFHJ00 dcos/http.pyimport getpass import os import sys import threading import requests from dcos import constants, util from dcos.errors import DCOSException, DCOSHTTPException from requests.auth import AuthBase, HTTPBasicAuth from six.moves import urllib from six.moves.urllib.parse import urlparse logger = util.get_logger(__name__) lock = threading.Lock() DEFAULT_TIMEOUT = 5 # only accessed from _request_with_auth AUTH_CREDS = {} # (hostname, auth_scheme, realm) -> AuthBase() def _default_is_success(status_code): """Returns true if the success status is between [200, 300). :param response_status: the http response status :type response_status: int :returns: True for success status; False otherwise :rtype: bool """ return 200 <= status_code < 300 @util.duration def _request(method, url, is_success=_default_is_success, timeout=DEFAULT_TIMEOUT, auth=None, verify=None, **kwargs): """Sends an HTTP request. :param method: method for the new Request object :type method: str :param url: URL for the new Request object :type url: str :param is_success: Defines successful status codes for the request :type is_success: Function from int to bool :param timeout: request timeout :type timeout: int :param auth: authentication :type auth: AuthBase :param verify: whether to verify SSL certs or path to cert(s) :type verify: bool | str :param kwargs: Additional arguments to requests.request (see http://docs.python-requests.org/en/latest/api/#requests.request) :type kwargs: dict :rtype: Response """ logger.info( 'Sending HTTP [%r] to [%r]: %r', method, url, kwargs.get('headers')) try: response = requests.request( method=method, url=url, timeout=timeout, auth=auth, verify=verify, **kwargs) except requests.exceptions.ConnectionError as e: logger.exception("HTTP Connection Error") raise DCOSException('URL [{0}] is unreachable: {1}'.format(url, e)) except requests.exceptions.Timeout as e: logger.exception("HTTP Timeout") raise DCOSException('Request to URL [{0}] timed out.'.format(url)) except requests.exceptions.RequestException as e: logger.exception("HTTP Exception") raise DCOSException('HTTP Exception: {}'.format(e)) logger.info('Received HTTP response [%r]: %r', response.status_code, response.headers) return response def _request_with_auth(response, method, url, is_success=_default_is_success, timeout=None, verify=None, **kwargs): """Try request (3 times) with credentials if 401 returned from server :param response: requests.response :type response: requests.Response :param method: method for the new Request object :type method: str :param url: URL for the new Request object :type url: str :param is_success: Defines successful status codes for the request :type is_success: Function from int to bool :param timeout: request timeout :type timeout: int :param verify: whether to verify SSL certs or path to cert(s) :type verify: bool | str :param kwargs: Additional arguments to requests.request (see http://docs.python-requests.org/en/latest/api/#requests.request) :type kwargs: dict :rtype: requests.Response """ i = 0 while i < 3 and response.status_code == 401: parsed_url = urlparse(response.url) hostname = parsed_url.hostname auth_scheme, realm = get_auth_scheme(response) creds = (hostname, auth_scheme, realm) with lock: if creds not in AUTH_CREDS: auth = _get_http_auth(response, parsed_url, auth_scheme) else: auth = AUTH_CREDS[creds] # try request again, with auth response = _request(method, url, is_success, timeout, auth, verify, **kwargs) # only store credentials if they're valid with lock: if creds not in AUTH_CREDS and response.status_code == 200: AUTH_CREDS[creds] = auth i += 1 if response.status_code == 401: raise DCOSException("Authentication failed") return response def request(method, url, is_success=_default_is_success, timeout=None, verify=None, **kwargs): """Sends an HTTP request. If the server responds with a 401, ask the user for their credentials, and try request again (up to 3 times). :param method: method for the new Request object :type method: str :param url: URL for the new Request object :type url: str :param is_success: Defines successful status codes for the request :type is_success: Function from int to bool :param timeout: request timeout :type timeout: int :param verify: whether to verify SSL certs or path to cert(s) :type verify: bool | str :param kwargs: Additional arguments to requests.request (see http://docs.python-requests.org/en/latest/api/#requests.request) :type kwargs: dict :rtype: Response """ if 'headers' not in kwargs: kwargs['headers'] = {'Accept': 'application/json'} if verify is None and constants.DCOS_SSL_VERIFY_ENV in os.environ: verify = os.environ[constants.DCOS_SSL_VERIFY_ENV] if verify.lower() == "true": verify = True elif verify.lower() == "false": verify = False # Silence 'Unverified HTTPS request' and 'SecurityWarning' for bad certs if verify is not None: silence_requests_warnings() response = _request(method, url, is_success, timeout, verify=verify, **kwargs) if response.status_code == 401: response = _request_with_auth(response, method, url, is_success, timeout, verify, **kwargs) if is_success(response.status_code): return response else: raise DCOSHTTPException(response) def head(url, **kwargs): """Sends a HEAD request. :param url: URL for the new Request object :type url: str :param kwargs: Additional arguments to requests.request (see py:func:`request`) :type kwargs: dict :rtype: Response """ return request('head', url, **kwargs) def get(url, **kwargs): """Sends a GET request. :param url: URL for the new Request object :type url: str :param kwargs: Additional arguments to requests.request (see py:func:`request`) :type kwargs: dict :rtype: Response """ return request('get', url, **kwargs) def post(url, data=None, json=None, **kwargs): """Sends a POST request. :param url: URL for the new Request object :type url: str :param data: Request body :type data: dict, bytes, or file-like object :param json: JSON request body :type data: dict :param kwargs: Additional arguments to requests.request (see py:func:`request`) :type kwargs: dict :rtype: Response """ return request('post', url, data=data, json=json, **kwargs) def put(url, data=None, **kwargs): """Sends a PUT request. :param url: URL for the new Request object :type url: str :param data: Request body :type data: dict, bytes, or file-like object :param kwargs: Additional arguments to requests.request (see py:func:`request`) :type kwargs: dict :rtype: Response """ return request('put', url, data=data, **kwargs) def patch(url, data=None, **kwargs): """Sends a PATCH request. :param url: URL for the new Request object :type url: str :param data: Request body :type data: dict, bytes, or file-like object :param kwargs: Additional arguments to requests.request (see py:func:`request`) :type kwargs: dict :rtype: Response """ return request('patch', url, data=data, **kwargs) def delete(url, **kwargs): """Sends a DELETE request. :param url: URL for the new Request object :type url: str :param kwargs: Additional arguments to requests.request (see py:func:`request`) :type kwargs: dict :rtype: Response """ return request('delete', url, **kwargs) def silence_requests_warnings(): """Silence warnings from requests.packages.urllib3. See DCOS-1007.""" requests.packages.urllib3.disable_warnings() def _get_auth_credentials(username, hostname): """Get username/password for auth :param username: username user for authentication :type username: str :param hostname: hostname for credentials :type hostname: str :returns: username, password :rtype: str, str """ if username is None: sys.stdout.write("{}'s username: ".format(hostname)) sys.stdout.flush() username = sys.stdin.readline().strip().lower() password = getpass.getpass("{}@{}'s password: ".format(username, hostname)) return username, password def get_auth_scheme(response): """Return authentication scheme and realm requested by server for 'Basic' or 'acsjwt' (DCOS acs auth) type or None :param response: requests.response :type response: requests.Response :returns: auth_scheme, realm :rtype: (str, str) | None """ if 'www-authenticate' in response.headers: auths = response.headers['www-authenticate'].split(',') scheme = next((auth_type.rstrip().lower() for auth_type in auths if auth_type.rstrip().lower().startswith("basic") or auth_type.rstrip().lower().startswith("acsjwt")), None) if scheme: scheme_info = scheme.split("=") auth_scheme = scheme_info[0].split(" ")[0].lower() realm = scheme_info[-1].strip(' \'\"').lower() return auth_scheme, realm else: return None else: return None def _get_http_auth(response, url, auth_scheme): """Get authentication mechanism required by server :param response: requests.response :type response: requests.Response :param url: parsed request url :type url: str :param auth_scheme: str :type auth_scheme: str :returns: AuthBase :rtype: AuthBase """ hostname = url.hostname username = url.username if 'www-authenticate' in response.headers: if auth_scheme not in ['basic', 'acsjwt']: msg = ("Server responded with an HTTP 'www-authenticate' field of " "'{}', DCOS only supports 'Basic'".format( response.headers['www-authenticate'])) raise DCOSException(msg) username, password = _get_auth_credentials(username, hostname) if auth_scheme == 'basic': return HTTPBasicAuth(username, password) else: return _get_dcos_acs_auth(username, password) else: msg = ("Invalid HTTP response: server returned an HTTP 401 response " "with no 'www-authenticate' field") raise DCOSException(msg) def _get_dcos_acs_auth(uid, password): """Get authentication flow for dcos acs auth :param uid: uid :type uid: str :param password: password :type password: str :returns: DCOSAcsAuth :rtype: AuthBase """ dcos_url = util.get_config_vals( ['core.dcos_url'], util.get_config())[0] url = urllib.parse.urljoin(dcos_url, 'acs/api/v1/auth/login') creds = {"uid": uid, "password": password} # using private method here, so we don't retry on this request # error here will be bubbled up to _request_with_auth response = _request('post', url, json=creds) token = None if response.status_code == 200: token = response.json()['token'] return DCOSAcsAuth(token) class DCOSAcsAuth(AuthBase): """Invokes DCOS Authentication flow for given Request object.""" def __init__(self, token): self.token = token def __call__(self, r): r.headers['Authorization'] = "token={}".format(self.token) return r PKցFHN^M dcos/auth.pyimport json import sys import uuid import pkg_resources from dcos import config, emitting, errors, http, jsonitem, util from dcos.errors import DCOSException from six import iteritems from oauth2client import client CLIENT_ID = '6a552732-ab9b-410d-9b7d-d8c6523b09a1' CLIENT_SECRET = 'f56c1e2b-8599-40ca-b6a0-3aba3e702eae' AUTH_URL = 'https://accounts.mesosphere.com/oauth/authorize' TOKEN_URL = 'https://accounts.mesosphere.com/oauth/token' USER_INFO_URL = 'https://accounts.mesosphere.com/api/v1/user.json' CORE_TOKEN_KEY = 'token' CORE_EMAIL_KEY = 'email' emitter = emitting.FlatEmitter() logger = util.get_logger(__name__) def _authorize(): """Create OAuth flow and authorize user :return: credentials dict :rtype: dict """ try: flow = client.OAuth2WebServerFlow( client_id=CLIENT_ID, client_secret=CLIENT_SECRET, scope='', auth_uri=AUTH_URL, token_uri=TOKEN_URL, redirect_uri=client.OOB_CALLBACK_URN, response_type='code' ) return _run(flow) except: logger.exception('Error during OAuth web flow') raise DCOSException('There was a problem with ' 'web authentication.') def make_oauth_request(code, flow): """Make request to auth server using auth_code. :param: code: auth_code read from cli :param: flow: OAuth2 web server flow :return: dict with the keys token and email :rtype: dict """ credential = flow.step2_exchange(code) token = credential.access_token headers = {'Authorization': str('Bearer ' + token)} data = http.requests.get(USER_INFO_URL, headers=headers).json() mail = data['email'] credentials = {CORE_TOKEN_KEY: credential.access_token, CORE_EMAIL_KEY: mail} return credentials def _run(flow): """Make authorization and retrieve access token and user email. :param flow: OAuth2 web server flow :param launch_browser: if possible to run browser :return: dict with the keys token and email :rtype: dict """ emitter.publish( errors.DefaultError( '\n\n\n{}\n\n {}\n\n'.format( 'Go to the following link in your browser:', flow.step1_get_authorize_url()))) sys.stderr.write('Enter verification code: ') code = sys.stdin.readline().strip() if not code: sys.stderr.write('Skipping authentication.\nEnter email address: ') email = sys.stdin.readline().strip() if not email: emitter.publish( errors.DefaultError( 'Skipping email input.')) email = str(uuid.uuid1()) return {CORE_EMAIL_KEY: email} return make_oauth_request(code, flow) def check_if_user_authenticated(): """Check if user is authenticated already :returns user auth status :rtype: boolean """ dcos_config = util.get_config() return dcos_config.get('core.email', '') != '' def force_auth(): """Make user authentication process :returns authentication process status :rtype: boolean """ credentials = _authorize() _save_auth_keys(credentials) def _save_auth_keys(key_dict): """ :param key_dict: auth parameters dict :type key_dict: dict :rtype: None """ toml_config = util.get_config(True) section = 'core' config_schema = json.loads( pkg_resources.resource_string( 'dcoscli', 'data/config-schema/core.json').decode('utf-8')) for k, v in iteritems(key_dict): python_value = jsonitem.parse_json_value(k, v, config_schema) name = '{}.{}'.format(section, k) toml_config[name] = python_value config.save(toml_config) return None PKFHX$dcos-0.3.1.dist-info/DESCRIPTION.rstDCOS Command Line Interface =========================== The DCOS Command Line Interface (CLI) is a command line utility that provides a user-friendly yet powerful way to manage DCOS installations. This project is open source. Please see GitHub_ to access source code and to contribute. Full documentation is available for the DCOS CLI on the `Mesosphere docs website`_. .. _GitHub: https://github.com/mesosphere/dcos-cli .. _Mesosphere docs website: http://docs.mesosphere.com/using/cli/ PKFHn,"dcos-0.3.1.dist-info/metadata.json{"classifiers": ["Development Status :: 4 - Beta", "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Topic :: Software Development :: User Interfaces", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.4"], "extensions": {"python.details": {"contacts": [{"email": "team@mesosphere.io", "name": "Mesosphere, Inc.", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}, "project_urls": {"Home": "https://github.com/mesosphere/dcos-cli"}}}, "extras": [], "generator": "bdist_wheel (0.28.0)", "keywords": ["mesos", "apache", "marathon", "mesosphere", "command", "datacenter"], "metadata_version": "2.0", "name": "dcos", "run_requires": [{"requires": ["futures (>=3.0,<4.0)", "gitpython (>=1.0,<2.0)", "jsonschema (==2.4)", "pager (>=3.3,<4.0)", "portalocker (>=0.5,<1.0)", "prettytable (>=0.7,<1.0)", "pygments (>=2.0,<3.0)", "pypng (==0.0.18)", "pystache (>=0.5,<1.0)", "requests (>=2.6,<3.0)", "six (>=1.9,<2.0)", "toml (>=0.9,<1.0)"]}], "summary": "DCOS Common Modules", "version": "0.3.1"}PKFHK1O$"dcos-0.3.1.dist-info/top_level.txtdcos PKFH>nndcos-0.3.1.dist-info/WHEELWheel-Version: 1.0 Generator: bdist_wheel (0.28.0) Root-Is-Purelib: true Tag: py2-none-any Tag: py3-none-any PKFHjodcos-0.3.1.dist-info/METADATAMetadata-Version: 2.0 Name: dcos Version: 0.3.1 Summary: DCOS Common Modules Home-page: https://github.com/mesosphere/dcos-cli Author: Mesosphere, Inc. Author-email: team@mesosphere.io License: UNKNOWN Keywords: mesos apache marathon mesosphere command datacenter Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Intended Audience :: Developers Classifier: Intended Audience :: Information Technology Classifier: Topic :: Software Development :: User Interfaces Classifier: License :: OSI Approved :: Apache Software License Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.4 Requires-Dist: futures (>=3.0,<4.0) Requires-Dist: gitpython (>=1.0,<2.0) Requires-Dist: jsonschema (==2.4) Requires-Dist: pager (>=3.3,<4.0) Requires-Dist: portalocker (>=0.5,<1.0) Requires-Dist: prettytable (>=0.7,<1.0) Requires-Dist: pygments (>=2.0,<3.0) Requires-Dist: pypng (==0.0.18) Requires-Dist: pystache (>=0.5,<1.0) Requires-Dist: requests (>=2.6,<3.0) Requires-Dist: six (>=1.9,<2.0) Requires-Dist: toml (>=0.9,<1.0) DCOS Command Line Interface =========================== The DCOS Command Line Interface (CLI) is a command line utility that provides a user-friendly yet powerful way to manage DCOS installations. This project is open source. Please see GitHub_ to access source code and to contribute. Full documentation is available for the DCOS CLI on the `Mesosphere docs website`_. .. _GitHub: https://github.com/mesosphere/dcos-cli .. _Mesosphere docs website: http://docs.mesosphere.com/using/cli/ PKFH6\dcos-0.3.1.dist-info/RECORDdcos/__init__.py,sha256=KSmY9Jovq9_f1YXyZceDIt_J-yBnwmUgdpn4G03PoOA,127 dcos/auth.py,sha256=_Sd6kreauRzen0bzga82J-RnO6_h2jF6cXVcCzojc6Y,3814 dcos/cmds.py,sha256=0m2OnENuryXwusbBBMXWhXzv4No3RC5CeP2wFkKhmuU,1376 dcos/config.py,sha256=_1Uf3zKHwNbICbsAyzw-MMs679U-H_YaIegeYh4faP0,5246 dcos/constants.py,sha256=X2kxjwtMhnotgfZHdJKDmsfPP6BjrO11o1nQBlP24gg,1194 dcos/emitting.py,sha256=fAWraTY8VbiUNGntx0FWMAjUekqa-0IXMa7CXn7l_l4,5210 dcos/errors.py,sha256=U_OtaC0T5aL9FN61fvd27BcWKLLZR3Imb_CFWUZrBro,1060 dcos/http.py,sha256=6bOpYfub_TXujO8uKLZqt2Q0eFULovlWJrVXuVosZyA,12526 dcos/jsonitem.py,sha256=inw1BIjSodSh2YCbnE-OjqYx2ADLQwY8KvpxAl4xAxg,8085 dcos/marathon.py,sha256=XUKwKFdRZDpwiXBIarIJH-2vbdiUBeL2Oc193RAghM4,22242 dcos/mesos.py,sha256=tlTF0Ysl5uRaQhBpQ996E_wSunBi5nh-x4z5pMGMUPU,27577 dcos/options.py,sha256=yDE6xv6-41xoFfdAX77qWI32m0lip7SxdbbiJPkTll8,649 dcos/package.py,sha256=BJ-xETIEvkg6Nq5JQUCgdPuuinoZhB1FkkHKWiqsWxc,49233 dcos/subcommand.py,sha256=GQlZl4PI5fIKhRZiXSyGSDCy_Hcv-nr8Au97nwZVY0A,12535 dcos/util.py,sha256=C55By8LBC03eNIbJAkGL0UUj7PWI-srHfXkRKt1ybEU,17126 dcos-0.3.1.dist-info/DESCRIPTION.rst,sha256=HUKLQlLswxgRXIXS5kUn8aan-ty5duxRFGPuvKQGOXY,493 dcos-0.3.1.dist-info/METADATA,sha256=1A5sqdeVLbPjSDImpmVbztv-sLeHm_X4_lpvRIYzlxY,1669 dcos-0.3.1.dist-info/RECORD,, dcos-0.3.1.dist-info/WHEEL,sha256=c5du820PMLPXFYzXDp0SSjIjJ-7MmVRpJa1kKfTaqlc,110 dcos-0.3.1.dist-info/metadata.json,sha256=LHRro71-pq6-zIyV5R1q4yH5CkY8zwiUh4G789JCOuk,1202 dcos-0.3.1.dist-info/top_level.txt,sha256=wJaZ72wizYIaVdXiuvv4wU2aKj9AP5TXAUNSItuEpK4,5 PKFH4W dcos/__init__.pyPKցFH~YVVdcos/marathon.pyPKցFHԻuWdcos/jsonitem.pyPKցFHD`kk wdcos/mesos.pyPKցFH0~~ddcos/config.pyPKցFHqZZdcos/emitting.pyPKցFHHiQQ dcos/package.pyPKցFH4F00dcos/subcommand.pyPKցFHPLl;dcos/constants.pyPKցFH Z`` dcos/cmds.pyPKցFH$$dcos/errors.pyPKցFHt dcos/options.pyPKցFHBB dcos/util.pyPKցFHJ00 Rdcos/http.pyPKցFHN^M ̃dcos/auth.pyPKFHX$ܒdcos-0.3.1.dist-info/DESCRIPTION.rstPKFHn," dcos-0.3.1.dist-info/metadata.jsonPKFHK1O$"dcos-0.3.1.dist-info/top_level.txtPKFH>nnBdcos-0.3.1.dist-info/WHEELPKFHjodcos-0.3.1.dist-info/METADATAPKFH6\dcos-0.3.1.dist-info/RECORDPKZ