PKցFHtests/unit/__init__.pyPKցFHc  tests/unit/test_tables.pyimport datetime from dcoscli import tables import mock import pytz from ..fixtures.marathon import (app_fixture, app_task_fixture, deployment_fixture, group_fixture) from ..fixtures.node import slave_fixture from ..fixtures.package import package_fixture, search_result_fixture from ..fixtures.service import framework_fixture from ..fixtures.task import browse_fixture, task_fixture def test_task_table(): _test_table(tables.task_table, [task_fixture()], 'tests/unit/data/task.txt') def test_app_table(): apps = [app_fixture()] deployments = [] table = tables.app_table(apps, deployments) with open('tests/unit/data/app.txt') as f: assert str(table) == f.read() def test_deployment_table(): _test_table(tables.deployment_table, [deployment_fixture()], 'tests/unit/data/deployment.txt') def test_app_task_table(): _test_table(tables.app_task_table, [app_task_fixture()], 'tests/unit/data/app_task.txt') def test_service_table(): _test_table(tables.service_table, [framework_fixture()], 'tests/unit/data/service.txt') def test_group_table(): _test_table(tables.group_table, [group_fixture()], 'tests/unit/data/group.txt') def test_package_table(): _test_table(tables.package_table, [package_fixture()], 'tests/unit/data/package.txt') def test_package_search_table(): _test_table(tables.package_search_table, [search_result_fixture()], 'tests/unit/data/package_search.txt') def test_node_table(): _test_table(tables.slave_table, [slave_fixture()], 'tests/unit/data/node.txt') def test_ls_long_table(): with mock.patch('dcoscli.tables._format_unix_timestamp', lambda ts: datetime.datetime.fromtimestamp( ts, pytz.utc).strftime('%b %d %H:%M')): _test_table(tables.ls_long_table, browse_fixture(), 'tests/unit/data/ls_long.txt') def _test_table(table_fn, fixture_fn, path): table = table_fn(fixture_fn) with open(path) as f: assert str(table) == f.read() PKցFHtests/integrations/__init__.pyPKցFH[`E@@!tests/integrations/test_config.pyimport json import os import dcoscli.constants as cli_constants import six from dcos import constants import pytest from .common import assert_command, config_set, config_unset, exec_command @pytest.fixture def env(): r = os.environ.copy() r.update({ constants.PATH_ENV: os.environ[constants.PATH_ENV], constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "dcos.toml"), cli_constants.DCOS_PRODUCTION_ENV: 'false' }) return r @pytest.fixture def missing_env(): r = os.environ.copy() r.update({ constants.PATH_ENV: os.environ[constants.PATH_ENV], constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "config", "missing_params_dcos.toml") }) return r def test_help(): with open('tests/data/help/config.txt') as content: assert_command(['dcos', 'config', '--help'], stdout=content.read().encode('utf-8')) def test_info(): stdout = b'Get and set DCOS CLI configuration properties\n' assert_command(['dcos', 'config', '--info'], stdout=stdout) def test_version(): stdout = b'dcos-config version SNAPSHOT\n' assert_command(['dcos', 'config', '--version'], stdout=stdout) def test_list_property(env): stdout = b"""core.dcos_url=http://dcos.snakeoil.mesosphere.com core.email=test@mail.com core.reporting=False core.ssl_verify=false core.timeout=5 package.cache=tmp/cache package.sources=['https://github.com/mesosphere/universe/archive/\ cli-test-3.zip'] """ assert_command(['dcos', 'config', 'show'], stdout=stdout, env=env) def test_get_existing_string_property(env): _get_value('core.dcos_url', 'http://dcos.snakeoil.mesosphere.com', env) def test_get_existing_boolean_property(env): _get_value('core.reporting', False, env) def test_get_existing_number_property(env): _get_value('core.timeout', 5, env) def test_get_missing_property(env): _get_missing_value('missing.property', env) def test_invalid_dcos_url(env): stderr = b'Please check url \'abc.com\'. Missing http(s)://\n' assert_command(['dcos', 'config', 'set', 'core.dcos_url', 'abc.com'], stderr=stderr, returncode=1) def test_get_top_property(env): stderr = ( b"Property 'package' doesn't fully specify a value - " b"possible properties are:\n" b"package.cache\n" b"package.sources\n" ) assert_command(['dcos', 'config', 'show', 'package'], stderr=stderr, returncode=1) def test_set_existing_string_property(env): config_set('core.dcos_url', 'http://dcos.snakeoil.mesosphere.com:5081', env) _get_value('core.dcos_url', 'http://dcos.snakeoil.mesosphere.com:5081', env) config_set('core.dcos_url', 'http://dcos.snakeoil.mesosphere.com', env) def test_set_existing_boolean_property(env): config_set('core.reporting', 'true', env) _get_value('core.reporting', True, env) config_set('core.reporting', 'true', env) def test_set_existing_number_property(env): config_set('core.timeout', '5', env) _get_value('core.timeout', 5, env) config_set('core.timeout', '5', env) def test_set_change_output(env): assert_command( ['dcos', 'config', 'set', 'core.dcos_url', 'http://dcos.snakeoil.mesosphere.com:5081'], stdout=(b"[core.dcos_url]: changed from " b"'http://dcos.snakeoil.mesosphere.com' to " b"'http://dcos.snakeoil.mesosphere.com:5081'\n"), env=env) config_set('core.dcos_url', 'http://dcos.snakeoil.mesosphere.com', env) def test_set_same_output(env): assert_command( ['dcos', 'config', 'set', 'core.dcos_url', 'http://dcos.snakeoil.mesosphere.com'], stdout=(b"[core.dcos_url]: already set to " b"'http://dcos.snakeoil.mesosphere.com'\n"), env=env) def test_set_new_output(env): config_unset('core.dcos_url', None, env) assert_command( ['dcos', 'config', 'set', 'core.dcos_url', 'http://dcos.snakeoil.mesosphere.com:5081'], stdout=(b"[core.dcos_url]: set to " b"'http://dcos.snakeoil.mesosphere.com:5081'\n"), env=env) config_set('core.dcos_url', 'http://dcos.snakeoil.mesosphere.com', env) def test_append_empty_list(env): config_set('package.sources', '[]', env) _append_value( 'package.sources', 'https://github.com/mesosphere/universe/archive/cli-test-3.zip', env) _get_value( 'package.sources', ['https://github.com/mesosphere/universe/archive/cli-test-3.zip'], env) def test_prepend_empty_list(env): config_set('package.sources', '[]', env) _prepend_value( 'package.sources', 'https://github.com/mesosphere/universe/archive/cli-test-3.zip', env) _get_value( 'package.sources', ['https://github.com/mesosphere/universe/archive/cli-test-3.zip'], env) def test_append_list(env): _append_value( 'package.sources', 'https://github.com/mesosphere/universe/archive/version-2.x.zip', env) _get_value( 'package.sources', ['https://github.com/mesosphere/universe/archive/cli-test-3.zip', 'https://github.com/mesosphere/universe/archive/version-2.x.zip'], env) config_unset('package.sources', '1', env) def test_prepend_list(env): _prepend_value( 'package.sources', 'https://github.com/mesosphere/universe/archive/version-2.x.zip', env) _get_value( 'package.sources', ['https://github.com/mesosphere/universe/archive/version-2.x.zip', 'https://github.com/mesosphere/universe/archive/cli-test-3.zip'], env) config_unset('package.sources', '0', env) def test_append_non_list(env): stderr = (b"Append/Prepend not supported on 'core.dcos_url' " b"properties - use 'dcos config set core.dcos_url new_uri'\n") assert_command( ['dcos', 'config', 'append', 'core.dcos_url', 'new_uri'], returncode=1, stderr=stderr, env=env) def test_prepend_non_list(env): stderr = (b"Append/Prepend not supported on 'core.dcos_url' " b"properties - use 'dcos config set core.dcos_url new_uri'\n") assert_command( ['dcos', 'config', 'prepend', 'core.dcos_url', 'new_uri'], returncode=1, stderr=stderr, env=env) def test_unset_property(env): config_unset('core.reporting', None, env) _get_missing_value('core.reporting', env) config_set('core.reporting', 'false', env) def test_unset_missing_property(env): assert_command( ['dcos', 'config', 'unset', 'missing.property'], returncode=1, stderr=b"Property 'missing.property' doesn't exist\n", env=env) def test_unset_output(env): assert_command(['dcos', 'config', 'unset', 'core.reporting'], stdout=b'Removed [core.reporting]\n', env=env) config_set('core.reporting', 'false', env) def test_unset_index_output(env): stdout = ( b"[package.sources]: removed element " b"'https://github.com/mesosphere/universe/archive/cli-test-3.zip' " b"at index '0'\n" ) assert_command(['dcos', 'config', 'unset', 'package.sources', '--index=0'], stdout=stdout, env=env) _prepend_value( 'package.sources', 'https://github.com/mesosphere/universe/archive/cli-test-3.zip', env) def test_set_whole_list(env): config_set( 'package.sources', '["https://github.com/mesosphere/universe/archive/cli-test-3.zip"]', env) def test_unset_top_property(env): stderr = ( b"Property 'package' doesn't fully specify a value - " b"possible properties are:\n" b"package.cache\n" b"package.sources\n" ) assert_command( ['dcos', 'config', 'unset', 'package'], returncode=1, stderr=stderr, env=env) def test_unset_list_index(env): config_unset('package.sources', '0', env) _get_value( 'package.sources', [], env) _prepend_value( 'package.sources', 'https://github.com/mesosphere/universe/archive/cli-test-3.zip', env) def test_unset_outbound_index(env): stderr = ( b'Index (3) is out of bounds - possible values are ' b'between 0 and 0\n' ) assert_command( ['dcos', 'config', 'unset', '--index=3', 'package.sources'], returncode=1, stderr=stderr, env=env) def test_unset_bad_index(env): stderr = b'Error parsing string as int\n' assert_command( ['dcos', 'config', 'unset', '--index=number', 'package.sources'], returncode=1, stderr=stderr, env=env) def test_unset_index_from_string(env): stderr = b'Unsetting based on an index is only supported for lists\n' assert_command( ['dcos', 'config', 'unset', '--index=0', 'core.dcos_url'], returncode=1, stderr=stderr, env=env) def test_validate(env): stdout = b'Congratulations, your configuration is valid!\n' assert_command(['dcos', 'config', 'validate'], env=env, stdout=stdout) def test_validation_error(env): source = ["https://github.com/mesosphere/universe/archive/cli-test-3.zip"] config_unset('package.sources', None, env) stdout = b"Error: missing required property 'sources'.\n" assert_command(['dcos', 'config', 'validate'], returncode=1, stdout=stdout, env=env) config_set('package.sources', json.dumps(source), env) _get_value('package.sources', source, env) def test_set_property_key(env): assert_command( ['dcos', 'config', 'set', 'path.to.value', 'cool new value'], returncode=1, stderr=b"'path' is not a dcos command.\n", env=env) def test_set_missing_property(missing_env): config_set('core.dcos_url', 'http://localhost:8080', missing_env) _get_value('core.dcos_url', 'http://localhost:8080', missing_env) config_unset('core.dcos_url', None, missing_env) def test_set_core_property(env): config_set('core.reporting', 'true', env) _get_value('core.reporting', True, env) config_set('core.reporting', 'false', env) def test_url_validation(env): key = 'core.dcos_url' default_value = 'http://dcos.snakeoil.mesosphere.com' config_set(key, 'http://localhost', env) config_set(key, 'https://localhost', env) config_set(key, 'http://dcos-1234', env) config_set(key, 'http://dcos-1234.mydomain.com', env) config_set(key, 'http://localhost:5050', env) config_set(key, 'https://localhost:5050', env) config_set(key, 'http://mesos-1234:5050', env) config_set(key, 'http://mesos-1234.mydomain.com:5050', env) config_set(key, 'http://localhost:8080', env) config_set(key, 'https://localhost:8080', env) config_set(key, 'http://marathon-1234:8080', env) config_set(key, 'http://marathon-1234.mydomain.com:5050', env) config_set(key, 'http://user@localhost:8080', env) config_set(key, 'http://u-ser@localhost:8080', env) config_set(key, 'http://user123_@localhost:8080', env) config_set(key, 'http://user:p-ssw_rd@localhost:8080', env) config_set(key, 'http://user123:password321@localhost:8080', env) config_set(key, 'http://us%r1$3:pa#sw*rd321@localhost:8080', env) config_set(key, default_value, env) def test_append_url_validation(env): default_value = ('["https://github.com/mesosphere/universe/archive/' 'cli-test-3.zip"]') config_set('package.sources', '[]', env) _append_value( 'package.sources', 'https://github.com/mesosphere/universe/archive/cli-test-3.zip', env) _append_value( 'package.sources', 'git@github.com:mesosphere/test.git', env) _append_value( 'package.sources', 'https://github.com/mesosphere/test.git', env) _append_value( 'package.sources', 'file://some-domain.com/path/to/file.extension', env) _append_value( 'package.sources', 'file:///path/to/file.extension', env) config_set('package.sources', default_value, env) def test_prepend_url_validation(env): default_value = ('["https://github.com/mesosphere/universe/archive/' 'cli-test-3.zip"]') config_set('package.sources', '[]', env) _prepend_value( 'package.sources', 'https://github.com/mesosphere/universe/archive/cli-test-3.zip', env) _prepend_value( 'package.sources', 'git@github.com:mesosphere/test.git', env) _prepend_value( 'package.sources', 'https://github.com/mesosphere/test.git', env) _prepend_value( 'package.sources', 'file://some-domain.com/path/to/file.extension', env) _prepend_value( 'package.sources', 'file:///path/to/file.extension', env) config_set('package.sources', default_value, env) def test_fail_url_validation(env): _fail_url_validation('set', 'core.dcos_url', 'http://bad_domain/', env) _fail_url_validation('set', 'core.dcos_url', 'http://@domain/', env) _fail_url_validation('set', 'core.dcos_url', 'http://user:pass@/', env) _fail_url_validation('set', 'core.dcos_url', 'http://us:r:pass@url', env) def test_bad_port_fail_url_validation(env): _fail_url_validation('set', 'core.dcos_url', 'http://localhost:bad_port/', env) def test_append_fail_validation(env): _fail_validation('append', 'package.sources', 'bad_url', env) def test_prepend_fail_validation(env): _fail_validation('prepend', 'package.sources', 'bad_url', env) def test_timeout(missing_env): config_set('marathon.url', 'http://1.2.3.4', missing_env) config_set('core.timeout', '1', missing_env) returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'list'], env=missing_env) assert returncode == 1 assert stdout == b'' assert "(connect timeout=1)".encode('utf-8') in stderr config_unset('core.timeout', None, missing_env) config_unset('marathon.url', None, missing_env) def test_parse_error(): env = os.environ.copy() path = os.path.join('tests', 'data', 'config', 'parse_error.toml') env['DCOS_CONFIG'] = path assert_command(['dcos', 'config', 'show'], returncode=1, stderr=six.b(("Error parsing config file at [{}]: Found " "invalid character in key name: ']'. " "Try quoting the key name.\n").format(path)), env=env) def _fail_url_validation(command, key, value, env): returncode_, stdout_, stderr_ = exec_command( ['dcos', 'config', command, key, value], env=env) assert returncode_ == 1 assert stdout_ == b'' assert stderr_.startswith(str( 'Unable to parse {!r} as a url'.format(value)).encode('utf-8')) def _fail_validation(command, key, value, env): returncode_, stdout_, stderr_ = exec_command( ['dcos', 'config', command, key, value], env=env) assert returncode_ == 1 assert stdout_ == b'' assert stderr_.startswith(str( 'Error: {!r} does not match'.format(value)).encode('utf-8')) def _append_value(key, value, env): assert_command( ['dcos', 'config', 'append', key, value], env=env) def _prepend_value(key, value, env): assert_command( ['dcos', 'config', 'prepend', key, value], env=env) def _get_value(key, value, env): returncode, stdout, stderr = exec_command( ['dcos', 'config', 'show', key], env) if isinstance(value, six.string_types): result = json.loads('"' + stdout.decode('utf-8').strip() + '"') else: result = json.loads(stdout.decode('utf-8').strip()) assert returncode == 0 assert result == value assert stderr == b'' def _get_missing_value(key, env): returncode, stdout, stderr = exec_command( ['dcos', 'config', 'show', key], env) assert returncode == 1 assert stdout == b'' assert (stderr.decode('utf-8') == "Property {!r} doesn't exist\n".format(key)) PKցFHD$tests/integrations/test_dcos.pyfrom .common import assert_command, exec_command def test_default(): returncode, stdout, stderr = exec_command(['dcos']) assert returncode == 0 assert stdout == """\ Command line utility for the Mesosphere Datacenter Operating System (DCOS). The Mesosphere DCOS is a distributed operating system built around Apache Mesos. This utility provides tools for easy management of a DCOS installation. Available DCOS commands: \tconfig \tGet and set DCOS CLI configuration properties \thelp \tDisplay command line usage information \tmarathon \tDeploy and manage applications on the DCOS \tnode \tManage DCOS nodes \tpackage \tInstall and manage DCOS packages \tservice \tManage DCOS services \ttask \tManage DCOS tasks Get detailed command description with 'dcos --help'. """.encode('utf-8') assert stderr == b'' def test_help(): with open('tests/data/help/dcos.txt') as content: assert_command(['dcos', '--help'], stdout=content.read().encode('utf-8')) def test_version(): assert_command(['dcos', '--version'], stdout=b'dcos version SNAPSHOT\n') def test_log_level_flag(): returncode, stdout, stderr = exec_command( ['dcos', '--log-level=info', 'config', '--info']) assert returncode == 0 assert stdout == b"Get and set DCOS CLI configuration properties\n" def test_capital_log_level_flag(): returncode, stdout, stderr = exec_command( ['dcos', '--log-level=INFO', 'config', '--info']) assert returncode == 0 assert stdout == b"Get and set DCOS CLI configuration properties\n" def test_invalid_log_level_flag(): stdout = (b"Log level set to an unknown value 'blah'. Valid " b"values are ['debug', 'info', 'warning', 'error', " b"'critical']\n") assert_command( ['dcos', '--log-level=blah', 'config', '--info'], returncode=1, stdout=stdout) PKցFHlnh;h;tests/integrations/common.pyimport collections import contextlib import json import os import pty import subprocess import sys import time import requests import six from dcos import util import mock from six.moves import urllib def exec_command(cmd, env=None, stdin=None): """Execute CLI command :param cmd: Program and arguments :type cmd: [str] :param env: Environment variables :type env: dict :param stdin: File to use for stdin :type stdin: file :returns: A tuple with the returncode, stdout and stderr :rtype: (int, bytes, bytes) """ print('CMD: {!r}'.format(cmd)) process = subprocess.Popen( cmd, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) # This is needed to get rid of '\r' from Windows's lines endings. stdout, stderr = [std_stream.replace(b'\r', b'') for std_stream in process.communicate()] # We should always print the stdout and stderr print('STDOUT: {}'.format(_truncate(stdout.decode('utf-8')))) print('STDERR: {}'.format(_truncate(stderr.decode('utf-8')))) return (process.returncode, stdout, stderr) def _truncate(s, length=8000): if len(s) > length: return s[:length-3] + '...' else: return s def assert_command( cmd, returncode=0, stdout=b'', stderr=b'', env=None, stdin=None): """Execute CLI command and assert expected behavior. :param cmd: Program and arguments :type cmd: list of str :param returncode: Expected return code :type returncode: int :param stdout: Expected stdout :type stdout: str :param stderr: Expected stderr :type stderr: str :param env: Environment variables :type env: dict of str to str :param stdin: File to use for stdin :type stdin: file :rtype: None """ returncode_, stdout_, stderr_ = exec_command(cmd, env, stdin) assert returncode_ == returncode assert stdout_ == stdout assert stderr_ == stderr def exec_mock(main, args): """Call a main function with sys.args mocked, and capture stdout/stderr :param main: main function to call :type main: function :param args: sys.args to mock, excluding the initial 'dcos' :type args: [str] :returns: (returncode, stdout, stderr) :rtype: (int, bytes, bytes) """ print('MOCK ARGS: {}'.format(' '.join(args))) with mock_args(args) as (stdout, stderr): returncode = main() stdout_val = six.b(stdout.getvalue()) stderr_val = six.b(stderr.getvalue()) print('STDOUT: {}'.format(stdout_val)) print('STDERR: {}'.format(stderr_val)) return (returncode, stdout_val, stderr_val) def assert_mock(main, args, returncode=0, stdout=b'', stderr=b''): """Mock and call a main function, and assert expected behavior. :param main: main function to call :type main: function :param args: sys.args to mock, excluding the initial 'dcos' :type args: [str] :type returncode: int :param stdout: Expected stdout :type stdout: str :param stderr: Expected stderr :type stderr: str :rtype: None """ returncode_, stdout_, stderr_ = exec_mock(main, args) assert returncode_ == returncode assert stdout_ == stdout assert stderr_ == stderr def mock_called_some_args(mock, *args, **kwargs): """Convience method for some mock assertions. Returns True if the arguments to one of the calls of `mock` contains `args` and `kwargs`. :param mock: the mock to check :type mock: mock.Mock :returns: True if the arguments to one of the calls for `mock` contains `args` and `kwargs`. :rtype: bool """ for call in mock.call_args_list: call_args, call_kwargs = call if any(arg not in call_args for arg in args): continue if any(k not in call_kwargs or call_kwargs[k] != v for k, v in kwargs.items()): continue return True return False def watch_deployment(deployment_id, count): """ Wait for a deployment to complete. :param deployment_id: deployment id :type deployment_id: str :param count: max number of seconds to wait :type count: int :rtype: None """ returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'deployment', 'watch', '--max-count={}'.format(count), deployment_id]) assert returncode == 0 assert stderr == b'' def watch_all_deployments(count=300): """ Wait for all deployments to complete. :param count: max number of seconds to wait :type count: int :rtype: None """ deps = list_deployments() for dep in deps: watch_deployment(dep['id'], count) def wait_for_service(service_name, number_of_services=1, max_count=300): """Wait for service to register with Mesos :param service_name: name of service :type service_name: str :param number_of_services: number of services with that name :type number_of_services: int :param max_count: max number of seconds to wait :type max_count: int :rtype: None """ count = 0 while count < max_count: services = get_services() if (len([service for service in services if service['name'] == service_name]) >= number_of_services): return count += 1 def add_app(app_path, wait=True): """ Add an app, and wait for it to deploy :param app_path: path to app's json definition :type app_path: str :param wait: whether to wait for the deploy :type wait: bool :rtype: None """ assert_command(['dcos', 'marathon', 'app', 'add', app_path]) if wait: watch_all_deployments() def remove_group(group_id): assert_command(['dcos', 'marathon', 'group', 'remove', group_id]) # Let's make sure that we don't return until the deployment has finished watch_all_deployments() def remove_app(app_id): """ Remove an app :param app_id: id of app to remove :type app_id: str :rtype: None """ assert_command(['dcos', 'marathon', 'app', 'remove', app_id]) def package_install(package, deploy=False, args=[]): """ Calls `dcos package install` :param package: name of the package to install :type package: str :param deploy: whether or not to wait for the deploy :type deploy: bool :param args: extra CLI args :type args: [str] :rtype: None """ returncode, stdout, stderr = exec_command( ['dcos', 'package', 'install', '--yes', package] + args) assert returncode == 0 assert stderr == b'' if deploy: watch_all_deployments() def package_uninstall(package, args=[], stderr=b''): """ Calls `dcos package uninstall` :param package: name of the package to uninstall :type package: str :param args: extra CLI args :type args: [str] :param stderr: expected string in stderr for package uninstall :type stderr: str :rtype: None """ assert_command( ['dcos', 'package', 'uninstall', package] + args, stderr=stderr) def get_services(expected_count=None, args=[]): """Get services :param expected_count: assert exactly this number of services are running :type expected_count: int | None :param args: cli arguments :type args: [str] :returns: services :rtype: [dict] """ returncode, stdout, stderr = exec_command( ['dcos', 'service', '--json'] + args) assert returncode == 0 assert stderr == b'' services = json.loads(stdout.decode('utf-8')) assert isinstance(services, collections.Sequence) if expected_count is not None: assert len(services) == expected_count return services def list_deployments(expected_count=None, app_id=None): """Get all active deployments. :param expected_count: assert that number of active deployments equals `expected_count` :type expected_count: int :param app_id: only get deployments for this app :type app_id: str :returns: active deployments :rtype: [dict] """ cmd = ['dcos', 'marathon', 'deployment', 'list', '--json'] if app_id is not None: cmd.append(app_id) returncode, stdout, stderr = exec_command(cmd) result = json.loads(stdout.decode('utf-8')) assert returncode == 0 if expected_count is not None: assert len(result) == expected_count assert stderr == b'' return result def show_app(app_id, version=None): """Show details of a Marathon application. :param app_id: The id for the application :type app_id: str :param version: The version, either absolute (date-time) or relative :type version: str :returns: The requested Marathon application :rtype: dict """ if version is None: cmd = ['dcos', 'marathon', 'app', 'show', app_id] else: cmd = ['dcos', 'marathon', 'app', 'show', '--app-version={}'.format(version), app_id] returncode, stdout, stderr = exec_command(cmd) assert returncode == 0 assert stderr == b'' result = json.loads(stdout.decode('utf-8')) assert isinstance(result, dict) assert result['id'] == '/' + app_id return result def service_shutdown(service_id): """Shuts down a service using the command line program :param service_id: the id of the service :type: service_id: str :rtype: None """ assert_command(['dcos', 'service', 'shutdown', service_id]) def delete_zk_nodes(): """Delete Zookeeper nodes that were created during the tests :rtype: None """ for znode in ['universe', 'cassandra-mesos', 'chronos']: delete_zk_node(znode) def delete_zk_node(znode): """Delete Zookeeper node :param znode: znode to delete :type znode: str :rtype: None """ dcos_url = util.get_config_vals(['core.dcos_url'])[0] znode_url = urllib.parse.urljoin( dcos_url, '/exhibitor/exhibitor/v1/explorer/znode/{}'.format(znode)) requests.delete(znode_url) def assert_lines(cmd, num_lines): """ Assert stdout contains the expected number of lines :param cmd: program and arguments :type cmd: [str] :param num_lines: expected number of lines for stdout :type num_lines: int :rtype: None """ returncode, stdout, stderr = exec_command(cmd) assert returncode == 0 assert stderr == b'' assert len(stdout.decode('utf-8').split('\n')) - 1 == num_lines def file_bytes(path): """ Read all bytes from a file :param path: path to file :type path: str :rtype: bytes :returns: bytes from the file """ with open(path) as f: return six.b(f.read()) def file_json(path): """ Returns formatted json from file :param path: path to file :type path: str :returns: formatted json as a string :rtype: bytes """ with open(path) as f: return six.b( json.dumps(json.load(f), sort_keys=True, indent=2, separators=(',', ': '))) + b'\n' @contextlib.contextmanager def app(path, app_id, wait=True): """Context manager that deploys an app on entrance, and removes it on exit. :param path: path to app's json definition: :type path: str :param app_id: app id :type app_id: str :param wait: whether to wait for the deploy :type wait: bool :rtype: None """ add_app(path, wait) try: yield finally: remove_app(app_id) watch_all_deployments() @contextlib.contextmanager def package(package_name, deploy=False, args=[]): """Context manager that deploys an app on entrance, and removes it on exit. :param package_name: package name :type package_name: str :param deploy: If True, block on the deploy :type deploy: bool :rtype: None """ package_install(package_name, deploy, args) try: yield finally: package_uninstall(package_name) watch_all_deployments() @contextlib.contextmanager def mock_args(args): """ Context manager that mocks sys.args and captures stdout/stderr :param args: sys.args values to mock :type args: [str] :rtype: None """ with mock.patch('sys.argv', [util.which('dcos')] + args): stdout, stderr = sys.stdout, sys.stderr sys.stdout, sys.stderr = six.StringIO(), six.StringIO() try: yield sys.stdout, sys.stderr finally: sys.stdout, sys.stderr = stdout, stderr def popen_tty(cmd): """Open a process with stdin connected to a pseudo-tty. Returns a :param cmd: command to run :type cmd: str :returns: (Popen, master) tuple, where master is the master side of the of the tty-pair. It is the responsibility of the caller to close the master fd, and to perform any cleanup (including waiting for completion) of the Popen object. :rtype: (Popen, int) """ master, slave = pty.openpty() proc = subprocess.Popen(cmd, stdin=slave, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid, close_fds=True, shell=True) os.close(slave) return (proc, master) def ssh_output(cmd): """ Runs an SSH command and returns the stdout/stderr/returncode. :param cmd: command to run :type cmd: str :rtype: (str, str, int) """ print('SSH COMMAND: {}'.format(cmd)) # ssh must run with stdin attached to a tty proc, master = popen_tty(cmd) # wait for the ssh connection time.sleep(8) proc.poll() returncode = proc.returncode # kill the whole process group try: os.killpg(os.getpgid(proc.pid), 15) except OSError: pass os.close(master) stdout, stderr = proc.communicate() print('SSH STDOUT: {}'.format(stdout.decode('utf-8'))) print('SSH STDERR: {}'.format(stderr.decode('utf-8'))) return stdout, stderr, returncode def config_set(key, value, env=None): """ dcos config set :param key: :type key: str :param value: :type value: str ;param env: env vars :type env: dict :rtype: None """ returncode, _, stderr = exec_command( ['dcos', 'config', 'set', key, value], env=env) assert returncode == 0 assert stderr == b'' def config_unset(key, index=None, env=None): """ dcos config unset --index= :param key: :type key: str :param index: :type index: str :param env: env vars :type env: dict :rtype: None """ cmd = ['dcos', 'config', 'unset', key] if index is not None: cmd.append('--index={}'.format(index)) returncode, stdout, stderr = exec_command(cmd, env=env) assert returncode == 0 assert stderr == b'' PKցFHWWtests/integrations/test_auth.pyimport os import webbrowser from dcos import auth, constants, util from dcoscli.main import main from mock import Mock, patch def test_no_browser_auth(): webbrowser.get = Mock(side_effect=webbrowser.Error()) with patch('webbrowser.open') as op: _mock_dcos_run([util.which('dcos')], False) assert op.call_count == 0 def test_when_authenticated(): with patch('dcos.auth.force_auth'): _mock_dcos_run([util.which('dcos')], True) assert auth.force_auth.call_count == 0 def test_anonymous_login(): with patch('sys.stdin.readline', return_value='\n'), \ patch('uuid.uuid1', return_value='anonymous@email'): assert _mock_dcos_run([util.which('dcos'), 'help'], False) == 0 assert _mock_dcos_run([util.which('dcos'), 'config', 'show', 'core.email'], False) == 0 assert _mock_dcos_run([util.which('dcos'), 'config', 'unset', 'core.email'], False) == 0 def _mock_dcos_run(args, authenticated=True): if authenticated: env = _config_with_credentials() else: env = _config_without_credentials() with patch('sys.argv', args), patch.dict(os.environ, env): return main() def _config_with_credentials(): return { constants.DCOS_CONFIG_ENV: os.path.join( 'tests', 'data', 'auth', 'dcos_with_credentials.toml') } def _config_without_credentials(): return { constants.DCOS_CONFIG_ENV: os.path.join( 'tests', 'data', 'auth', 'dcos_without_credentials.toml') } PKցFHJ$  tests/integrations/test_ssl.pyimport os import dcoscli.constants as cli_constants from dcos import constants import pytest from .common import config_set, config_unset, exec_command @pytest.fixture def env(): r = os.environ.copy() r.update({ constants.PATH_ENV: os.environ[constants.PATH_ENV], constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "ssl", "ssl.toml"), cli_constants.DCOS_PRODUCTION_ENV: 'false' }) return r def test_dont_verify_ssl_with_env_var(env): env[constants.DCOS_SSL_VERIFY_ENV] = 'false' returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'list'], env) assert returncode == 0 assert stderr == b'' env.pop(constants.DCOS_SSL_VERIFY_ENV) def test_dont_verify_ssl_with_config(env): config_set('core.ssl_verify', 'false', env) returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'list'], env) assert returncode == 0 assert stderr == b'' config_unset('core.ssl_verify', None, env) def test_verify_ssl_without_cert_env_var(env): env[constants.DCOS_SSL_VERIFY_ENV] = 'true' returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'list'], env) assert returncode == 1 assert "certificate verify failed" in stderr.decode('utf-8') env.pop(constants.DCOS_SSL_VERIFY_ENV) def test_verify_ssl_without_cert_config(env): config_set('core.ssl_verify', 'true', env) returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'list'], env) assert returncode == 1 assert "certificate verify failed" in stderr.decode('utf-8') config_unset('core.ssl_verify', None, env) def test_verify_ssl_with_bad_cert_env_var(env): env[constants.DCOS_SSL_VERIFY_ENV] = 'tests/data/ssl/fake.pem' returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'list'], env) assert returncode == 1 assert "PEM lib" in stderr.decode('utf-8') # wrong private key env.pop(constants.DCOS_SSL_VERIFY_ENV) def test_verify_ssl_with_bad_cert_config(env): config_set('core.ssl_verify', 'tests/data/ssl/fake.pem', env) returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'list'], env) assert returncode == 1 assert "PEM lib" in stderr.decode('utf-8') # wrong private key config_unset('core.ssl_verify', None, env) def test_verify_ssl_with_good_cert_env_var(env): env[constants.DCOS_SSL_VERIFY_ENV] = '/dcos-cli/adminrouter/snakeoil.crt' returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'list'], env) assert returncode == 0 assert stderr == b'' env.pop(constants.DCOS_SSL_VERIFY_ENV) def test_verify_ssl_with_good_cert_config(env): config_set('core.ssl_verify', '/dcos-cli/adminrouter/snakeoil.crt', env) returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'list'], env) assert returncode == 0 assert stderr == b'' config_unset('core.ssl_verify', None, env) PKցFHiIF  *tests/integrations/test_marathon_groups.pyimport contextlib import json from .common import (assert_command, assert_lines, exec_command, remove_group, show_app, watch_all_deployments) GOOD_GROUP = 'tests/data/marathon/groups/good.json' def test_deploy_group(): _deploy_group(GOOD_GROUP) remove_group('test-group') def test_group_list_table(): with _group(GOOD_GROUP, 'test-group'): assert_lines(['dcos', 'marathon', 'group', 'list'], 3) def test_validate_complicated_group_and_app(): _deploy_group('tests/data/marathon/groups/complicated.json') remove_group('test-group') def test_optional_deploy_group(): _deploy_group(GOOD_GROUP, False) remove_group('test-group') def test_add_existing_group(): with _group(GOOD_GROUP, 'test-group'): with open(GOOD_GROUP) as fd: stderr = b"Group '/test-group' already exists\n" assert_command(['dcos', 'marathon', 'group', 'add'], returncode=1, stderr=stderr, stdin=fd) def test_show_group(): with _group(GOOD_GROUP, 'test-group'): _show_group('test-group') def test_add_bad_complicated_group(): with open('tests/data/marathon/groups/complicated_bad.json') as fd: returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'group', 'add'], stdin=fd) err = b"""{ "details": [ { "errors": [ "error.path.missing" ], "path": "/groups(0)/apps(0)/id" } ], "message": "Invalid JSON" } """ assert returncode == 1 assert stdout == b'' assert err in stderr def test_update_group(): with _group(GOOD_GROUP, 'test-group'): newapp = json.dumps([{"id": "appadded", "cmd": "sleep 0"}]) appjson = "apps={}".format(newapp) returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'group', 'update', 'test-group/sleep', appjson]) assert returncode == 0 assert stdout.decode().startswith('Created deployment ') assert stderr == b'' watch_all_deployments() show_app('test-group/sleep/appadded') def test_update_group_from_stdin(): with _group(GOOD_GROUP, 'test-group'): _update_group( 'test-group', 'tests/data/marathon/groups/update_good.json') show_app('test-group/updated') def test_update_missing_group(): assert_command(['dcos', 'marathon', 'group', 'update', 'missing-id'], stderr=b"Error: Group '/missing-id' does not exist\n", returncode=1) def test_scale_group(): _deploy_group('tests/data/marathon/groups/scale.json') returncode, stdout, stderr = exec_command(['dcos', 'marathon', 'group', 'scale', 'scale-group', '2']) assert stderr == b'' assert returncode == 0 watch_all_deployments() returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'group', 'show', 'scale-group']) res = json.loads(stdout.decode('utf-8')) assert res['groups'][0]['apps'][0]['instances'] == 2 remove_group('scale-group') def test_scale_group_not_exist(): returncode, stdout, stderr = exec_command(['dcos', 'marathon', 'group', 'scale', 'scale-group', '2']) assert stderr == b'' watch_all_deployments() returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'group', 'show', 'scale-group']) res = json.loads(stdout.decode('utf-8')) assert len(res['apps']) == 0 remove_group('scale-group') def test_scale_group_when_scale_factor_negative(): _deploy_group('tests/data/marathon/groups/scale.json') returncode, stdout, stderr = exec_command(['dcos', 'marathon', 'group', 'scale', 'scale-group', '-2']) assert b'Command not recognized' in stdout assert returncode == 1 watch_all_deployments() remove_group('scale-group') def test_scale_group_when_scale_factor_not_float(): _deploy_group('tests/data/marathon/groups/scale.json') returncode, stdout, stderr = exec_command(['dcos', 'marathon', 'group', 'scale', 'scale-group', '1.a']) assert stderr == b'Error parsing string as float\n' assert returncode == 1 watch_all_deployments() remove_group('scale-group') def _deploy_group(file_path, stdin=True): if stdin: with open(file_path) as fd: assert_command(['dcos', 'marathon', 'group', 'add'], stdin=fd) else: assert_command(['dcos', 'marathon', 'group', 'add', file_path]) # Let's make sure that we don't return until the deployment has finished watch_all_deployments() def _show_group(group_id, version=None): if version is None: cmd = ['dcos', 'marathon', 'group', 'show', group_id] else: cmd = ['dcos', 'marathon', 'group', 'show', '--group-version={}'.format(version), group_id] returncode, stdout, stderr = exec_command(cmd) result = json.loads(stdout.decode('utf-8')) assert returncode == 0 assert isinstance(result, dict) assert result['id'] == '/' + group_id assert stderr == b'' return result def _update_group(group_id, file_path): with open(file_path) as fd: returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'group', 'update', group_id], stdin=fd) assert returncode == 0 assert stdout.decode().startswith('Created deployment ') assert stderr == b'' # Let's make sure that we don't return until the deployment has finished watch_all_deployments() @contextlib.contextmanager def _group(path, group_id): """Context manager that deploys a group on entrance, and removes it on exit. :param path: path to group's json definition :type path: str :param group_id: group id :type group_id: str :rtype: None """ _deploy_group(path) try: yield finally: remove_group(group_id) PKցFH$tests/integrations/test_http_auth.pyimport copy from dcos import http from dcos.errors import DCOSException from requests.auth import HTTPBasicAuth import pytest from mock import Mock, patch from six.moves.urllib.parse import urlparse def test_get_auth_scheme_basic(): with patch('requests.Response') as mock: mock.headers = {'www-authenticate': 'Basic realm="Restricted"'} auth_scheme, realm = http.get_auth_scheme(mock) assert auth_scheme == "basic" assert realm == "restricted" def test_get_auth_scheme_acs(): with patch('requests.Response') as mock: mock.headers = {'www-authenticate': 'acsjwt'} auth_scheme, realm = http.get_auth_scheme(mock) assert auth_scheme == "acsjwt" assert realm == "acsjwt" def test_get_auth_scheme_bad_request(): with patch('requests.Response') as mock: mock.headers = {'www-authenticate': ''} res = http.get_auth_scheme(mock) assert res is None @patch('requests.Response') def test_get_http_auth_not_supported(mock): mock.headers = {'www-authenticate': 'test'} mock.url = '' with pytest.raises(DCOSException) as e: http._get_http_auth(mock, url=urlparse(''), auth_scheme='foo') msg = ("Server responded with an HTTP 'www-authenticate' field of " "'test', DCOS only supports 'Basic'") assert e.exconly().split(':')[1].strip() == msg @patch('requests.Response') def test_get_http_auth_bad_response(mock): mock.headers = {} mock.url = '' with pytest.raises(DCOSException) as e: http._get_http_auth(mock, url=urlparse(''), auth_scheme='') msg = ("Invalid HTTP response: server returned an HTTP 401 response " "with no 'www-authenticate' field") assert e.exconly().split(':', 1)[1].strip() == msg @patch('dcos.http._get_auth_credentials') def test_get_http_auth_credentials_basic(auth_mock): m = Mock() m.url = 'http://domain.com' m.headers = {'www-authenticate': 'Basic realm="Restricted"'} auth_mock.return_value = ("username", "password") returned_auth = http._get_http_auth(m, urlparse(m.url), "basic") assert type(returned_auth) == HTTPBasicAuth assert returned_auth.username == "username" assert returned_auth.password == "password" @patch('dcos.http._get_auth_credentials') def test_get_http_auth_credentials_acl(auth_mock): m = Mock() m.url = 'http://domain.com' m.headers = {'www-authenticate': 'acsjwt"'} auth_mock.return_value = ("username", "password") returned_auth = http._get_http_auth(m, urlparse(m.url), "acsjwt") assert type(returned_auth) == http.DCOSAcsAuth @patch('requests.Response') @patch('dcos.http._request') @patch('dcos.http._get_http_auth') def test_request_with_bad_auth_basic(mock, req_mock, auth_mock): mock.url = 'http://domain.com' mock.headers = {'www-authenticate': 'Basic realm="Restricted"'} mock.status_code = 401 auth_mock.return_value = HTTPBasicAuth("username", "password") req_mock.return_value = mock with pytest.raises(DCOSException) as e: http._request_with_auth(mock, "method", mock.url) assert e.exconly().split(':')[1].strip() == "Authentication failed" @patch('requests.Response') @patch('dcos.http._request') @patch('dcos.http._get_http_auth') def test_request_with_bad_auth_acl(mock, req_mock, auth_mock): mock.url = 'http://domain.com' mock.headers = {'www-authenticate': 'acsjwt'} mock.status_code = 401 auth_mock.return_value = http.DCOSAcsAuth("token") req_mock.return_value = mock with pytest.raises(DCOSException) as e: http._request_with_auth(mock, "method", mock.url) assert e.exconly().split(':')[1].strip() == "Authentication failed" @patch('requests.Response') @patch('dcos.http._request') @patch('dcos.http._get_http_auth') def test_request_with_auth_basic(mock, req_mock, auth_mock): mock.url = 'http://domain.com' mock.headers = {'www-authenticate': 'Basic realm="Restricted"'} mock.status_code = 401 auth = HTTPBasicAuth("username", "password") auth_mock.return_value = auth mock2 = copy.deepcopy(mock) mock2.status_code = 200 req_mock.return_value = mock2 response = http._request_with_auth(mock, "method", mock.url) assert response.status_code == 200 @patch('requests.Response') @patch('dcos.http._request') @patch('dcos.http._get_http_auth') def test_request_with_auth_acl(mock, req_mock, auth_mock): mock.url = 'http://domain.com' mock.headers = {'www-authenticate': 'acsjwt'} mock.status_code = 401 auth = http.DCOSAcsAuth("token") auth_mock.return_value = auth mock2 = copy.deepcopy(mock) mock2.status_code = 200 req_mock.return_value = mock2 response = http._request_with_auth(mock, "method", mock.url) assert response.status_code == 200 PKցFHM$tests/integrations/test_analytics.pyimport os from functools import wraps import dcoscli.analytics import rollbar from dcos import constants, http, util from dcoscli.analytics import _base_properties from dcoscli.config.main import main as config_main from dcoscli.constants import (SEGMENT_IO_CLI_ERROR_EVENT, SEGMENT_IO_CLI_EVENT, SEGMENT_URL) from dcoscli.main import main from mock import patch from .common import mock_called_some_args ANON_ID = 0 USER_ID = 'test@mail.com' def _mock(fn): @wraps(fn) def wrapper(*args, **kwargs): with patch('rollbar.init'), \ patch('rollbar.report_message'), \ patch('dcos.http.post'), \ patch('dcos.http.get'), \ patch('dcoscli.analytics.session_id'): dcoscli.analytics.session_id = ANON_ID fn() return wrapper @_mock def test_config_set(): '''Tests that a `dcos config set core.email ` makes a segment.io identify call''' args = [util.which('dcos'), 'config', 'set', 'core.email', 'test@mail.com'] env = _env_reporting() with patch('sys.argv', args), patch.dict(os.environ, env): assert config_main() == 0 # segment.io assert mock_called_some_args(http.post, '{}/identify'.format(SEGMENT_URL), json={'userId': 'test@mail.com'}, timeout=(1, 1)) @_mock def test_cluster_id_sent(): '''Tests that cluster_id is sent to segment.io''' args = [util.which('dcos')] env = _env_reporting_with_url() version = 'release' with patch('sys.argv', args), \ patch.dict(os.environ, env), \ patch('dcoscli.version', version): assert main() == 0 props = _base_properties() # segment.io data = {'userId': USER_ID, 'event': SEGMENT_IO_CLI_EVENT, 'properties': props} assert props.get('CLUSTER_ID') assert mock_called_some_args(http.post, '{}/track'.format(SEGMENT_URL), json=data, timeout=(1, 1)) @_mock def test_no_exc(): '''Tests that a command which does not raise an exception does not report an exception. ''' args = [util.which('dcos')] env = _env_reporting() version = 'release' with patch('sys.argv', args), \ patch.dict(os.environ, env), \ patch('dcoscli.version', version): assert main() == 0 # segment.io data = {'userId': USER_ID, 'event': SEGMENT_IO_CLI_EVENT, 'properties': _base_properties()} assert mock_called_some_args(http.post, '{}/track'.format(SEGMENT_URL), json=data, timeout=(1, 1)) # rollbar assert rollbar.report_message.call_count == 0 @_mock def test_exc(): '''Tests that a command which does raise an exception does report an exception. ''' args = [util.which('dcos')] env = _env_reporting() version = 'release' with patch('sys.argv', args), \ patch('dcoscli.version', version), \ patch.dict(os.environ, env), \ patch('dcoscli.analytics.wait_and_capture', return_value=(1, 'Traceback')): assert main() == 1 # segment.io props = _base_properties() props['err'] = 'Traceback' props['exit_code'] = 1 data = {'userId': USER_ID, 'event': SEGMENT_IO_CLI_ERROR_EVENT, 'properties': props} assert mock_called_some_args(http.post, '{}/track'.format(SEGMENT_URL), json=data, timeout=(1, 1)) # rollbar props = _base_properties() props['exit_code'] = 1 props['stderr'] = 'Traceback' rollbar.report_message.assert_called_with('Traceback', 'error', extra_data=props) @_mock def test_config_reporting_false(): '''Test that "core.reporting = false" blocks exception reporting.''' args = [util.which('dcos')] env = _env_no_reporting() version = 'release' with patch('sys.argv', args), \ patch('dcoscli.version', version), \ patch.dict(os.environ, env), \ patch('dcoscli.analytics.wait_and_capture', return_value=(1, 'Traceback')): assert main() == 1 assert rollbar.report_message.call_count == 0 assert http.post.call_count == 0 assert http.get.call_count == 0 def _env_reporting(): path = os.path.join('tests', 'data', 'analytics', 'dcos_reporting.toml') return {constants.DCOS_CONFIG_ENV: path} def _env_no_reporting(): path = os.path.join('tests', 'data', 'analytics', 'dcos_no_reporting.toml') return {constants.DCOS_CONFIG_ENV: path} def _env_reporting_with_url(): path = os.path.join('tests', 'data', 'analytics', 'dcos_reporting_with_url.toml') return {constants.DCOS_CONFIG_ENV: path} PKցFH2 tests/integrations/test_help.pyfrom .common import assert_command def test_help(): with open('tests/data/help/help.txt') as content: assert_command(['dcos', 'help', '--help'], stdout=content.read().encode('utf-8')) def test_info(): assert_command(['dcos', 'help', '--info'], stdout=b'Display command line usage information\n') def test_version(): assert_command(['dcos', 'help', '--version'], stdout=b'dcos-help version SNAPSHOT\n') def test_list(): stdout = """\ Command line utility for the Mesosphere Datacenter Operating System (DCOS). The Mesosphere DCOS is a distributed operating system built around Apache Mesos. This utility provides tools for easy management of a DCOS installation. Available DCOS commands: \tconfig \tGet and set DCOS CLI configuration properties \thelp \tDisplay command line usage information \tmarathon \tDeploy and manage applications on the DCOS \tnode \tManage DCOS nodes \tpackage \tInstall and manage DCOS packages \tservice \tManage DCOS services \ttask \tManage DCOS tasks Get detailed command description with 'dcos --help'. """.encode('utf-8') assert_command(['dcos', 'help'], stdout=stdout) def test_help_config(): with open('tests/data/help/config.txt') as content: assert_command(['dcos', 'help', 'config'], stdout=content.read().encode('utf-8')) def test_help_marathon(): with open('tests/data/help/marathon.txt') as content: assert_command(['dcos', 'help', 'marathon'], stdout=content.read().encode('utf-8')) def test_help_node(): with open('tests/data/help/node.txt') as content: assert_command(['dcos', 'help', 'node'], stdout=content.read().encode('utf-8')) def test_help_package(): with open('tests/data/help/package.txt') as content: assert_command(['dcos', 'help', 'package'], stdout=content.read().encode('utf-8')) def test_help_service(): with open('tests/data/help/service.txt') as content: assert_command(['dcos', 'help', 'service'], stdout=content.read().encode('utf-8')) def test_help_task(): with open('tests/data/help/task.txt') as content: assert_command(['dcos', 'help', 'task'], stdout=content.read().encode('utf-8')) PKցFHU#tests/integrations/test_node.pyimport json import os import re import dcos.util as util import six from dcos import mesos from dcos.util import create_schema from ..fixtures.node import slave_fixture from .common import assert_command, assert_lines, exec_command, ssh_output def test_help(): with open('tests/data/help/node.txt') as content: stdout = six.b(content.read()) assert_command(['dcos', 'node', '--help'], stdout=stdout) def test_info(): stdout = b"Manage DCOS nodes\n" assert_command(['dcos', 'node', '--info'], stdout=stdout) def test_node(): returncode, stdout, stderr = exec_command(['dcos', 'node', '--json']) assert returncode == 0 assert stderr == b'' nodes = json.loads(stdout.decode('utf-8')) schema = _get_schema(slave_fixture()) for node in nodes: assert not util.validate_json(node, schema) def test_node_table(): returncode, stdout, stderr = exec_command(['dcos', 'node']) assert returncode == 0 assert stderr == b'' assert len(stdout.decode('utf-8').split('\n')) > 2 def test_node_log_empty(): stderr = b"You must choose one of --master or --slave.\n" assert_command(['dcos', 'node', 'log'], returncode=1, stderr=stderr) def test_node_log_master(): assert_lines(['dcos', 'node', 'log', '--master'], 10) def test_node_log_slave(): slave_id = _node()[0]['id'] assert_lines(['dcos', 'node', 'log', '--slave={}'.format(slave_id)], 10) def test_node_log_missing_slave(): returncode, stdout, stderr = exec_command( ['dcos', 'node', 'log', '--slave=bogus']) assert returncode == 1 assert stdout == b'' assert stderr == b'No slave found with ID "bogus".\n' def test_node_log_master_slave(): slave_id = _node()[0]['id'] returncode, stdout, stderr = exec_command( ['dcos', 'node', 'log', '--master', '--slave={}'.format(slave_id)]) assert returncode == 0 assert stderr == b'' lines = stdout.decode('utf-8').split('\n') assert len(lines) == 23 assert re.match('===>.*<===', lines[0]) assert re.match('===>.*<===', lines[11]) def test_node_log_lines(): assert_lines(['dcos', 'node', 'log', '--master', '--lines=4'], 4) def test_node_log_invalid_lines(): assert_command(['dcos', 'node', 'log', '--master', '--lines=bogus'], stdout=b'', stderr=b'Error parsing string as int\n', returncode=1) def test_node_ssh_master(): _node_ssh(['--master']) def test_node_ssh_slave(): slave_id = mesos.DCOSClient().get_state_summary()['slaves'][0]['id'] _node_ssh(['--slave={}'.format(slave_id), '--master-proxy']) def test_node_ssh_option(): stdout, stderr, _ = _node_ssh_output( ['--master', '--option', 'Protocol=0']) assert stdout == b'' assert b'ignoring bad proto spec' in stderr def test_node_ssh_config_file(): stdout, stderr, _ = _node_ssh_output( ['--master', '--config-file', 'tests/data/node/ssh_config']) assert stdout == b'' assert b'ignoring bad proto spec' in stderr def test_node_ssh_user(): stdout, stderr, _ = _node_ssh_output( ['--master-proxy', '--master', '--user=bogus', '--option', 'PasswordAuthentication=no']) assert stdout == b'' assert b'Permission denied' in stderr def test_node_ssh_master_proxy_no_agent(): env = os.environ.copy() env.pop('SSH_AUTH_SOCK', None) stderr = (b"There is no SSH_AUTH_SOCK env variable, which likely means " b"you aren't running `ssh-agent`. `dcos node ssh " b"--master-proxy` depends on `ssh-agent` to safely use your " b"private key to hop between nodes in your cluster. Please " b"run `ssh-agent`, then add your private key with `ssh-add`.\n") assert_command(['dcos', 'node', 'ssh', '--master-proxy', '--master'], stderr=stderr, returncode=1, env=env) def test_node_ssh_master_proxy(): _node_ssh(['--master', '--master-proxy']) def _node_ssh_output(args): cli_test_ssh_key_path = os.environ['CLI_TEST_SSH_KEY_PATH'] cmd = ('ssh-agent /bin/bash -c "ssh-add {} 2> /dev/null && ' + 'dcos node ssh --option StrictHostKeyChecking=no {}"').format( cli_test_ssh_key_path, ' '.join(args)) return ssh_output(cmd) def _node_ssh(args): if os.environ.get('CLI_TEST_MASTER_PROXY') and \ '--master-proxy' not in args: args.append('--master-proxy') stdout, stderr, returncode = _node_ssh_output(args) assert returncode is None assert stdout assert b"Running `" in stderr num_lines = len(stderr.decode().split('\n')) expected_num_lines = 2 if '--master-proxy' in args else 3 assert (num_lines == expected_num_lines or (num_lines == (expected_num_lines + 1) and b'Warning: Permanently added' in stderr)) def _get_schema(slave): schema = create_schema(slave) schema['required'].remove('reregistered_time') schema['required'].remove('reserved_resources') schema['properties']['reserved_resources']['required'] = [] schema['properties']['reserved_resources']['additionalProperties'] = True schema['required'].remove('unreserved_resources') schema['properties']['unreserved_resources']['required'] = [] schema['properties']['unreserved_resources']['additionalProperties'] = True schema['properties']['used_resources']['required'].remove('ports') schema['properties']['offered_resources']['required'].remove('ports') schema['properties']['attributes']['additionalProperties'] = True return schema def _node(): returncode, stdout, stderr = exec_command(['dcos', 'node', '--json']) assert returncode == 0 assert stderr == b'' return json.loads(stdout.decode('utf-8')) PKցFH,|W$W$tests/integrations/test_task.pyimport collections import fcntl import json import os import re import subprocess import time import dcos.util as util from dcos import mesos from dcos.errors import DCOSException from dcos.util import create_schema from dcoscli.task.main import main from mock import MagicMock, patch from ..fixtures.task import task_fixture from .common import (add_app, app, assert_command, assert_lines, assert_mock, exec_command, remove_app, watch_all_deployments) SLEEP_COMPLETED = 'tests/data/marathon/apps/sleep-completed.json' SLEEP1 = 'tests/data/marathon/apps/sleep1.json' SLEEP2 = 'tests/data/marathon/apps/sleep2.json' FOLLOW = 'tests/data/file/follow.json' TWO_TASKS = 'tests/data/file/two_tasks.json' TWO_TASKS_FOLLOW = 'tests/data/file/two_tasks_follow.json' LS = 'tests/data/tasks/ls-app.json' INIT_APPS = ((LS, 'ls-app'), (SLEEP1, 'test-app1'), (SLEEP2, 'test-app2')) NUM_TASKS = len(INIT_APPS) def setup_module(): # create a completed task with app(SLEEP_COMPLETED, 'test-app-completed'): pass for app_ in INIT_APPS: add_app(app_[0]) def teardown_module(): for app_ in INIT_APPS: remove_app(app_[1]) def test_help(): with open('tests/data/help/task.txt') as content: assert_command(['dcos', 'task', '--help'], stdout=content.read().encode('utf-8')) def test_info(): stdout = b"Manage DCOS tasks\n" assert_command(['dcos', 'task', '--info'], stdout=stdout) def test_task(): # test `dcos task` output returncode, stdout, stderr = exec_command(['dcos', 'task', '--json']) assert returncode == 0 assert stderr == b'' tasks = json.loads(stdout.decode('utf-8')) assert isinstance(tasks, collections.Sequence) assert len(tasks) == NUM_TASKS schema = create_schema(task_fixture().dict()) schema['required'].remove('labels') for task in tasks: assert not util.validate_json(task, schema) def test_task_table(): assert_lines(['dcos', 'task'], NUM_TASKS+1) def test_task_completed(): returncode, stdout, stderr = exec_command( ['dcos', 'task', '--completed', '--json']) assert returncode == 0 assert stderr == b'' assert len(json.loads(stdout.decode('utf-8'))) > NUM_TASKS returncode, stdout, stderr = exec_command( ['dcos', 'task', '--json']) assert returncode == 0 assert stderr == b'' assert len(json.loads(stdout.decode('utf-8'))) == NUM_TASKS def test_task_none(): assert_command(['dcos', 'task', 'bogus', '--json'], stdout=b'[]\n') def test_filter(): returncode, stdout, stderr = exec_command( ['dcos', 'task', 'test-app2', '--json']) assert returncode == 0 assert stderr == b'' assert len(json.loads(stdout.decode('utf-8'))) == 1 def test_log_no_files(): """ Tail stdout on nonexistant task """ assert_command(['dcos', 'task', 'log', 'bogus'], returncode=1, stderr=b'No matching tasks. Exiting.\n') def test_log_single_file(): """ Tail a single file on a single task """ returncode, stdout, stderr = exec_command( ['dcos', 'task', 'log', 'test-app1']) assert returncode == 0 assert stderr == b'' assert len(stdout.decode('utf-8').split('\n')) == 5 def test_log_missing_file(): """ Tail a single file on a single task """ returncode, stdout, stderr = exec_command( ['dcos', 'task', 'log', 'test-app', 'bogus']) assert returncode == 1 assert stdout == b'' assert stderr == b'No files exist. Exiting.\n' def test_log_lines(): """ Test --lines """ assert_lines(['dcos', 'task', 'log', 'test-app1', '--lines=2'], 2) def test_log_lines_invalid(): """ Test invalid --lines value """ assert_command(['dcos', 'task', 'log', 'test-app1', '--lines=bogus'], stdout=b'', stderr=b'Error parsing string as int\n', returncode=1) def test_log_follow(): """ Test --follow """ # verify output with app(FOLLOW, 'follow'): proc = subprocess.Popen(['dcos', 'task', 'log', 'follow', '--follow'], stdout=subprocess.PIPE) # mark stdout as non-blocking, so we can read all available data # before EOF _mark_non_blocking(proc.stdout) # wait for data to be output time.sleep(10) # assert lines before and after sleep assert len(proc.stdout.read().decode('utf-8').split('\n')) >= 5 time.sleep(5) assert len(proc.stdout.read().decode('utf-8').split('\n')) >= 3 proc.kill() def test_log_two_tasks(): """ Test tailing a single file on two separate tasks """ returncode, stdout, stderr = exec_command( ['dcos', 'task', 'log', 'test-app']) assert returncode == 0 assert stderr == b'' lines = stdout.decode('utf-8').split('\n') assert len(lines) == 11 assert re.match('===>.*<===', lines[0]) assert re.match('===>.*<===', lines[5]) def test_log_two_tasks_follow(): """ Test tailing a single file on two separate tasks with --follow """ with app(TWO_TASKS_FOLLOW, 'two-tasks-follow'): proc = subprocess.Popen( ['dcos', 'task', 'log', 'two-tasks-follow', '--follow'], stdout=subprocess.PIPE) # mark stdout as non-blocking, so we can read all available data # before EOF _mark_non_blocking(proc.stdout) # wait for data to be output time.sleep(10) # get output before and after the task's sleep first_lines = proc.stdout.read().decode('utf-8').split('\n') time.sleep(5) second_lines = proc.stdout.read().decode('utf-8').split('\n') # assert both tasks have printed the expected amount of output assert len(first_lines) >= 5 # assert there is some difference after sleeping assert len(second_lines) >= 3 proc.kill() def test_log_completed(): """ Test `dcos task log --completed` """ # create a completed task # ensure that tail lists nothing # ensure that tail --completed lists a completed task returncode, stdout, stderr = exec_command( ['dcos', 'task', 'log', 'test-app-completed']) assert returncode == 1 assert stdout == b'' assert stderr.startswith(b'No running tasks match ID [test-app-completed]') returncode, stdout, stderr = exec_command( ['dcos', 'task', 'log', '--completed', 'test-app-completed']) assert returncode == 0 assert stderr == b'' assert len(stdout.decode('utf-8').split('\n')) > 4 def test_log_master_unavailable(): """ Test master's state.json being unavailable """ client = mesos.DCOSClient() client.get_master_state = _mock_exception() with patch('dcos.mesos.DCOSClient', return_value=client): args = ['task', 'log', '_'] assert_mock(main, args, returncode=1, stderr=(b"exception\n")) def test_log_slave_unavailable(): """ Test slave's state.json being unavailable """ client = mesos.DCOSClient() client.get_slave_state = _mock_exception() with patch('dcos.mesos.DCOSClient', return_value=client): args = ['task', 'log', 'test-app1'] stderr = (b"""Error accessing slave: exception\n""" b"""No matching tasks. Exiting.\n""") assert_mock(main, args, returncode=1, stderr=stderr) def test_log_file_unavailable(): """ Test a file's read.json being unavailable """ files = [mesos.MesosFile('bogus')] files[0].read = _mock_exception('exception') with patch('dcoscli.task.main._mesos_files', return_value=files): args = ['task', 'log', 'test-app1'] stderr = b"No files exist. Exiting.\n" assert_mock(main, args, returncode=1, stderr=stderr) def test_ls(): assert_command(['dcos', 'task', 'ls', 'test-app1'], stdout=b'stderr stdout\n') def test_ls_multiple_tasks(): returncode, stdout, stderr = exec_command( ['dcos', 'task', 'ls', 'test-app']) assert returncode == 1 assert stdout == b'' assert stderr.startswith(b'There are multiple tasks with ID matching ' b'[test-app]. Please choose one:\n\t') def test_ls_long(): assert_lines(['dcos', 'task', 'ls', '--long', 'test-app1'], 2) def test_ls_path(): assert_command(['dcos', 'task', 'ls', 'ls-app', 'test'], stdout=b'test1 test2\n') def test_ls_bad_path(): assert_command( ['dcos', 'task', 'ls', 'test-app1', 'bogus'], stderr=b'Cannot access [bogus]: No such file or directory\n', returncode=1) def _mock_exception(contents='exception'): return MagicMock(side_effect=DCOSException(contents)) def _mark_non_blocking(file_): fcntl.fcntl(file_.fileno(), fcntl.F_SETFL, os.O_NONBLOCK) def _install_sleep_task(app_path=SLEEP1, app_name='test-app'): args = ['dcos', 'marathon', 'app', 'add', app_path] assert_command(args) watch_all_deployments() def _uninstall_helloworld(args=[]): assert_command(['dcos', 'package', 'uninstall', 'helloworld'] + args) def _uninstall_sleep(app_id='test-app'): assert_command(['dcos', 'marathon', 'app', 'remove', app_id]) PKցFHp"tests/integrations/test_service.pyimport os import subprocess import time import dcos.util as util from dcos.util import create_schema import pytest from ..fixtures.service import framework_fixture from .common import (assert_command, assert_lines, delete_zk_node, delete_zk_nodes, exec_command, get_services, package_install, package_uninstall, remove_app, service_shutdown, ssh_output, wait_for_service, watch_all_deployments) def setup_module(module): package_install('chronos', True) def teardown_module(module): package_uninstall( 'chronos', stderr=b'Uninstalled package [chronos] version [2.4.0]\n' b'The Chronos DCOS Service has been uninstalled and will no ' b'longer run.\nPlease follow the instructions at http://docs.' b'mesosphere.com/services/chronos/#uninstall to clean up any ' b'persisted state\n') delete_zk_nodes() def test_help(): with open('tests/data/help/service.txt') as content: assert_command(['dcos', 'service', '--help'], stdout=content.read().encode('utf-8')) def test_info(): stdout = b"Manage DCOS services\n" assert_command(['dcos', 'service', '--info'], stdout=stdout) def test_service(): services = get_services(2) schema = _get_schema(framework_fixture()) for srv in services: assert not util.validate_json(srv, schema) def test_service_table(): assert_lines(['dcos', 'service'], 3) def test_service_inactive(): package_install('cassandra', True) # wait long enough for it to register time.sleep(5) # assert marathon, chronos, and cassandra are listed get_services(3) # uninstall cassandra using marathon. For now, need to explicitly remove # the group that is left by cassandra. See MARATHON-144 assert_command(['dcos', 'marathon', 'group', 'remove', '/cassandra']) watch_all_deployments() # I'm not quite sure why we have to sleep, but it seems cassandra # only transitions to "inactive" after a few seconds. time.sleep(5) # assert only marathon and chronos are active get_services(2) # assert marathon, chronos, and cassandra are inactive services = get_services(args=['--inactive']) assert len(services) >= 3 # shutdown the cassandra framework for framework in services: if framework['name'] == 'cassandra.dcos': service_shutdown(framework['id']) # assert marathon, chronos are only listed with --inactive get_services(2, ['--inactive']) delete_zk_node('cassandra-mesos') package_uninstall('cassandra') def test_service_completed(): package_install('cassandra', True) time.sleep(5) services = get_services(3) # get cassandra's framework ID cassandra_id = None for service in services: if service['name'] == 'cassandra.dcos': cassandra_id = service['id'] break assert cassandra_id is not None assert_command(['dcos', 'marathon', 'group', 'remove', '/cassandra']) service_shutdown(cassandra_id) delete_zk_node('cassandra-mesos') # assert cassandra is not running services = get_services(2) assert not any(service['id'] == cassandra_id for service in services) # assert cassandra is completed services = get_services(args=['--completed']) assert len(services) >= 3 assert any(service['id'] == cassandra_id for service in services) package_uninstall('cassandra') def test_log(): returncode, stdout, stderr = exec_command( ['dcos', 'service', 'log', 'chronos']) assert returncode == 0 assert len(stdout.decode('utf-8').split('\n')) > 1 assert stderr == b'' def test_log_file(): returncode, stdout, stderr = exec_command( ['dcos', 'service', 'log', 'chronos', 'stderr']) assert returncode == 0 assert len(stdout.decode('utf-8').split('\n')) > 1 assert stderr == b'' def test_log_marathon_file(): assert_command(['dcos', 'service', 'log', 'marathon', 'stderr'], stderr=(b'The argument is invalid for marathon. ' + b'The systemd journal is always used for the ' + b'marathon log.\n'), returncode=1) def test_log_marathon_config(): stdout, stderr, _ = ssh_output( 'dcos service log marathon ' + '--ssh-config-file=tests/data/node/ssh_config') assert stdout == b'' assert b'ignoring bad proto spec' in stderr @pytest.mark.skipif(True, reason=( "Now that we test against an AWS cluster, this test " "is blocked on DCOS-3104requires python3.3")) def test_log_marathon(): stdout, stderr = ssh_output( 'dcos service log marathon ' + '--ssh-config-file=tests/data/service/ssh_config') assert len(stdout.decode('utf-8').split('\n')) > 10 assert b"Running `" in stderr num_lines = len(stderr.decode('utf-8').split('\n')) assert ((num_lines == 2) or (num_lines == 3 and b'Warning: Permanently added' in stderr)) def test_log_config(): assert_command( ['dcos', 'service', 'log', 'chronos', '--ssh-config-file=/path'], stderr=(b'The `--ssh-config-file` argument is invalid for ' b'non-marathon services. SSH is not used.\n'), returncode=1) def test_log_follow(): wait_for_service('chronos') proc = subprocess.Popen(['dcos', 'service', 'log', 'chronos', '--follow'], preexec_fn=os.setsid, stdout=subprocess.PIPE, stderr=subprocess.PIPE) time.sleep(10) proc.poll() assert proc.returncode is None os.killpg(os.getpgid(proc.pid), 15) stdout = proc.stdout.read() stderr = proc.stderr.read() print('STDOUT: {}'.format(stdout)) print('STDERR: {}'.format(stderr)) assert len(stdout.decode('utf-8').split('\n')) > 3 def test_log_lines(): assert_lines(['dcos', 'service', 'log', 'chronos', '--lines=4'], 4) @pytest.mark.skipif(True, reason='Broken Marathon but we need to release') def test_log_multiple_apps(): package_install('marathon', True, ['--options=tests/data/service/marathon-user.json']) package_install('marathon', True, ['--options=tests/data/service/marathon-user2.json', '--app-id=marathon-user2']) wait_for_service('marathon-user', number_of_services=2) try: stderr = (b'Multiple marathon apps found for service name ' + b'[marathon-user]: [/marathon-user], [/marathon-user2]\n') assert_command(['dcos', 'service', 'log', 'marathon-user'], returncode=1, stderr=stderr) finally: # We can't use `dcos package uninstall`. The services have the same # name. Manually remove the dcos services. remove_app('marathon-user') remove_app('marathon-user2') for service in get_services(): if service['name'] == 'marathon-user': service_shutdown(service['id']) delete_zk_node('universe') def test_log_no_apps(): assert_command(['dcos', 'service', 'log', 'bogus'], stderr=b'No marathon apps found for service [bogus]\n', returncode=1) def _get_schema(service): schema = create_schema(service.dict()) schema['required'].remove('reregistered_time') schema['required'].remove('pid') schema['required'].remove('executors') schema['properties']['offered_resources']['required'].remove('ports') schema['properties']['resources']['required'].remove('ports') schema['properties']['used_resources']['required'].remove('ports') return schema PKցFH}ss"tests/integrations/test_package.pyimport base64 import contextlib import json import os import pkg_resources import six from dcos import package, subcommand from dcos.errors import DCOSException import pytest from mock import patch from .common import (assert_command, assert_lines, delete_zk_node, delete_zk_nodes, exec_command, file_bytes, file_json, get_services, package_install, package_uninstall, service_shutdown, wait_for_service, watch_all_deployments) @pytest.fixture(scope="module") def zk_znode(request): request.addfinalizer(delete_zk_nodes) return request def _chronos_description(app_ids): """ :param app_ids: a list of application id :type app_ids: [str] :returns: a binary string representing the chronos description :rtype: str """ result = [ {"apps": app_ids, "description": "A fault tolerant job scheduler for Mesos which " "handles dependencies and ISO8601 based schedules.", "framework": True, "images": { "icon-large": "https://downloads.mesosphere.io/chronos/assets/" "icon-service-chronos-large.png", "icon-medium": "https://downloads.mesosphere.io/chronos/assets/" "icon-service-chronos-medium.png", "icon-small": "https://downloads.mesosphere.io/chronos/assets/" "icon-service-chronos-small.png" }, "licenses": [ { "name": "Apache License Version 2.0", "url": "https://github.com/mesos/chronos/blob/master/LICENSE" } ], "maintainer": "support@mesosphere.io", "name": "chronos", "packageSource": "https://github.com/mesosphere/universe/archive/\ cli-test-3.zip", "postInstallNotes": "Chronos DCOS Service has been successfully " "installed!\n\n\tDocumentation: http://mesos." "github.io/chronos\n\tIssues: https://github.com/" "mesos/chronos/issues", "postUninstallNotes": "The Chronos DCOS Service has been uninstalled " "and will no longer run.\nPlease follow the " "instructions at http://docs.mesosphere." "com/services/chronos/#uninstall to clean up " "any persisted state", "preInstallNotes": "We recommend a minimum of one node with at least " "1 CPU and 2GB of RAM available for the Chronos " "Service.", "releaseVersion": "1", "scm": "https://github.com/mesos/chronos.git", "tags": [ "cron", "analytics", "batch" ], "version": "2.4.0" }] return (json.dumps(result, sort_keys=True, indent=2).replace(' \n', '\n') + '\n').encode('utf-8') def test_package(): stdout = pkg_resources.resource_string( 'tests', 'data/help/package.txt') assert_command(['dcos', 'package', '--help'], stdout=stdout) def test_info(): assert_command(['dcos', 'package', '--info'], stdout=b'Install and manage DCOS packages\n') def test_version(): assert_command(['dcos', 'package', '--version'], stdout=b'dcos-package version SNAPSHOT\n') def test_sources_list(): stdout = b"fd40db7f075490e0c92ec6fcd62ec1caa361b313 " + \ b"https://github.com/mesosphere/universe/archive/cli-test-3.zip\n" assert_command(['dcos', 'package', 'sources'], stdout=stdout) def test_update_without_validation(): returncode, stdout, stderr = exec_command(['dcos', 'package', 'update']) assert returncode == 0 assert b'source' in stdout assert b'Validating package definitions...' not in stdout assert b'OK' not in stdout assert stderr == b'' def test_update_with_validation(): returncode, stdout, stderr = exec_command( ['dcos', 'package', 'update', '--validate']) assert returncode == 0 assert b'source' in stdout assert b'Validating package definitions...' in stdout assert b'OK' in stdout assert stderr == b'' def test_describe_nonexistent(): assert_command(['dcos', 'package', 'describe', 'xyzzy'], stderr=b'Package [xyzzy] not found\n', returncode=1) def test_describe_nonexistent_version(): stderr = b'Version a.b.c of package [marathon] is not available\n' assert_command(['dcos', 'package', 'describe', 'marathon', '--package-version=a.b.c'], stderr=stderr, returncode=1) def test_describe(): stdout = file_json( 'tests/data/package/json/test_describe_marathon.json') assert_command(['dcos', 'package', 'describe', 'marathon'], stdout=stdout) def test_describe_cli(): stdout = file_json( 'tests/data/package/json/test_describe_cli_cassandra.json') assert_command(['dcos', 'package', 'describe', 'cassandra', '--cli'], stdout=stdout) def test_describe_app(): stdout = file_bytes( 'tests/data/package/json/test_describe_app_marathon.json') assert_command(['dcos', 'package', 'describe', 'marathon', '--app'], stdout=stdout) def test_describe_config(): stdout = file_json( 'tests/data/package/json/test_describe_marathon_config.json') assert_command(['dcos', 'package', 'describe', 'marathon', '--config'], stdout=stdout) def test_describe_render(): stdout = file_json( 'tests/data/package/json/test_describe_marathon_app_render.json') assert_command( ['dcos', 'package', 'describe', 'marathon', '--app', '--render'], stdout=stdout) def test_describe_package_version(): stdout = file_json( 'tests/data/package/json/test_describe_marathon_package_version.json') assert_command( ['dcos', 'package', 'describe', 'marathon', '--package-version=0.8.1'], stdout=stdout) def test_describe_package_version_missing(): stderr = b'Version bogus of package [marathon] is not available\n' assert_command( ['dcos', 'package', 'describe', 'marathon', '--package-version=bogus'], returncode=1, stderr=stderr) def test_describe_package_versions(): stdout = file_bytes( 'tests/data/package/json/test_describe_marathon_package_versions.json') assert_command( ['dcos', 'package', 'describe', 'marathon', '--package-versions'], stdout=stdout) def test_describe_package_versions_others(): stderr = (b'If --package-versions is provided, no other option can be ' b'provided\n') assert_command( ['dcos', 'package', 'describe', 'marathon', '--package-versions', '--app'], returncode=1, stderr=stderr) def test_describe_options(): stdout = file_json( 'tests/data/package/json/test_describe_app_options.json') assert_command(['dcos', 'package', 'describe', '--app', '--options', 'tests/data/package/marathon.json', 'marathon'], stdout=stdout) def test_describe_app_cli(): stdout = file_bytes( 'tests/data/package/json/test_describe_app_cli.json') assert_command( ['dcos', 'package', 'describe', 'cassandra', '--app', '--cli'], stdout=stdout) def test_describe_specific_version(): stdout = file_bytes( 'tests/data/package/json/test_describe_marathon_0.8.1.json') assert_command(['dcos', 'package', 'describe', '--package-version=0.8.1', 'marathon'], stdout=stdout) def test_bad_install(): args = ['--options=tests/data/package/chronos-bad.json', '--yes'] stderr = b"""Error: False is not of type 'string' Path: chronos.zk-hosts Value: false Please create a JSON file with the appropriate options, and pass the \ /path/to/file as an --options argument. """ _install_chronos(args=args, returncode=1, stdout=b'', stderr=stderr, postInstallNotes=b'') def test_install(zk_znode): _install_chronos() watch_all_deployments() wait_for_service('chronos') _uninstall_chronos() watch_all_deployments() services = get_services(args=['--inactive']) assert len([service for service in services if service['name'] == 'chronos']) == 0 def test_install_missing_options_file(): """Test that a missing options file results in the expected stderr message.""" assert_command( ['dcos', 'package', 'install', 'chronos', '--yes', '--options=asdf.json'], returncode=1, stderr=b"Error opening file [asdf.json]: No such file or directory\n") def test_install_specific_version(): stdout = (b'We recommend a minimum of one node with at least 2 ' b'CPU\'s and 1GB of RAM available for the Marathon Service.\n' b'Installing Marathon app for package [marathon] ' b'version [0.8.1]\n' b'Marathon DCOS Service has been successfully installed!\n\n' b'\tDocumentation: https://mesosphere.github.io/marathon\n' b'\tIssues: https:/github.com/mesosphere/marathon/issues\n\n') uninstall_stderr = ( b'Uninstalled package [marathon] version [0.8.1]\n' b'The Marathon DCOS Service has been uninstalled and will no longer ' b'run.\nPlease follow the instructions at http://docs.mesosphere.com/' b'services/marathon/#uninstall to clean up any persisted state\n' ) with _package('marathon', stdout=stdout, uninstall_stderr=uninstall_stderr, args=['--yes', '--package-version=0.8.1']): returncode, stdout, stderr = exec_command( ['dcos', 'package', 'list', 'marathon', '--json']) assert returncode == 0 assert stderr == b'' assert json.loads(stdout.decode('utf-8'))[0]['version'] == "0.8.1" def test_install_bad_package_version(): stderr = b'Version a.b.c of package [cassandra] is not available\n' assert_command( ['dcos', 'package', 'install', 'cassandra', '--package-version=a.b.c'], returncode=1, stderr=stderr) def test_package_metadata(): _install_helloworld() # test marathon labels expected_metadata = b"""eyJkZXNjcmlwdGlvbiI6ICJFeGFtcGxlIERDT1MgYXBwbGljYX\ Rpb24gcGFja2FnZSIsICJtYWludGFpbmVyIjogInN1cHBvcnRAbWVzb3NwaGVyZS5pbyIsICJuYW1l\ IjogImhlbGxvd29ybGQiLCAicG9zdEluc3RhbGxOb3RlcyI6ICJBIHNhbXBsZSBwb3N0LWluc3RhbG\ xhdGlvbiBtZXNzYWdlIiwgInByZUluc3RhbGxOb3RlcyI6ICJBIHNhbXBsZSBwcmUtaW5zdGFsbGF0\ aW9uIG1lc3NhZ2UiLCAidGFncyI6IFsibWVzb3NwaGVyZSIsICJleGFtcGxlIiwgInN1YmNvbW1hbm\ QiXSwgInZlcnNpb24iOiAiMC4xLjAiLCAid2Vic2l0ZSI6ICJodHRwczovL2dpdGh1Yi5jb20vbWVz\ b3NwaGVyZS9kY29zLWhlbGxvd29ybGQifQ==""" expected_command = b"""eyJwaXAiOiBbImRjb3M8MS4wIiwgImdpdCtodHRwczovL2dpdGh\ 1Yi5jb20vbWVzb3NwaGVyZS9kY29zLWhlbGxvd29ybGQuZ2l0I2Rjb3MtaGVsbG93b3JsZD0wLjEuM\ CJdfQ==""" expected_source = b"""https://github.com/mesosphere/universe/archive/\ cli-test-3.zip""" expected_labels = { 'DCOS_PACKAGE_METADATA': expected_metadata, 'DCOS_PACKAGE_COMMAND': expected_command, 'DCOS_PACKAGE_REGISTRY_VERSION': b'2.0.0-rc1', 'DCOS_PACKAGE_NAME': b'helloworld', 'DCOS_PACKAGE_VERSION': b'0.1.0', 'DCOS_PACKAGE_SOURCE': expected_source, 'DCOS_PACKAGE_RELEASE': b'0', } app_labels = _get_app_labels('helloworld') for label, value in expected_labels.items(): assert value == six.b(app_labels.get(label)) # test local package.json package = { "description": "Example DCOS application package", "maintainer": "support@mesosphere.io", "name": "helloworld", "postInstallNotes": "A sample post-installation message", "preInstallNotes": "A sample pre-installation message", "tags": ["mesosphere", "example", "subcommand"], "version": "0.1.0", "website": "https://github.com/mesosphere/dcos-helloworld", } package_dir = subcommand.package_dir('helloworld') # test local package.json package_path = os.path.join(package_dir, 'package.json') with open(package_path) as f: assert json.load(f) == package # test local source source_path = os.path.join(package_dir, 'source') with open(source_path) as f: assert six.b(f.read()) == expected_source # test local version version_path = os.path.join(package_dir, 'version') with open(version_path) as f: assert six.b(f.read()) == b'0' # uninstall helloworld _uninstall_helloworld() def test_images_in_metadata(): package_install('cassandra') labels = _get_app_labels('/cassandra/dcos') dcos_package_metadata = labels.get("DCOS_PACKAGE_METADATA") images = json.loads( base64.b64decode(dcos_package_metadata).decode('utf-8'))["images"] assert images.get("icon-small") is not None assert images.get("icon-medium") is not None assert images.get("icon-large") is not None # uninstall stderr = (b'Uninstalled package [cassandra] version [0.2.0-1]\n' b'The Apache Cassandra DCOS Service has been uninstalled and ' b'will no longer run.\n' b'Please follow the instructions at http://docs.mesosphere.com/' b'services/cassandra/#uninstall to clean up any persisted ' b'state\n') package_uninstall('cassandra', stderr=stderr) assert_command(['dcos', 'marathon', 'group', 'remove', '/cassandra']) delete_zk_node('cassandra-mesos') def test_install_with_id(zk_znode): args = ['--app-id=chronos-1', '--yes'] stdout = (b'Installing Marathon app for package [chronos] version [2.4.0] ' b'with app id [chronos-1]\n') _install_chronos(args=args, stdout=stdout) args = ['--app-id=chronos-2', '--yes'] stdout = (b'Installing Marathon app for package [chronos] version [2.4.0] ' b'with app id [chronos-2]\n') _install_chronos(args=args, stdout=stdout) def test_install_missing_package(): stderr = b"""Package [missing-package] not found You may need to run 'dcos package update' to update your repositories """ assert_command(['dcos', 'package', 'install', 'missing-package'], returncode=1, stderr=stderr) def test_uninstall_with_id(zk_znode): _uninstall_chronos(args=['--app-id=chronos-1']) def test_uninstall_all(zk_znode): _uninstall_chronos(args=['--all']) get_services(expected_count=1, args=['--inactive']) def test_uninstall_missing(): stderr = 'Package [chronos] is not installed.\n' _uninstall_chronos(returncode=1, stderr=stderr) stderr = 'Package [chronos] with id [chronos-1] is not installed.\n' _uninstall_chronos( args=['--app-id=chronos-1'], returncode=1, stderr=stderr) def test_uninstall_subcommand(): _install_helloworld() _uninstall_helloworld() _list() def test_uninstall_cli(): _install_helloworld() _uninstall_helloworld(args=['--cli']) stdout = b"""[ { "apps": [ "/helloworld" ], "description": "Example DCOS application package", "maintainer": "support@mesosphere.io", "name": "helloworld", "packageSource": "https://github.com/mesosphere/universe/archive/\ cli-test-3.zip", "postInstallNotes": "A sample post-installation message", "preInstallNotes": "A sample pre-installation message", "releaseVersion": "0", "tags": [ "mesosphere", "example", "subcommand" ], "version": "0.1.0", "website": "https://github.com/mesosphere/dcos-helloworld" } ] """ _list(stdout=stdout) _uninstall_helloworld() def test_uninstall_multiple_apps(): stdout = (b'A sample pre-installation message\n' b'Installing Marathon app for package [helloworld] version ' b'[0.1.0] with app id [/helloworld-1]\n' b'Installing CLI subcommand for package [helloworld] ' b'version [0.1.0]\n' b'New command available: dcos helloworld\n' b'A sample post-installation message\n') _install_helloworld(['--yes', '--app-id=/helloworld-1'], stdout=stdout) stdout = (b'A sample pre-installation message\n' b'Installing Marathon app for package [helloworld] version ' b'[0.1.0] with app id [/helloworld-2]\n' b'Installing CLI subcommand for package [helloworld] ' b'version [0.1.0]\n' b'New command available: dcos helloworld\n' b'A sample post-installation message\n') _install_helloworld(['--yes', '--app-id=/helloworld-2'], stdout=stdout) stderr = (b"Multiple apps named [helloworld] are installed: " b"[/helloworld-1, /helloworld-2].\n" b"Please use --app-id to specify the ID of the app " b"to uninstall, or use --all to uninstall all apps.\n") _uninstall_helloworld(stderr=stderr, returncode=1) assert_command(['dcos', 'package', 'uninstall', 'helloworld', '--all']) watch_all_deployments() def test_list(zk_znode): _list() _list(args=['xyzzy', '--json']) _list(args=['--app-id=/xyzzy', '--json']) _install_chronos() expected_output = _chronos_description(['/chronos']) _list(stdout=expected_output) _list(args=['--json', 'chronos'], stdout=expected_output) _list(args=['--json', '--app-id=/chronos'], stdout=expected_output) _list(args=['--json', 'ceci-nest-pas-une-package']) _list(args=['--json', '--app-id=/ceci-nest-pas-une-package']) _uninstall_chronos() def test_list_table(): with _helloworld(): assert_lines(['dcos', 'package', 'list'], 2) def test_install_yes(): with open('tests/data/package/assume_yes.txt') as yes_file: _install_helloworld( args=[], stdin=yes_file, stdout=b'A sample pre-installation message\n' b'Continue installing? [yes/no] ' b'Installing Marathon app for package [helloworld] version ' b'[0.1.0]\n' b'Installing CLI subcommand for package [helloworld] ' b'version [0.1.0]\n' b'New command available: dcos helloworld\n' b'A sample post-installation message\n') _uninstall_helloworld() def test_install_no(): with open('tests/data/package/assume_no.txt') as no_file: _install_helloworld( args=[], stdin=no_file, stdout=b'A sample pre-installation message\n' b'Continue installing? [yes/no] Exiting installation.\n') def test_list_cli(): _install_helloworld() stdout = b"""\ [ { "apps": [ "/helloworld" ], "command": { "name": "helloworld" }, "description": "Example DCOS application package", "maintainer": "support@mesosphere.io", "name": "helloworld", "packageSource": "https://github.com/mesosphere/universe/archive/\ cli-test-3.zip", "postInstallNotes": "A sample post-installation message", "preInstallNotes": "A sample pre-installation message", "releaseVersion": "0", "tags": [ "mesosphere", "example", "subcommand" ], "version": "0.1.0", "website": "https://github.com/mesosphere/dcos-helloworld" } ] """ _list(stdout=stdout) _uninstall_helloworld() stdout = (b"A sample pre-installation message\n" b"Installing CLI subcommand for package [helloworld] " + b"version [0.1.0]\n" b"New command available: dcos helloworld\n" b"A sample post-installation message\n") _install_helloworld(args=['--cli', '--yes'], stdout=stdout) stdout = b"""\ [ { "command": { "name": "helloworld" }, "description": "Example DCOS application package", "maintainer": "support@mesosphere.io", "name": "helloworld", "packageSource": "https://github.com/mesosphere/universe/archive/\ cli-test-3.zip", "postInstallNotes": "A sample post-installation message", "preInstallNotes": "A sample pre-installation message", "releaseVersion": "0", "tags": [ "mesosphere", "example", "subcommand" ], "version": "0.1.0", "website": "https://github.com/mesosphere/dcos-helloworld" } ] """ _list(stdout=stdout) _uninstall_helloworld() def test_uninstall_multiple_frameworknames(zk_znode): _install_chronos( args=['--yes', '--options=tests/data/package/chronos-1.json']) _install_chronos( args=['--yes', '--options=tests/data/package/chronos-2.json']) watch_all_deployments() expected_output = _chronos_description( ['/chronos-user-1', '/chronos-user-2']) _list(stdout=expected_output) _list(args=['--json', 'chronos'], stdout=expected_output) _list(args=['--json', '--app-id=/chronos-user-1'], stdout=_chronos_description(['/chronos-user-1'])) _list(args=['--json', '--app-id=/chronos-user-2'], stdout=_chronos_description(['/chronos-user-2'])) _uninstall_chronos( args=['--app-id=chronos-user-1'], returncode=1, stderr='Uninstalled package [chronos] version [2.4.0]\n' 'The Chronos DCOS Service has been uninstalled and will no ' 'longer run.\nPlease follow the instructions at http://docs.' 'mesosphere.com/services/chronos/#uninstall to clean up any ' 'persisted state\n' 'Unable to shutdown the framework for [chronos-user] because ' 'there are multiple frameworks with the same name: ') _uninstall_chronos( args=['--app-id=chronos-user-2'], returncode=1, stderr='Uninstalled package [chronos] version [2.4.0]\n' 'The Chronos DCOS Service has been uninstalled and will no ' 'longer run.\nPlease follow the instructions at http://docs.' 'mesosphere.com/services/chronos/#uninstall to clean up any ' 'persisted state\n' 'Unable to shutdown the framework for [chronos-user] because ' 'there are multiple frameworks with the same name: ') for framework in get_services(args=['--inactive']): if framework['name'] == 'chronos-user': service_shutdown(framework['id']) def test_search(): returncode, stdout, stderr = exec_command( ['dcos', 'package', 'search', 'cron', '--json']) assert returncode == 0 assert b'chronos' in stdout assert stderr == b'' returncode, stdout, stderr = exec_command( ['dcos', 'package', 'search', 'xyzzy', '--json']) assert returncode == 0 assert b'"packages": []' in stdout assert b'"source": "https://github.com/mesosphere/universe/archive/\ cli-test-3.zip"' in stdout assert stderr == b'' returncode, stdout, stderr = exec_command( ['dcos', 'package', 'search', 'xyzzy']) assert returncode == 1 assert b'' == stdout assert stderr == b'No packages found.\n' returncode, stdout, stderr = exec_command( ['dcos', 'package', 'search', '--json']) registries = json.loads(stdout.decode('utf-8')) for registry in registries: # assert the number of packages is gte the number at the time # this test was written assert len(registry['packages']) >= 5 assert returncode == 0 assert stderr == b'' def test_search_table(): returncode, stdout, stderr = exec_command( ['dcos', 'package', 'search']) assert returncode == 0 assert b'chronos' in stdout assert len(stdout.decode('utf-8').split('\n')) > 5 assert stderr == b'' def test_search_ends_with_wildcard(): returncode, stdout, stderr = exec_command( ['dcos', 'package', 'search', 'c*', '--json']) assert returncode == 0 assert b'chronos' in stdout assert b'cassandra' in stdout assert stderr == b'' registries = json.loads(stdout.decode('utf-8')) for registry in registries: assert len(registry['packages']) == 2 def test_search_start_with_wildcard(): returncode, stdout, stderr = exec_command( ['dcos', 'package', 'search', '*nos', '--json']) assert returncode == 0 assert b'chronos' in stdout assert stderr == b'' registries = json.loads(stdout.decode('utf-8')) for registry in registries: assert len(registry['packages']) == 1 def test_search_middle_with_wildcard(): returncode, stdout, stderr = exec_command( ['dcos', 'package', 'search', 'c*s', '--json']) assert returncode == 0 assert b'chronos' in stdout assert stderr == b'' registries = json.loads(stdout.decode('utf-8')) for registry in registries: assert len(registry['packages']) == 1 @patch('dcos.package.Package.package_json') @patch('dcos.package.Package.config_json') def test_bad_config_schema_msg(config_mock, package_mock): pkg = package.Package("", "/") config_mock.return_value = {} package_mock.return_value = {'maintainer': 'support@test'} with pytest.raises(DCOSException) as e: pkg.options("1", {}) msg = ("An object in the package's config.json is missing the " "required 'properties' feature:\n {}" "\nPlease contact the project maintainer: support@test") assert e.exconly().split(':', 1)[1].strip() == msg def _get_app_labels(app_id): returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'show', app_id]) assert returncode == 0 assert stderr == b'' app_json = json.loads(stdout.decode('utf-8')) return app_json.get('labels') def _install_helloworld( args=['--yes'], stdout=b'A sample pre-installation message\n' b'Installing Marathon app for package [helloworld] ' b'version [0.1.0]\n' b'Installing CLI subcommand for package [helloworld] ' b'version [0.1.0]\n' b'New command available: dcos helloworld\n' b'A sample post-installation message\n', returncode=0, stdin=None): assert_command( ['dcos', 'package', 'install', 'helloworld'] + args, stdout=stdout, returncode=returncode, stdin=stdin) def _uninstall_helloworld( args=[], stdout=b'', stderr=b'', returncode=0): assert_command(['dcos', 'package', 'uninstall', 'helloworld'] + args, stdout=stdout, stderr=stderr, returncode=returncode) def _uninstall_chronos(args=[], returncode=0, stdout=b'', stderr=''): result_returncode, result_stdout, result_stderr = exec_command( ['dcos', 'package', 'uninstall', 'chronos'] + args) assert result_returncode == returncode assert result_stdout == stdout assert result_stderr.decode('utf-8').startswith(stderr) def _install_chronos( args=['--yes'], returncode=0, stdout=b'Installing Marathon app for package [chronos] ' b'version [2.4.0]\n', stderr=b'', preInstallNotes=b'We recommend a minimum of one node with at least 1 ' b'CPU and 2GB of RAM available for the Chronos ' b'Service.\n', postInstallNotes=b'Chronos DCOS Service has been successfully ' b'''installed! \tDocumentation: http://mesos.github.io/chronos \tIssues: https://github.com/mesos/chronos/issues\n''', stdin=None): cmd = ['dcos', 'package', 'install', 'chronos'] + args assert_command( cmd, returncode, preInstallNotes + stdout + postInstallNotes, stderr, stdin=stdin) def _list(args=['--json'], stdout=b'[]\n'): assert_command(['dcos', 'package', 'list'] + args, stdout=stdout) def _helloworld(): stdout = b'''A sample pre-installation message Installing Marathon app for package [helloworld] version [0.1.0] Installing CLI subcommand for package [helloworld] version [0.1.0] New command available: dcos helloworld A sample post-installation message ''' return _package('helloworld', stdout=stdout) @contextlib.contextmanager def _package(name, stdout=b'', uninstall_stderr=b'', args=['--yes']): """Context manager that installs a package on entrace, and uninstalls it on exit. :param name: package name :type name: str :param stdout: Expected stdout :type stdout: str :param uninstall_stderr: Expected stderr :type uninstall_stderr: str :param args: extra CLI args :type args: [str] :rtype: None """ assert_command(['dcos', 'package', 'install', name] + args, stdout=stdout) try: yield finally: assert_command( ['dcos', 'package', 'uninstall', name], stderr=uninstall_stderr) watch_all_deployments() PKցFH{۵#[#[#tests/integrations/test_marathon.pyimport contextlib import json import os import re import threading from dcos import constants import pytest from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer from .common import (app, assert_command, assert_lines, config_set, config_unset, exec_command, list_deployments, popen_tty, show_app, watch_all_deployments, watch_deployment) _ZERO_INSTANCE_APP_INSTANCES = 100 def test_help(): with open('tests/data/help/marathon.txt') as content: assert_command(['dcos', 'marathon', '--help'], stdout=content.read().encode('utf-8')) def test_version(): assert_command(['dcos', 'marathon', '--version'], stdout=b'dcos-marathon version SNAPSHOT\n') def test_info(): assert_command(['dcos', 'marathon', '--info'], stdout=b'Deploy and manage applications on the DCOS\n') def test_about(): returncode, stdout, stderr = exec_command(['dcos', 'marathon', 'about']) assert returncode == 0 assert stderr == b'' result = json.loads(stdout.decode('utf-8')) assert result['name'] == "marathon" @pytest.fixture def missing_env(): env = os.environ.copy() env.update({ constants.PATH_ENV: os.environ[constants.PATH_ENV], constants.DCOS_CONFIG_ENV: os.path.join("tests", "data", "marathon", "missing_marathon_params.toml") }) return env def test_missing_config(missing_env): assert_command( ['dcos', 'marathon', 'app', 'list'], returncode=1, stderr=(b'Missing required config parameter: "core.dcos_url". ' b'Please run `dcos config set core.dcos_url `.\n'), env=missing_env) def test_empty_list(): _list_apps() def test_add_app(): with _zero_instance_app(): _list_apps('zero-instance-app') def test_add_app_through_http(): with _zero_instance_app_through_http(): _list_apps('zero-instance-app') def test_add_app_bad_resource(): stderr = (b'Can\'t read from resource: bad_resource.\n' b'Please check that it exists.\n') assert_command(['dcos', 'marathon', 'app', 'add', 'bad_resource'], returncode=1, stderr=stderr) def test_add_app_with_filename(): with _zero_instance_app(): _list_apps('zero-instance-app') def test_remove_app(): with _zero_instance_app(): pass _list_apps() def test_add_bad_json_app(): with open('tests/data/marathon/apps/bad.json') as fd: returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'add'], stdin=fd) assert returncode == 1 assert stdout == b'' assert stderr.decode('utf-8').startswith('Error loading JSON: ') def test_add_existing_app(): with _zero_instance_app(): app_path = 'tests/data/marathon/apps/zero_instance_sleep_v2.json' with open(app_path) as fd: stderr = b"Application '/zero-instance-app' already exists\n" assert_command(['dcos', 'marathon', 'app', 'add'], returncode=1, stderr=stderr, stdin=fd) def test_show_app(): with _zero_instance_app(): show_app('zero-instance-app') def test_show_absolute_app_version(): with _zero_instance_app(): _update_app( 'zero-instance-app', 'tests/data/marathon/apps/update_zero_instance_sleep.json') result = show_app('zero-instance-app') show_app('zero-instance-app', result['version']) def test_show_relative_app_version(): with _zero_instance_app(): _update_app( 'zero-instance-app', 'tests/data/marathon/apps/update_zero_instance_sleep.json') show_app('zero-instance-app', "-1") def test_show_missing_relative_app_version(): with _zero_instance_app(): _update_app( 'zero-instance-app', 'tests/data/marathon/apps/update_zero_instance_sleep.json') stderr = b"Application 'zero-instance-app' only has 2 version(s).\n" assert_command(['dcos', 'marathon', 'app', 'show', '--app-version=-2', 'zero-instance-app'], returncode=1, stderr=stderr) def test_show_missing_absolute_app_version(): with _zero_instance_app(): _update_app( 'zero-instance-app', 'tests/data/marathon/apps/update_zero_instance_sleep.json') returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'show', '--app-version=2000-02-11T20:39:32.972Z', 'zero-instance-app']) assert returncode == 1 assert stdout == b'' assert stderr.decode('utf-8').startswith( "Error: App '/zero-instance-app' does not exist") def test_show_bad_app_version(): with _zero_instance_app(): _update_app( 'zero-instance-app', 'tests/data/marathon/apps/update_zero_instance_sleep.json') stderr = (b'Error: Invalid format: "20:39:32.972Z" is malformed at ' b'":39:32.972Z"\n') assert_command( ['dcos', 'marathon', 'app', 'show', '--app-version=20:39:32.972Z', 'zero-instance-app'], returncode=1, stderr=stderr) def test_show_bad_relative_app_version(): with _zero_instance_app(): _update_app( 'zero-instance-app', 'tests/data/marathon/apps/update_zero_instance_sleep.json') assert_command( ['dcos', 'marathon', 'app', 'show', '--app-version=2', 'zero-instance-app'], returncode=1, stderr=b"Relative versions must be negative: 2\n") def test_start_missing_app(): assert_command( ['dcos', 'marathon', 'app', 'start', 'missing-id'], returncode=1, stderr=b"Error: App '/missing-id' does not exist\n") def test_start_app(): with _zero_instance_app(): _start_app('zero-instance-app') def test_start_already_started_app(): with _zero_instance_app(): _start_app('zero-instance-app') stdout = (b"Application 'zero-instance-app' already " b"started: 1 instances.\n") assert_command( ['dcos', 'marathon', 'app', 'start', 'zero-instance-app'], returncode=1, stdout=stdout) def test_stop_missing_app(): assert_command(['dcos', 'marathon', 'app', 'stop', 'missing-id'], returncode=1, stderr=b"Error: App '/missing-id' does not exist\n") def test_stop_app(): with _zero_instance_app(): _start_app('zero-instance-app', 3) watch_all_deployments() returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'stop', 'zero-instance-app']) assert returncode == 0 assert stdout.decode().startswith('Created deployment ') assert stderr == b'' def test_stop_already_stopped_app(): with _zero_instance_app(): stdout = (b"Application 'zero-instance-app' already " b"stopped: 0 instances.\n") assert_command( ['dcos', 'marathon', 'app', 'stop', 'zero-instance-app'], returncode=1, stdout=stdout) def test_update_missing_app(): assert_command(['dcos', 'marathon', 'app', 'update', 'missing-id'], stderr=b"Error: App '/missing-id' does not exist\n", returncode=1) def test_update_bad_type(): with _zero_instance_app(): returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'update', 'zero-instance-app', 'cpus="a string"']) stderr_end = b"""{ "details": [ { "errors": [ "error.expected.jsnumber" ], "path": "/cpus" } ], "message": "Invalid JSON" } """ assert returncode == 1 assert stderr_end in stderr assert stdout == b'' def test_update_invalid_request(): returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'update', '{', 'instances']) assert returncode == 1 assert stdout == b'' stderr = stderr.decode() # TODO (tamar): this becomes 'Error: App '/{' does not exist\n"' # in Marathon 0.11.0 assert stderr.startswith('Error on request') assert stderr.endswith('HTTP 400: Bad Request\n') def test_app_add_invalid_request(): path = os.path.join( 'tests', 'data', 'marathon', 'apps', 'app_add_400.json') returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'add', path]) assert returncode == 1 assert stdout == b'' assert re.match(b"Error on request \[POST .*\]: HTTP 400: Bad Request:", stderr) stderr_end = b"""{ "details": [ { "errors": [ "host is not a valid network type" ], "path": "/container/docker/network" } ], "message": "Invalid JSON" } """ assert stderr.endswith(stderr_end) def test_update_app(): with _zero_instance_app(): returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'update', 'zero-instance-app', 'cpus=1', 'mem=20', "cmd='sleep 100'"]) assert returncode == 0 assert stdout.decode().startswith('Created deployment ') assert stderr == b'' def test_update_app_from_stdin(): with _zero_instance_app(): _update_app( 'zero-instance-app', 'tests/data/marathon/apps/update_zero_instance_sleep.json') def test_restarting_stopped_app(): with _zero_instance_app(): stdout = (b"Unable to perform rolling restart of application '" b"/zero-instance-app' because it has no running tasks\n") assert_command( ['dcos', 'marathon', 'app', 'restart', 'zero-instance-app'], returncode=1, stdout=stdout) def test_restarting_missing_app(): assert_command(['dcos', 'marathon', 'app', 'restart', 'missing-id'], returncode=1, stderr=b"Error: App '/missing-id' does not exist\n") def test_restarting_app(): with _zero_instance_app(): _start_app('zero-instance-app', 3) watch_all_deployments() returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'restart', 'zero-instance-app']) assert returncode == 0 assert stdout.decode().startswith('Created deployment ') assert stderr == b'' def test_killing_app(): with _zero_instance_app(): _start_app('zero-instance-app', 3) watch_all_deployments() task_set_1 = set([task['id'] for task in _list_tasks(3, 'zero-instance-app')]) returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'kill', 'zero-instance-app']) assert returncode == 0 assert stdout.decode().startswith('Killed tasks: ') assert stderr == b'' watch_all_deployments() task_set_2 = set([task['id'] for task in _list_tasks(app_id='zero-instance-app')]) assert len(task_set_1.intersection(task_set_2)) == 0 def test_killing_scaling_app(): with _zero_instance_app(): _start_app('zero-instance-app', 3) watch_all_deployments() _list_tasks(3) command = ['dcos', 'marathon', 'app', 'kill', '--scale', 'zero-instance-app'] returncode, stdout, stderr = exec_command(command) assert returncode == 0 assert stdout.decode().startswith('Started deployment: ') assert stdout.decode().find('version') > -1 assert stdout.decode().find('deploymentId') > -1 assert stderr == b'' watch_all_deployments() _list_tasks(0) def test_killing_with_host_app(): with _zero_instance_app(): _start_app('zero-instance-app', 3) watch_all_deployments() existing_tasks = _list_tasks(3, 'zero-instance-app') task_hosts = set([task['host'] for task in existing_tasks]) if len(task_hosts) <= 1: pytest.skip('test needs 2 or more agents to succeed, ' 'only {} agents available'.format(len(task_hosts))) assert len(task_hosts) > 1 kill_host = list(task_hosts)[0] expected_to_be_killed = set([task['id'] for task in existing_tasks if task['host'] == kill_host]) not_to_be_killed = set([task['id'] for task in existing_tasks if task['host'] != kill_host]) assert len(not_to_be_killed) > 0 assert len(expected_to_be_killed) > 0 command = ['dcos', 'marathon', 'app', 'kill', '--host', kill_host, 'zero-instance-app'] returncode, stdout, stderr = exec_command(command) assert stdout.decode().startswith('Killed tasks: ') assert stderr == b'' new_tasks = set([task['id'] for task in _list_tasks()]) assert not_to_be_killed.intersection(new_tasks) == not_to_be_killed assert len(expected_to_be_killed.intersection(new_tasks)) == 0 def test_kill_stopped_app(): with _zero_instance_app(): returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'kill', 'zero-instance-app']) assert returncode == 1 assert stdout.decode().startswith('Killed tasks: []') def test_kill_missing_app(): returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'kill', 'app']) assert returncode == 1 assert stdout.decode() == '' stderr_expected = "Error: App '/app' does not exist" assert stderr.decode().strip() == stderr_expected def test_list_version_missing_app(): assert_command( ['dcos', 'marathon', 'app', 'version', 'list', 'missing-id'], returncode=1, stderr=b"Error: App '/missing-id' does not exist\n") def test_list_version_negative_max_count(): assert_command(['dcos', 'marathon', 'app', 'version', 'list', 'missing-id', '--max-count=-1'], returncode=1, stderr=b'Maximum count must be a positive number: -1\n') def test_list_version_app(): with _zero_instance_app(): _list_versions('zero-instance-app', 1) _update_app( 'zero-instance-app', 'tests/data/marathon/apps/update_zero_instance_sleep.json') _list_versions('zero-instance-app', 2) def test_list_version_max_count(): with _zero_instance_app(): _update_app( 'zero-instance-app', 'tests/data/marathon/apps/update_zero_instance_sleep.json') _list_versions('zero-instance-app', 1, 1) _list_versions('zero-instance-app', 2, 2) _list_versions('zero-instance-app', 2, 3) def test_list_empty_deployment(): list_deployments(0) def test_list_deployment(): with _zero_instance_app(): _start_app('zero-instance-app', _ZERO_INSTANCE_APP_INSTANCES) list_deployments(1) def test_list_deployment_table(): """Simple sanity check for listing deployments with a table output. The more specific testing is done in unit tests. """ with _zero_instance_app(): _start_app('zero-instance-app', _ZERO_INSTANCE_APP_INSTANCES) assert_lines(['dcos', 'marathon', 'deployment', 'list'], 2) def test_list_deployment_missing_app(): with _zero_instance_app(): _start_app('zero-instance-app') list_deployments(0, 'missing-id') def test_list_deployment_app(): with _zero_instance_app(): _start_app('zero-instance-app', _ZERO_INSTANCE_APP_INSTANCES) list_deployments(1, 'zero-instance-app') def test_rollback_missing_deployment(): assert_command( ['dcos', 'marathon', 'deployment', 'rollback', 'missing-deployment'], returncode=1, stderr=b'Error: DeploymentPlan missing-deployment does not exist\n') def test_rollback_deployment(): with _zero_instance_app(): _start_app('zero-instance-app', _ZERO_INSTANCE_APP_INSTANCES) result = list_deployments(1, 'zero-instance-app') returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'deployment', 'rollback', result[0]['id']]) result = json.loads(stdout.decode('utf-8')) assert returncode == 0 assert 'deploymentId' in result assert 'version' in result assert stderr == b'' watch_all_deployments() list_deployments(0) def test_stop_deployment(): with _zero_instance_app(): _start_app('zero-instance-app', _ZERO_INSTANCE_APP_INSTANCES) result = list_deployments(1, 'zero-instance-app') assert_command( ['dcos', 'marathon', 'deployment', 'stop', result[0]['id']]) list_deployments(0) def test_watching_missing_deployment(): watch_deployment('missing-deployment', 1) def test_watching_deployment(): with _zero_instance_app(): _start_app('zero-instance-app', _ZERO_INSTANCE_APP_INSTANCES) result = list_deployments(1, 'zero-instance-app') watch_deployment(result[0]['id'], 60) assert_command( ['dcos', 'marathon', 'deployment', 'stop', result[0]['id']]) list_deployments(0, 'zero-instance-app') def test_list_empty_task(): _list_tasks(0) def test_list_empty_task_not_running_app(): with _zero_instance_app(): _list_tasks(0) def test_list_tasks(): with _zero_instance_app(): _start_app('zero-instance-app', 3) watch_all_deployments() _list_tasks(3) def test_list_tasks_table(): with _zero_instance_app(): _start_app('zero-instance-app', 3) watch_all_deployments() assert_lines(['dcos', 'marathon', 'task', 'list'], 4) def test_list_app_tasks(): with _zero_instance_app(): _start_app('zero-instance-app', 3) watch_all_deployments() _list_tasks(3, 'zero-instance-app') def test_list_missing_app_tasks(): with _zero_instance_app(): _start_app('zero-instance-app', 3) watch_all_deployments() _list_tasks(0, 'missing-id') def test_show_missing_task(): returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'task', 'show', 'missing-id']) stderr = stderr.decode('utf-8') assert returncode == 1 assert stdout == b'' assert stderr.startswith("Task '") assert stderr.endswith("' does not exist\n") def test_show_task(): with _zero_instance_app(): _start_app('zero-instance-app', 3) watch_all_deployments() result = _list_tasks(3, 'zero-instance-app') returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'task', 'show', result[0]['id']]) result = json.loads(stdout.decode('utf-8')) assert returncode == 0 assert result['appId'] == '/zero-instance-app' assert stderr == b'' def test_bad_configuration(): config_set('marathon.url', 'http://localhost:88888') returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'list']) assert returncode == 1 assert stdout == b'' assert stderr.startswith( b"URL [http://localhost:88888/v2/info] is unreachable") config_unset('marathon.url') def test_app_locked_error(): with app('tests/data/marathon/apps/sleep_many_instances.json', '/sleep-many-instances', wait=False): assert_command( ['dcos', 'marathon', 'app', 'stop', 'sleep-many-instances'], returncode=1, stderr=(b'App or group is locked by one or more deployments. ' b'Override with --force.\n')) def test_app_add_no_tty(): proc, master = popen_tty('dcos marathon app add') stdout, stderr = proc.communicate() os.close(master) print(stdout) print(stderr) assert proc.wait() == 1 assert stdout == b'' assert stderr == (b"We currently don't support reading from the TTY. " b"Please specify an application JSON.\n" b"E.g.: dcos marathon app add < app_resource.json\n") def _list_apps(app_id=None): returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'list', '--json']) result = json.loads(stdout.decode('utf-8')) if app_id is None: assert len(result) == 0 else: assert len(result) == 1 assert result[0]['id'] == '/' + app_id assert returncode == 0 assert stderr == b'' return result def _start_app(app_id, instances=None): cmd = ['dcos', 'marathon', 'app', 'start', app_id] if instances is not None: cmd.append(str(instances)) returncode, stdout, stderr = exec_command(cmd) assert returncode == 0 assert stdout.decode().startswith('Created deployment ') assert stderr == b'' def _update_app(app_id, file_path): with open(file_path) as fd: returncode, stdout, stderr = exec_command( ['dcos', 'marathon', 'app', 'update', app_id], stdin=fd) assert returncode == 0 assert stdout.decode().startswith('Created deployment ') assert stderr == b'' def _list_versions(app_id, expected_count, max_count=None): cmd = ['dcos', 'marathon', 'app', 'version', 'list', app_id] if max_count is not None: cmd.append('--max-count={}'.format(max_count)) returncode, stdout, stderr = exec_command(cmd) result = json.loads(stdout.decode('utf-8')) assert returncode == 0 assert isinstance(result, list) assert len(result) == expected_count assert stderr == b'' def _list_tasks(expected_count=None, app_id=None): cmd = ['dcos', 'marathon', 'task', 'list', '--json'] if app_id is not None: cmd.append(app_id) returncode, stdout, stderr = exec_command(cmd) result = json.loads(stdout.decode('utf-8')) assert returncode == 0 if expected_count: assert len(result) == expected_count assert stderr == b'' return result @contextlib.contextmanager def _zero_instance_app(): with app('tests/data/marathon/apps/zero_instance_sleep.json', 'zero-instance-app'): yield @contextlib.contextmanager def _zero_instance_app_through_http(): class JSONRequestHandler (BaseHTTPRequestHandler): def do_GET(self): self.send_response(200) self.send_header("Content-type", "application/json") self.end_headers() self.wfile.write(open( 'tests/data/marathon/apps/zero_instance_sleep.json', 'rb').read()) host = 'localhost' port = 12345 server = HTTPServer((host, port), JSONRequestHandler) thread = threading.Thread(target=server.serve_forever) thread.setDaemon(True) thread.start() with app('http://{}:{}'.format(host, port), 'zero-instance-app'): try: yield finally: server.shutdown() PKցFHtests/fixtures/__init__.pyPKցFHKǘtests/fixtures/marathon.pydef app_fixture(): """ Marathon app fixture. :rtype: dict """ return { "acceptedResourceRoles": None, "args": None, "backoffFactor": 1.15, "backoffSeconds": 1, "cmd": "sleep 1000", "constraints": [], "container": None, "cpus": 0.1, "dependencies": [], "deployments": [], "disk": 0.0, "env": {}, "executor": "", "healthChecks": [], "id": "/test-app", "instances": 1, "labels": { "PACKAGE_ID": "test-app", "PACKAGE_VERSION": "1.2.3" }, "maxLaunchDelaySeconds": 3600, "mem": 16.0, "ports": [ 10000 ], "requirePorts": False, "storeUrls": [], "tasksHealthy": 0, "tasksRunning": 1, "tasksStaged": 0, "tasksUnhealthy": 0, "upgradeStrategy": { "maximumOverCapacity": 1.0, "minimumHealthCapacity": 1.0 }, "uris": [], "user": None, "version": "2015-05-28T21:21:05.064Z" } def deployment_fixture(): """ Marathon deployment fixture. :rtype: dict """ return { "affectedApps": [ "/cassandra/dcos" ], "currentActions": [ { "action": "ScaleApplication", "app": "/cassandra/dcos" } ], "currentStep": 2, "id": "bebb8ffd-118e-4067-8fcb-d19e44126911", "steps": [ [ { "action": "StartApplication", "app": "/cassandra/dcos" } ], [ { "action": "ScaleApplication", "app": "/cassandra/dcos" } ] ], "totalSteps": 2, "version": "2015-05-29T01:13:47.694Z" } def app_task_fixture(): """ Marathon task fixture. :rtype: dict """ return { "appId": "/zero-instance-app", "host": "dcos-01", "id": "zero-instance-app.027b3a83-063d-11e5-84a3-56847afe9799", "ports": [ 8165 ], "servicePorts": [ 10001 ], "stagedAt": "2015-05-29T19:58:00.907Z", "startedAt": "2015-05-29T19:58:01.114Z", "version": "2015-05-29T18:50:58.941Z" } def group_fixture(): """ Marathon group fixture. :rtype: dict """ return { "apps": [], "dependencies": [], "groups": [ { "apps": [ { "acceptedResourceRoles": None, "args": None, "backoffFactor": 1.15, "backoffSeconds": 1, "cmd": "sleep 1", "constraints": [], "container": None, "cpus": 1.0, "dependencies": [], "disk": 0.0, "env": {}, "executor": "", "healthChecks": [], "id": "/test-group/sleep/goodnight", "instances": 0, "labels": {}, "maxLaunchDelaySeconds": 3600, "mem": 128.0, "ports": [ 10000 ], "requirePorts": False, "storeUrls": [], "upgradeStrategy": { "maximumOverCapacity": 1.0, "minimumHealthCapacity": 1.0 }, "uris": [], "user": None, "version": "2015-05-29T23:12:46.187Z" } ], "dependencies": [], "groups": [], "id": "/test-group/sleep", "version": "2015-05-29T23:12:46.187Z" } ], "id": "/test-group", "version": "2015-05-29T23:12:46.187Z" } PKցFHtests/fixtures/task.pyfrom dcos.mesos import Slave, Task import mock def task_fixture(): """ Task fixture :rtype: Task """ task = Task({ "executor_id": "", "framework_id": "20150502-231327-16842879-5050-3889-0000", "id": "test-app.d44dd7f2-f9b7-11e4-bb43-56847afe9799", "labels": [], "name": "test-app", "resources": { "cpus": 0.1, "disk": 0.0, "mem": 16.0, "ports": "[31651-31651]" }, "slave_id": "20150513-185808-177048842-5050-1220-S0", "state": "TASK_RUNNING", "statuses": [ { "container_status": { "network_infos": [ { "ip_address": "172.17.8.12" } ] }, "state": "TASK_RUNNING", "timestamp": 1431552866.52692 } ] }, None) task.user = mock.Mock(return_value='root') slave = Slave({"hostname": "mock-hostname"}, None, None) task.slave = mock.Mock(return_value=slave) return task def browse_fixture(): return [ {u'uid': u'root', u'mtime': 1437089500, u'nlink': 1, u'mode': u'-rw-r--r--', u'gid': u'root', u'path': (u'/var/lib/mesos/slave/slaves/' + u'20150716-183440-1695027628-5050-2710-S0/frameworks/' + u'20150716-183440-1695027628-5050-2710-0000/executors/' + u'chronos.8810d396-2c09-11e5-af1a-080027d3e806/runs/' + u'aaecec57-7c7c-4030-aca3-d7aac2f9fd29/stderr'), u'size': 4507}, {u'uid': u'root', u'mtime': 1437089604, u'nlink': 1, u'mode': u'-rw-r--r--', u'gid': u'root', u'path': (u'/var/lib/mesos/slave/slaves/' + u'20150716-183440-1695027628-5050-2710-S0/frameworks/' + u'20150716-183440-1695027628-5050-2710-0000/executors/' + u'chronos.8810d396-2c09-11e5-af1a-080027d3e806/runs/' + u'aaecec57-7c7c-4030-aca3-d7aac2f9fd29/stdout'), u'size': 353857} ] PKցFH _ggtests/fixtures/node.pydef slave_fixture(): """ Slave node fixture. :rtype: dict """ return { "TASK_ERROR": 0, "TASK_FAILED": 0, "TASK_FINISHED": 0, "TASK_KILLED": 0, "TASK_LOST": 0, "TASK_RUNNING": 0, "TASK_STAGING": 0, "TASK_STARTING": 0, "active": True, "attributes": {}, "framework_ids": [], "hostname": "dcos-01", "id": "20150630-004309-1695027628-5050-1649-S0", "offered_resources": { "cpus": 0.0, "disk": 0.0, "mem": 0.0, "ports": ("[1025-2180, 2182-3887, 3889-5049, 5052-8079, " + "8082-8180, 8182-65535]") }, "pid": "slave(1)@172.17.8.101:5051", "registered_time": 1435625024.42234, "reregistered_time": 1435625024.42234, "resources": { "cpus": 4.0, "disk": 10823.0, "mem": 2933.0, "ports": ("[1025-2180, 2182-3887, 3889-5049, 5052-8079, " + "8082-8180, 8182-65535]") }, "reserved_resources": {}, "unreserved_resources": {}, "used_resources": { "cpus": 0.0, "disk": 0.0, "mem": 0.0, "ports": ("[1025-2180, 2182-3887, 3889-5049, 5052-8079, " + "8082-8180, 8182-65535]") } } PKցFH,tests/fixtures/service.pyfrom dcos.mesos import Framework def framework_fixture(): """ Framework fixture :rtype: Framework """ return Framework({ "active": True, "capabilities": [], "checkpoint": True, "completed_tasks": [], "executors": [], "failover_timeout": 604800.0, "hostname": "mesos.vm", "id": "20150502-231327-16842879-5050-3889-0000", "name": "marathon", "offered_resources": { "cpus": 0.0, "disk": 0.0, "mem": 0.0, "ports": "[1379-1379, 10000-10000]" }, "offers": [], "pid": "scheduler-a58cd5ba-f566-42e0-a283-b5f39cb66e88@172.17.8.101:55130", "registered_time": 1431543498.31955, "reregistered_time": 1431543498.31959, "resources": { "cpus": 0.2, "disk": 0.0, "mem": 32.0, "ports": "[1379-1379, 10000-10000]" }, "role": "*", "tasks": [], "unregistered_time": 0.0, "used_resources": { "cpus": 0.2, "disk": 0.0, "mem": 32.0, "ports": "[1379-1379, 10000-10000]" }, "user": "root", "webui_url": "http://mesos:8080" }, None) PKցFH=O1?tests/fixtures/package.pyfrom dcos.package import HttpSource, IndexEntries def package_fixture(): """ DCOS package fixture. :rtype: dict """ return { "apps": [ "/helloworld" ], "command": { "name": "helloworld" }, "description": "Example DCOS application package", "maintainer": "support@mesosphere.io", "name": "helloworld", "packageSource": "https://github.com/mesosphere/universe/archive/master.zip", "postInstallNotes": "A sample post-installation message", "preInstallNotes": "A sample pre-installation message", "releaseVersion": "0", "tags": [ "mesosphere", "example", "subcommand" ], "version": "0.1.0", "website": "https://github.com/mesosphere/dcos-helloworld" } def search_result_fixture(): """ DCOS package search result fixture. :rtype: dict """ return IndexEntries( HttpSource( "https://github.com/mesosphere/universe/archive/master.zip"), [ { "currentVersion": "0.1.0-SNAPSHOT-447-master-3ad1bbf8f7", "description": "Apache Cassandra running on Apache Mesos", "framework": True, "name": "cassandra", "tags": [ "mesosphere", "framework" ], "versions": [ "0.1.0-SNAPSHOT-447-master-3ad1bbf8f7" ] }, { "currentVersion": "2.3.4", "description": ("A fault tolerant job scheduler for Mesos " + "which handles dependencies and ISO8601 " + "based schedules."), "framework": True, "name": "chronos", "tags": [ "mesosphere", "framework" ], "versions": [ "2.3.4" ] }, { "currentVersion": "0.1.1", "description": ("Hadoop Distributed File System (HDFS), " + "Highly Available"), "framework": True, "name": "hdfs", "tags": [ "mesosphere", "framework", "filesystem" ], "versions": [ "0.1.1" ] }, { "currentVersion": "0.1.0", "description": "Example DCOS application package", "framework": False, "name": "helloworld", "tags": [ "mesosphere", "example", "subcommand" ], "versions": [ "0.1.0" ] }, { "currentVersion": "0.9.0-beta", "description": "Apache Kafka running on top of Apache Mesos", "framework": True, "name": "kafka", "tags": [ "mesosphere", "framework", "bigdata" ], "versions": [ "0.9.0-beta" ] }, { "currentVersion": "0.8.1", "description": ("A cluster-wide init and control system for " + "services in cgroups or Docker containers."), "framework": True, "name": "marathon", "tags": [ "mesosphere", "framework" ], "versions": [ "0.8.1" ] }, { "currentVersion": "1.4.0-SNAPSHOT", "description": ("Spark is a fast and general cluster " + "computing system for Big Data"), "framework": True, "name": "spark", "tags": [ "mesosphere", "framework", "bigdata" ], "versions": [ "1.4.0-SNAPSHOT" ] } ]).as_dict() PKFH]rڃdcoscli/__init__.py# Version is set for releases by our build system. # Be extremely careful when modifying. version = '0.3.1' """DCOS CLI version""" PKցFHmBBdcoscli/log.pyimport functools import sys import time from dcos import emitting, util from dcos.errors import DCOSException logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() def _no_file_exception(): return DCOSException('No files exist. Exiting.') def log_files(mesos_files, follow, lines): """Print the contents of the given `mesos_files`. Behaves like unix tail. :param mesos_files: file objects to print :type mesos_files: [MesosFile] :param follow: same as unix tail's -f :type follow: bool :param lines: number of lines to print :type lines: int :rtype: None """ fn = functools.partial(_read_last_lines, lines) curr_header, mesos_files = _stream_files(None, fn, mesos_files) if not mesos_files: raise _no_file_exception() while follow: # This flush is needed only for testing, since stdout is fully # buffered (as opposed to line-buffered) when redirected to a # pipe. So if we don't flush, our --follow tests, which use a # pipe, never see the data sys.stdout.flush() curr_header, mesos_files = _stream_files(curr_header, _read_rest, mesos_files) if not mesos_files: raise _no_file_exception() time.sleep(1) def _stream_files(curr_header, fn, mesos_files): """Apply `fn` in parallel to each file in `mesos_files`. `fn` must return a list of strings, and these strings are then printed serially as separate lines. `curr_header` is the most recently printed header. It's used to group lines. Each line has an associated header (e.g. a string representation of the MesosFile it was read from), and we only print the header before printing a line with a different header than the previous line. This effectively groups lines together when the have the same header. :param curr_header: Most recently printed header :type curr_header: str :param fn: function that reads a sequence of lines from a MesosFile :type fn: MesosFile -> [str] :param mesos_files: files to read :type mesos_files: [MesosFile] :returns: Returns the most recently printed header, and a list of files that are still reachable. Once we detect a file is unreachable, we stop trying to read from it. :rtype: (str, [MesosFile]) """ reachable_files = list(mesos_files) # TODO switch to map for job, mesos_file in util.stream(fn, mesos_files): try: lines = job.result() except DCOSException as e: # The read function might throw an exception if read.json # is unavailable, or if the file doesn't exist in the # sandbox. In any case, we silently remove the file and # continue. logger.exception("Error reading file: {}".format(e)) reachable_files.remove(mesos_file) continue if lines: curr_header = _output(curr_header, len(reachable_files) > 1, str(mesos_file), lines) return curr_header, reachable_files def _output(curr_header, output_header, header, lines): """Prints a sequence of lines. If `header` is different than `curr_header`, first print the header. :param curr_header: most recently printed header :type curr_header: str :param output_header: whether or not to output the header :type output_header: bool :param header: header for `lines` :type header: str :param lines: lines to print :type lines: [str] :returns: `header` :rtype: str """ if lines: if output_header and header != curr_header: emitter.publish('===> {} <==='.format(header)) for line in lines: emitter.publish(line) return header # A liberal estimate of a line size. Used to estimate how much data # we need to fetch from a file when we want to read N lines. LINE_SIZE = 200 def _read_last_lines(num_lines, mesos_file): """Returns the last `num_lines` of a file, or less if the file is smaller. Seeks to EOF. :param num_lines: number of lines to read :type num_lines: int :param mesos_file: file to read :type mesos_file: MesosFile :returns: lines read :rtype: [str] """ file_size = mesos_file.size() # estimate how much data we need to fetch to read `num_lines`. fetch_size = LINE_SIZE * num_lines end = file_size start = max(end - fetch_size, 0) data = '' while True: # fetch data mesos_file.seek(start) data = mesos_file.read(end - start) + data # break if we have enough lines data_tmp = _strip_trailing_newline(data) lines = data_tmp.split('\n') if len(lines) > num_lines: ret = lines[-num_lines:] break elif start == 0: ret = lines break # otherwise shift our read window and repeat end = start start = max(end - fetch_size, 0) mesos_file.seek(file_size) return ret def _read_rest(mesos_file): """ Reads the rest of the file, and returns the lines. :param mesos_file: file to read :type mesos_file: MesosFile :returns: lines read :rtype: [str] """ data = mesos_file.read() if data == '': return [] else: data_tmp = _strip_trailing_newline(data) return data_tmp.split('\n') def _strip_trailing_newline(s): """Returns a modified version of the string with the last character truncated if it's a newline. :param s: string to trim :type s: str :returns: modified string :rtype: str """ if s == "": return s else: return s[:-1] if s[-1] == '\n' else s PKցFHȺdcoscli/common.pyimport subprocess def exec_command(cmd, env=None, stdin=None): """Execute CLI command :param cmd: Program and arguments :type cmd: [str] :param env: Environment variables :type env: dict :param stdin: File to use for stdin :type stdin: file :returns: A tuple with the returncode, stdout and stderr :rtype: (int, bytes, bytes) """ process = subprocess.Popen( cmd, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) # This is needed to get rid of '\r' from Windows's lines endings. stdout, stderr = [std_stream.replace(b'\r', b'') for std_stream in process.communicate()] return (process.returncode, stdout, stderr) PKցFH`,`,dcoscli/tables.pyimport copy import datetime import posixpath from collections import OrderedDict import prettytable from dcos import mesos, util EMPTY_ENTRY = '---' DEPLOYMENT_DISPLAY = {'ResolveArtifacts': 'artifacts', 'ScaleApplication': 'scale', 'StartApplication': 'start', 'StopApplication': 'stop', 'RestartApplication': 'restart', 'KillAllOldTasksOf': 'kill-tasks'} logger = util.get_logger(__name__) def task_table(tasks): """Returns a PrettyTable representation of the provided mesos tasks. :param tasks: tasks to render :type tasks: [Task] :rtype: PrettyTable """ fields = OrderedDict([ ("NAME", lambda t: t["name"]), ("HOST", lambda t: t.slave()["hostname"]), ("USER", lambda t: t.user()), ("STATE", lambda t: t["state"].split("_")[-1][0]), ("ID", lambda t: t["id"]), ]) tb = table(fields, tasks, sortby="NAME") tb.align["NAME"] = "l" tb.align["HOST"] = "l" tb.align["ID"] = "l" return tb def app_table(apps, deployments): """Returns a PrettyTable representation of the provided apps. :param tasks: apps to render :type tasks: [dict] :rtype: PrettyTable """ deployment_map = {} for deployment in deployments: deployment_map[deployment['id']] = deployment def get_cmd(app): if app["cmd"] is not None: return app["cmd"] else: return app["args"] def get_container(app): if app["container"] is not None: return app["container"]["type"] else: return "mesos" def get_health(app): if app["healthChecks"]: return "{}/{}".format(app["tasksHealthy"], app["tasksRunning"]) else: return EMPTY_ENTRY def get_deployment(app): deployment_ids = {deployment['id'] for deployment in app['deployments']} actions = [] for deployment_id in deployment_ids: deployment = deployment_map.get(deployment_id) if deployment: for action in deployment['currentActions']: if action['app'] == app['id']: actions.append(DEPLOYMENT_DISPLAY[action['action']]) if len(actions) == 0: return EMPTY_ENTRY elif len(actions) == 1: return actions[0] else: return "({})".format(", ".join(actions)) fields = OrderedDict([ ("ID", lambda a: a["id"]), ("MEM", lambda a: a["mem"]), ("CPUS", lambda a: a["cpus"]), ("TASKS", lambda a: "{}/{}".format(a["tasksRunning"], a["instances"])), ("HEALTH", get_health), ("DEPLOYMENT", get_deployment), ("CONTAINER", get_container), ("CMD", get_cmd) ]) tb = table(fields, apps, sortby="ID") tb.align["CMD"] = "l" tb.align["ID"] = "l" return tb def app_task_table(tasks): """Returns a PrettyTable representation of the provided marathon tasks. :param tasks: tasks to render :type tasks: [dict] :rtype: PrettyTable """ fields = OrderedDict([ ("APP", lambda t: t["appId"]), ("HEALTHY", lambda t: all(check['alive'] for check in t.get('healthCheckResults', []))), ("STARTED", lambda t: t["startedAt"]), ("HOST", lambda t: t["host"]), ("ID", lambda t: t["id"]) ]) tb = table(fields, tasks, sortby="APP") tb.align["APP"] = "l" tb.align["ID"] = "l" return tb def deployment_table(deployments): """Returns a PrettyTable representation of the provided marathon deployments. :param deployments: deployments to render :type deployments: [dict] :rtype: PrettyTable """ def get_action(deployment): multiple_apps = len({action['app'] for action in deployment['currentActions']}) > 1 ret = [] for action in deployment['currentActions']: try: action_display = DEPLOYMENT_DISPLAY[action['action']] except KeyError: logger.exception('Missing action entry') raise ValueError( 'Unknown Marathon action: {}'.format(action['action'])) if multiple_apps: ret.append('{0} {1}'.format(action_display, action['app'])) else: ret.append(action_display) return '\n'.join(ret) fields = OrderedDict([ ('APP', lambda d: '\n'.join(d['affectedApps'])), ('ACTION', get_action), ('PROGRESS', lambda d: '{0}/{1}'.format(d['currentStep']-1, d['totalSteps'])), ('ID', lambda d: d['id']) ]) tb = table(fields, deployments, sortby="APP") tb.align['APP'] = 'l' tb.align['ACTION'] = 'l' tb.align['ID'] = 'l' return tb def service_table(services): """Returns a PrettyTable representation of the provided DCOS services. :param services: services to render :type services: [Framework] :rtype: PrettyTable """ fields = OrderedDict([ ("NAME", lambda s: s['name']), ("HOST", lambda s: s['hostname']), ("ACTIVE", lambda s: s['active']), ("TASKS", lambda s: len(s['tasks'])), ("CPU", lambda s: s['resources']['cpus']), ("MEM", lambda s: s['resources']['mem']), ("DISK", lambda s: s['resources']['disk']), ("ID", lambda s: s['id']), ]) tb = table(fields, services, sortby="NAME") tb.align["ID"] = 'l' tb.align["NAME"] = 'l' return tb def _count_apps(group, group_dict): """Counts how many apps are registered for each group. Recursively populates the profided `group_dict`, which maps group_id -> (group, count). :param group: nested group dictionary :type group: dict :param group_dict: group map that maps group_id -> (group, count) :type group_dict: dict :rtype: dict """ for child_group in group['groups']: _count_apps(child_group, group_dict) count = (len(group['apps']) + sum(group_dict[child_group['id']][1] for child_group in group['groups'])) group_dict[group['id']] = (group, count) def group_table(groups): """Returns a PrettyTable representation of the provided marathon groups :param groups: groups to render :type groups: [dict] :rtype: PrettyTable """ group_dict = {} for group in groups: _count_apps(group, group_dict) fields = OrderedDict([ ('ID', lambda g: g[0]['id']), ('APPS', lambda g: g[1]), ]) tb = table(fields, group_dict.values(), sortby="ID") tb.align['ID'] = 'l' return tb def package_table(packages): """Returns a PrettyTable representation of the provided DCOS packages :param packages: packages to render :type packages: [dict] :rtype: PrettyTable """ fields = OrderedDict([ ('NAME', lambda p: p['name']), ('VERSION', lambda p: p['version']), ('APP', lambda p: '\n'.join(p['apps']) if p.get('apps') else EMPTY_ENTRY), ('COMMAND', lambda p: p['command']['name'] if 'command' in p else EMPTY_ENTRY), ('DESCRIPTION', lambda p: p['description']) ]) tb = table(fields, packages, sortby="NAME") tb.align['NAME'] = 'l' tb.align['VERSION'] = 'l' tb.align['APP'] = 'l' tb.align['COMMAND'] = 'l' tb.align['DESCRIPTION'] = 'l' return tb def package_search_table(search_results): """Returns a PrettyTable representation of the provided DCOS package search results :param search_results: search_results, in the format of dcos.package.IndexEntries::as_dict() :type search_results: [dict] :rtype: PrettyTable """ fields = OrderedDict([ ('NAME', lambda p: p['name']), ('VERSION', lambda p: p['currentVersion']), ('FRAMEWORK', lambda p: p['framework']), ('SOURCE', lambda p: p['source']), ('DESCRIPTION', lambda p: p['description']) ]) packages = [] for result in search_results: for package in result['packages']: package_ = copy.deepcopy(package) package_['source'] = result['source'] packages.append(package_) tb = table(fields, packages, sortby="NAME") tb.align['NAME'] = 'l' tb.align['VERSION'] = 'l' tb.align['FRAMEWORK'] = 'l' tb.align['SOURCE'] = 'l' tb.align['DESCRIPTION'] = 'l' return tb def slave_table(slaves): """Returns a PrettyTable representation of the provided DCOS slaves :param slaves: slaves to render. dicts from /mesos/state-summary :type slaves: [dict] :rtype: PrettyTable """ fields = OrderedDict([ ('HOSTNAME', lambda s: s['hostname']), ('IP', lambda s: mesos.parse_pid(s['pid'])[1]), ('ID', lambda s: s['id']) ]) tb = table(fields, slaves, sortby="HOSTNAME") return tb def _format_unix_timestamp(ts): """ Formats a unix timestamp in a `dcos task ls --long` format. :param ts: unix timestamp :type ts: int :rtype: str """ return datetime.datetime.fromtimestamp(ts).strftime('%b %d %H:%M') def ls_long_table(files): """Returns a PrettyTable representation of `files` :param files: Files to render. Of the form returned from the mesos /files/browse.json endpoint. :param files: [dict] :rtype: PrettyTable """ fields = OrderedDict([ ('MODE', lambda f: f['mode']), ('NLINK', lambda f: f['nlink']), ('UID', lambda f: f['uid']), ('GID', lambda f: f['gid']), ('SIZE', lambda f: f['size']), ('DATE', lambda f: _format_unix_timestamp(int(f['mtime']))), ('PATH', lambda f: posixpath.basename(f['path']))]) tb = table(fields, files, sortby="PATH", header=False) tb.align = 'r' return tb def table(fields, objs, **kwargs): """Returns a PrettyTable. `fields` represents the header schema of the table. `objs` represents the objects to be rendered into rows. :param fields: An OrderedDict, where each element represents a column. The key is the column header, and the value is the function that transforms an element of `objs` into a value for that column. :type fields: OrderdDict(str, function) :param objs: objects to render into rows :type objs: [object] :param **kwargs: kwargs to pass to `prettytable.PrettyTable` :type **kwargs: dict :rtype: PrettyTable """ tb = prettytable.PrettyTable( [k.upper() for k in fields.keys()], border=False, hrules=prettytable.NONE, vrules=prettytable.NONE, left_padding_width=0, right_padding_width=1, **kwargs ) # Set these explicitly due to a bug in prettytable where # '0' values are not honored. tb._left_padding_width = 0 tb._right_padding_width = 2 for obj in objs: row = [fn(obj) for fn in fields.values()] tb.add_row(row) return tb PKցFHÆ  dcoscli/constants.pyROLLBAR_SERVER_POST_KEY = '62f87c5df3674629b143a137de3d3244' SEGMENT_IO_WRITE_KEY_PROD = '51ybGTeFEFU1xo6u10XMDrr6kATFyRyh' SEGMENT_IO_CLI_EVENT = 'dcos-cli' SEGMENT_IO_CLI_ERROR_EVENT = 'dcos-cli-error' SEGMENT_URL = 'https://api.segment.io/v1' DCOS_PRODUCTION_ENV = 'DCOS_PRODUCTION' PKցFH+4dcoscli/analytics.pyimport json import sys import uuid import dcoscli import docopt import rollbar import six from concurrent.futures import ThreadPoolExecutor from dcos import http, mesos, util from dcoscli.constants import (ROLLBAR_SERVER_POST_KEY, SEGMENT_IO_CLI_ERROR_EVENT, SEGMENT_IO_CLI_EVENT, SEGMENT_IO_WRITE_KEY_PROD, SEGMENT_URL) from requests.auth import HTTPBasicAuth logger = util.get_logger(__name__) session_id = uuid.uuid4().hex def wait_and_track(subproc): """ Run a command and report it to analytics services. :param subproc: Subprocess to capture :type subproc: Popen :returns: exit code of subproc :rtype: int """ rollbar.init(ROLLBAR_SERVER_POST_KEY, 'prod') conf = util.get_config() report = conf.get('core.reporting', True) with ThreadPoolExecutor(max_workers=2) as pool: if report: _segment_track_cli(pool, conf) exit_code, err = wait_and_capture(subproc) # We only want to catch exceptions, not other stderr messages # (such as "task does not exist", so we look for the 'Traceback' # string. This only works for python, so we'll need to revisit # this in the future when we support subcommands written in other # languages. if report and 'Traceback' in err: _track_err(pool, exit_code, err, conf) return exit_code def wait_and_capture(subproc): """ Run a subprocess and capture its stderr. :param subproc: Subprocess to capture :type subproc: Popen :returns: exit code of subproc :rtype: int """ err = '' while subproc.poll() is None: line = subproc.stderr.readline().decode('utf-8') err += line sys.stderr.write(line) sys.stderr.flush() exit_code = subproc.poll() return exit_code, err def _segment_track(event, conf, properties): """ Send a segment.io 'track' event :param event: name of event :type event: string :param conf: dcos config file :type conf: Toml :param properties: event properties :type properties: dict :rtype: None """ data = {'event': event, 'properties': properties} if 'core.email' in conf: data['userId'] = conf['core.email'] else: data['anonymousId'] = session_id _segment_request('track', data) def segment_identify(conf): """ Send a segment.io 'identify' event :param conf: dcos config file :type conf: Toml :rtype: None """ if 'core.email' in conf: data = {'userId': conf.get('core.email')} _segment_request('identify', data) def _segment_request(path, data): """ Send a segment.io HTTP request :param path: URL path :type path: str :param data: json POST data :type data: dict :rtype: None """ key = SEGMENT_IO_WRITE_KEY_PROD try: # Set both the connect timeout and the request timeout to 1s, # to prevent rollbar from hanging the CLI commands http.post('{}/{}'.format(SEGMENT_URL, path), json=data, auth=HTTPBasicAuth(key, ''), timeout=(1, 1)) except Exception as e: logger.exception(e) def _track_err(pool, exit_code, err, conf): """ Report error details to analytics services. :param pool: thread pool :type pool: ThreadPoolExecutor :param exit_code: exit code of tracked process :type exit_code: int :param err: stderr of tracked process :type err: str :param conf: dcos config file :type conf: Toml :rtype: None """ # Segment.io calls are async, but rollbar is not, so for # parallelism, we must call segment first. _segment_track_err(pool, conf, err, exit_code) _rollbar_track_err(conf, err, exit_code) def _segment_track_cli(pool, conf): """ Send segment.io cli event. :param pool: thread pool :type pool: ThreadPoolExecutor :param conf: dcos config file :type conf: Toml :rtype: None """ props = _base_properties(conf) pool.submit(_segment_track, SEGMENT_IO_CLI_EVENT, conf, props) def _segment_track_err(pool, conf, err, exit_code): """ Send segment.io error event. :param pool: thread pool :type segment: ThreadPoolExecutor :param conf: dcos config file :type conf: Toml :param err: stderr of tracked process :type err: str :param exit_code: exit code of tracked process :type exit_code: int :rtype: None """ props = _base_properties(conf) props['err'] = err props['exit_code'] = exit_code pool.submit(_segment_track, SEGMENT_IO_CLI_ERROR_EVENT, conf, props) def _rollbar_track_err(conf, err, exit_code): """ Report to rollbar. Synchronous. :param exit_code: exit code of tracked process :type exit_code: int :param err: stderr of tracked process :type err: str :param conf: dcos config file :type conf: Toml :rtype: None """ props = _base_properties(conf) props['exit_code'] = exit_code lines = err.split('\n') if len(lines) >= 2: title = lines[-2] else: title = err props['stderr'] = err try: rollbar.report_message(title, 'error', extra_data=props) except Exception as e: logger.exception(e) def _command(): """ Return the subcommand used in this dcos process. :returns: subcommand used in this dcos process :rtype: str """ # avoid circular import import dcoscli.main args = docopt.docopt(dcoscli.main._doc(), help=False, options_first=True) return args[''] def _base_properties(conf=None): """ These properties are sent with every analytics event. :param conf: dcos config file :type conf: Toml :rtype: dict """ if not conf: conf = util.get_config() if len(sys.argv) > 1: cmd = 'dcos ' + _command() full_cmd = 'dcos ' + ' '.join(sys.argv[1:]) else: cmd = 'dcos' full_cmd = 'dcos' try: dcos_hostname = six.moves.urllib.parse.urlparse( conf.get('core.dcos_url')).hostname except: logger.exception('Unable to find the hostname of the cluster.') dcos_hostname = None try: cluster_id = mesos.DCOSClient().metadata().get('CLUSTER_ID') except: logger.exception('Unable to get the cluster_id of the cluster.') cluster_id = None return { 'cmd': cmd, 'full_cmd': full_cmd, 'dcoscli.version': dcoscli.version, 'python_version': str(sys.version_info), 'config': json.dumps(list(conf.property_items())), 'DCOS_HOSTNAME': dcos_hostname, 'CLUSTER_ID': cluster_id } PKցFH dcoscli/main.pyimport os import signal import sys from functools import wraps from subprocess import PIPE, Popen import dcoscli import docopt import pkg_resources from dcos import auth, constants, emitting, errors, http, subcommand, util from dcos.errors import DCOSException from dcoscli import analytics emitter = emitting.FlatEmitter() def main(): try: return _main() except DCOSException as e: emitter.publish(e) return 1 def _main(): signal.signal(signal.SIGINT, signal_handler) args = docopt.docopt( _doc(), version='dcos version {}'.format(dcoscli.version), options_first=True) log_level = args['--log-level'] if log_level and not _config_log_level_environ(log_level): return 1 if args['--debug']: os.environ[constants.DCOS_DEBUG_ENV] = 'true' util.configure_process_from_environ() if args[''] != 'config' and \ not auth.check_if_user_authenticated(): auth.force_auth() config = util.get_config() set_ssl_info_env_vars(config) command = args[''] http.silence_requests_warnings() if not command: command = "help" executable = subcommand.command_executables(command) subproc = Popen([executable, command] + args[''], stderr=PIPE) if dcoscli.version != 'SNAPSHOT': return analytics.wait_and_track(subproc) else: return analytics.wait_and_capture(subproc)[0] def _doc(): """ :rtype: str """ return pkg_resources.resource_string( 'dcoscli', 'data/help/dcos.txt').decode('utf-8') def _config_log_level_environ(log_level): """ :param log_level: Log level to set :type log_level: str :returns: True if the log level was configured correctly; False otherwise. :rtype: bool """ log_level = log_level.lower() if log_level in constants.VALID_LOG_LEVEL_VALUES: os.environ[constants.DCOS_LOG_LEVEL_ENV] = log_level return True msg = 'Log level set to an unknown value {!r}. Valid values are {!r}' emitter.publish(msg.format(log_level, constants.VALID_LOG_LEVEL_VALUES)) return False def signal_handler(signal, frame): emitter.publish( errors.DefaultError("User interrupted command with Ctrl-C")) sys.exit(0) def decorate_docopt_usage(func): """Handle DocoptExit exception :param func: function :type func: function :return: wrapped function :rtype: function """ @wraps(func) def wrapper(*args, **kwargs): try: result = func(*args, **kwargs) except docopt.DocoptExit as e: emitter.publish("Command not recognized\n") emitter.publish(e) return 1 return result return wrapper def set_ssl_info_env_vars(config): """Set SSL info from config to environment variable if enviornment variable doesn't exist :param config: config :type config: Toml :rtype: None """ if 'core.ssl_verify' in config and ( not os.environ.get(constants.DCOS_SSL_VERIFY_ENV)): os.environ[constants.DCOS_SSL_VERIFY_ENV] = str( config['core.ssl_verify']) PKցFHdcoscli/marathon/__init__.pyPKցFH8 5X5Xdcoscli/marathon/main.pyimport json import os import sys import time import dcoscli import docopt import pkg_resources from dcos import cmds, emitting, http, jsonitem, marathon, options, util from dcos.errors import DCOSException from dcoscli import tables from dcoscli.main import decorate_docopt_usage logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() def main(): try: return _main() except DCOSException as e: emitter.publish(e) return 1 @decorate_docopt_usage def _main(): util.configure_process_from_environ() args = docopt.docopt( _doc(), version='dcos-marathon version {}'.format(dcoscli.version)) return cmds.execute(_cmds(), args) def _doc(): """ :rtype: str """ return pkg_resources.resource_string( 'dcoscli', 'data/help/marathon.txt').decode('utf-8') def _cmds(): """ :returns: all the supported commands :rtype: dcos.cmds.Command """ return [ cmds.Command( hierarchy=['marathon', 'version', 'list'], arg_keys=['', '--max-count'], function=_version_list), cmds.Command( hierarchy=['marathon', 'deployment', 'list'], arg_keys=['', '--json'], function=_deployment_list), cmds.Command( hierarchy=['marathon', 'deployment', 'rollback'], arg_keys=[''], function=_deployment_rollback), cmds.Command( hierarchy=['marathon', 'deployment', 'stop'], arg_keys=[''], function=_deployment_stop), cmds.Command( hierarchy=['marathon', 'deployment', 'watch'], arg_keys=['', '--max-count', '--interval'], function=_deployment_watch), cmds.Command( hierarchy=['marathon', 'task', 'list'], arg_keys=['', '--json'], function=_task_list), cmds.Command( hierarchy=['marathon', 'task', 'show'], arg_keys=[''], function=_task_show), cmds.Command( hierarchy=['marathon', 'app', 'add'], arg_keys=[''], function=_add), cmds.Command( hierarchy=['marathon', 'app', 'list'], arg_keys=['--json'], function=_list), cmds.Command( hierarchy=['marathon', 'app', 'remove'], arg_keys=['', '--force'], function=_remove), cmds.Command( hierarchy=['marathon', 'app', 'show'], arg_keys=['', '--app-version'], function=_show), cmds.Command( hierarchy=['marathon', 'app', 'start'], arg_keys=['', '', '--force'], function=_start), cmds.Command( hierarchy=['marathon', 'app', 'stop'], arg_keys=['', '--force'], function=_stop), cmds.Command( hierarchy=['marathon', 'app', 'update'], arg_keys=['', '', '--force'], function=_update), cmds.Command( hierarchy=['marathon', 'app', 'restart'], arg_keys=['', '--force'], function=_restart), cmds.Command( hierarchy=['marathon', 'app', 'kill'], arg_keys=['', '--scale', '--host'], function=_kill), cmds.Command( hierarchy=['marathon', 'group', 'add'], arg_keys=[''], function=_group_add), cmds.Command( hierarchy=['marathon', 'group', 'list'], arg_keys=['--json'], function=_group_list), cmds.Command( hierarchy=['marathon', 'group', 'show'], arg_keys=['', '--group-version'], function=_group_show), cmds.Command( hierarchy=['marathon', 'group', 'remove'], arg_keys=['', '--force'], function=_group_remove), cmds.Command( hierarchy=['marathon', 'group', 'update'], arg_keys=['', '', '--force'], function=_group_update), cmds.Command( hierarchy=['marathon', 'group', 'scale'], arg_keys=['', '', '--force'], function=_group_scale), cmds.Command( hierarchy=['marathon', 'about'], arg_keys=[], function=_about), cmds.Command( hierarchy=['marathon'], arg_keys=['--config-schema', '--info'], function=_marathon) ] def _marathon(config_schema, info): """ :param config_schema: Whether to output the config schema :type config_schema: boolean :param info: Whether to output a description of this subcommand :type info: boolean :returns: process return code :rtype: int """ if config_schema: schema = _cli_config_schema() emitter.publish(schema) elif info: _info() else: emitter.publish(options.make_generic_usage_message(_doc())) return 1 return 0 def _info(): """ :returns: process return code :rtype: int """ emitter.publish(_doc().split('\n')[0]) return 0 def _about(): """ :returns: process return code :rtype: int """ client = marathon.create_client() emitter.publish(client.get_about()) return 0 def _get_resource(resource): """ :param resource: optional filename or http(s) url for the application or group resource :type resource: str :returns: resource :rtype: dict """ if resource is not None: if os.path.isfile(resource): with util.open_file(resource) as resource_file: return util.load_json(resource_file) else: try: http.silence_requests_warnings() req = http.get(resource) if req.status_code == 200: data = b'' for chunk in req.iter_content(1024): data += chunk return util.load_jsons(data.decode('utf-8')) else: raise Exception except Exception: logger.exception('Cannot read from resource %s', resource) raise DCOSException( "Can't read from resource: {0}.\n" "Please check that it exists.".format(resource)) # Check that stdin is not tty if sys.stdin.isatty(): # We don't support TTY right now. In the future we will start an # editor raise DCOSException( "We currently don't support reading from the TTY. Please " "specify an application JSON.\n" "E.g.: dcos marathon app add < app_resource.json") return util.load_json(sys.stdin) def _add(app_resource): """ :param app_resource: optional filename for the application resource :type app_resource: str :returns: process return code :rtype: int """ application_resource = _get_resource(app_resource) # Add application to marathon client = marathon.create_client() # Check that the application doesn't exist app_id = client.normalize_app_id(application_resource['id']) try: client.get_app(app_id) except DCOSException as e: logger.exception(e) else: raise DCOSException("Application '{}' already exists".format(app_id)) client.add_app(application_resource) return 0 def _list(json_): """ :param json_: output json if True :type json_: bool :returns: process return code :rtype: int """ client = marathon.create_client() apps = client.get_apps() if json_: emitter.publish(apps) else: deployments = client.get_deployments() table = tables.app_table(apps, deployments) output = str(table) if output: emitter.publish(output) return 0 def _group_list(json_): """ :param json_: output json if True :type json_: bool :rtype: int :returns: process return code """ client = marathon.create_client() groups = client.get_groups() emitting.publish_table(emitter, groups, tables.group_table, json_) return 0 def _group_add(group_resource): """ :param group_resource: optional filename for the group resource :type group_resource: str :returns: process return code :rtype: int """ group_resource = _get_resource(group_resource) client = marathon.create_client() # Check that the group doesn't exist group_id = client.normalize_app_id(group_resource['id']) try: client.get_group(group_id) except DCOSException as e: logger.exception(e) else: raise DCOSException("Group '{}' already exists".format(group_id)) client.create_group(group_resource) return 0 def _remove(app_id, force): """ :param app_id: ID of the app to remove :type app_id: str :param force: Whether to override running deployments. :type force: bool :returns: process return code :rtype: int """ client = marathon.create_client() client.remove_app(app_id, force) return 0 def _group_remove(group_id, force): """ :param group_id: ID of the app to remove :type group_id: str :param force: Whether to override running deployments. :type force: bool :returns: process return code :rtype: int """ client = marathon.create_client() client.remove_group(group_id, force) return 0 def _show(app_id, version): """Show details of a Marathon application. :param app_id: The id for the application :type app_id: str :param version: The version, either absolute (date-time) or relative :type version: str :returns: process return code :rtype: int """ client = marathon.create_client() if version is not None: version = _calculate_version(client, app_id, version) app = client.get_app(app_id, version=version) emitter.publish(app) return 0 def _group_show(group_id, version=None): """Show details of a Marathon application. :param group_id: The id for the application :type group_id: str :param version: The version, either absolute (date-time) or relative :type version: str :returns: process return code :rtype: int """ client = marathon.create_client() app = client.get_group(group_id, version=version) emitter.publish(app) return 0 def _group_update(group_id, properties, force): """ :param group_id: the id of the group :type group_id: str :param properties: json items used to update group :type properties: [str] :param force: whether to override running deployments :type force: bool :returns: process return code :rtype: int """ client = marathon.create_client() # Ensure that the group exists client.get_group(group_id) properties = _parse_properties(properties) deployment = client.update_group(group_id, properties, force) emitter.publish('Created deployment {}'.format(deployment)) return 0 def _start(app_id, instances, force): """Start a Marathon application. :param app_id: the id for the application :type app_id: str :param instances: the number of instances to start :type instances: str :param force: whether to override running deployments :type force: bool :returns: process return code :rtype: int """ # Check that the application exists client = marathon.create_client() desc = client.get_app(app_id) if desc['instances'] > 0: emitter.publish( 'Application {!r} already started: {!r} instances.'.format( app_id, desc['instances'])) return 1 # Need to add the 'id' because it is required app_json = {'id': app_id} # Set instances to 1 if not specified if instances is None: instances = 1 else: instances = util.parse_int(instances) if instances <= 0: emitter.publish( 'The number of instances must be positive: {!r}.'.format( instances)) return 1 app_json['instances'] = instances deployment = client.update_app(app_id, app_json, force) emitter.publish('Created deployment {}'.format(deployment)) return 0 def _stop(app_id, force): """Stop a Marathon application :param app_id: the id of the application :type app_id: str :param force: whether to override running deployments :type force: bool :returns: process return code :rtype: int """ # Check that the application exists client = marathon.create_client() desc = client.get_app(app_id) if desc['instances'] <= 0: emitter.publish( 'Application {!r} already stopped: {!r} instances.'.format( app_id, desc['instances'])) return 1 app_json = {'instances': 0} deployment = client.update_app(app_id, app_json, force) emitter.publish('Created deployment {}'.format(deployment)) def _update(app_id, properties, force): """ :param app_id: the id of the application :type app_id: str :param properties: json items used to update resource :type properties: [str] :param force: whether to override running deployments :type force: bool :returns: process return code :rtype: int """ client = marathon.create_client() # Ensure that the application exists client.get_app(app_id) properties = _parse_properties(properties) deployment = client.update_app(app_id, properties, force) emitter.publish('Created deployment {}'.format(deployment)) return 0 def _group_scale(group_id, scale_factor, force): """ :param group_id: the id of the group :type group_id: str :param scale_factor: scale factor for application group :type scale_factor: str :param force: whether to override running deployments :type force: bool :returns: process return code :rtype: int """ client = marathon.create_client() scale_factor = util.parse_float(scale_factor) deployment = client.scale_group(group_id, scale_factor, force) emitter.publish('Created deployment {}'.format(deployment)) return 0 def _parse_properties(properties): """ :param properties: JSON items in the form key=value :type properties: [str] :returns: resource JSON :rtype: dict """ if len(properties) == 0: if sys.stdin.isatty(): # We don't support TTY right now. In the future we will start an # editor raise DCOSException( "We currently don't support reading from the TTY. Please " "specify an application JSON.\n" "E.g. dcos marathon app update your-app-id < app_update.json") else: return util.load_jsons(sys.stdin.read()) resource_json = {} for prop in properties: key, value = jsonitem.parse_json_item(prop, None) key = jsonitem.clean_value(key) if key in resource_json: raise DCOSException( 'Key {!r} was specified more than once'.format(key)) resource_json[key] = value return resource_json def _restart(app_id, force): """ :param app_id: the id of the application :type app_id: str :param force: whether to override running deployments :type force: bool :returns: process return code :rtype: int """ client = marathon.create_client() desc = client.get_app(app_id) if desc['instances'] <= 0: app_id = client.normalize_app_id(app_id) emitter.publish( 'Unable to perform rolling restart of application {!r} ' 'because it has no running tasks'.format( app_id, desc['instances'])) return 1 payload = client.restart_app(app_id, force) emitter.publish('Created deployment {}'.format(payload['deploymentId'])) return 0 def _kill(app_id, scale, host): """ :param app_id: the id of the application :type app_id: str :param: scale: Scale the app down :type: scale: bool :param: host: Kill only those tasks running on host specified :type: string :returns: process return code :rtype: int """ client = marathon.create_client() payload = client.kill_tasks(app_id, host=host, scale=scale) # If scale is provided, the API return a "deploymentResult" # https://github.com/mesosphere/marathon/blob/50366c8/src/main/scala/mesosphere/marathon/api/RestResource.scala#L34-L36 if scale: emitter.publish("Started deployment: {}".format(payload)) else: if 'tasks' in payload: emitter.publish('Killed tasks: {}'.format(payload['tasks'])) if len(payload['tasks']) == 0: return 1 else: emitter.publish('Killed tasks: []') return 1 return 0 def _version_list(app_id, max_count): """ :param app_id: the id of the application :type app_id: str :param max_count: the maximum number of version to fetch and return :type max_count: str :returns: process return code :rtype: int """ if max_count is not None: max_count = util.parse_int(max_count) client = marathon.create_client() versions = client.get_app_versions(app_id, max_count) emitter.publish(versions) return 0 def _deployment_list(app_id, json_): """ :param app_id: the application id :type app_id: str :param json_: output json if True :type json_: bool :returns: process return code :rtype: int """ client = marathon.create_client() deployments = client.get_deployments(app_id) if not deployments and not json_: msg = "There are no deployments" if app_id: msg += " for '{}'".format(app_id) raise DCOSException(msg) emitting.publish_table(emitter, deployments, tables.deployment_table, json_) return 0 def _deployment_stop(deployment_id): """ :param deployment_id: the application id :type deployment_di: str :returns: process return code :rtype: int """ client = marathon.create_client() client.stop_deployment(deployment_id) return 0 def _deployment_rollback(deployment_id): """ :param deployment_id: the application id :type deployment_di: str :returns: process return code :rtype: int """ client = marathon.create_client() deployment = client.rollback_deployment(deployment_id) emitter.publish(deployment) return 0 def _deployment_watch(deployment_id, max_count, interval): """ :param deployment_id: the application id :type deployment_di: str :param max_count: maximum number of polling calls :type max_count: str :param interval: wait interval in seconds between polling calls :type interval: str :returns: process return code :rtype: int """ if max_count is not None: max_count = util.parse_int(max_count) interval = 1 if interval is None else util.parse_int(interval) client = marathon.create_client() count = 0 while max_count is None or count < max_count: deployment = client.get_deployment(deployment_id) if deployment is None: return 0 if util.is_windows_platform(): os.system('cls') else: if 'TERM' in os.environ: os.system('clear') emitter.publish('Deployment update time: ' '{} \n'.format(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))) emitter.publish(deployment) time.sleep(interval) count += 1 return 0 def _task_list(app_id, json_): """ :param app_id: the id of the application :type app_id: str :param json_: output json if True :type json_: bool :returns: process return code :rtype: int """ client = marathon.create_client() tasks = client.get_tasks(app_id) emitting.publish_table(emitter, tasks, tables.app_task_table, json_) return 0 def _task_show(task_id): """ :param task_id: the task id :type task_id: str :returns: process return code :rtype: int """ client = marathon.create_client() task = client.get_task(task_id) if task is None: raise DCOSException("Task '{}' does not exist".format(task_id)) emitter.publish(task) return 0 def _calculate_version(client, app_id, version): """ :param client: Marathon client :type client: dcos.marathon.Client :param app_id: The ID of the application :type app_id: str :param version: Relative or absolute version or None :type version: str :returns: The absolute version as an ISO8601 date-time :rtype: str """ # First let's try to parse it as a negative integer try: value = util.parse_int(version) except DCOSException: logger.exception('Unable to parse version %s', version) return version else: if value < 0: value = -1 * value # We have a negative value let's ask Marathon for the last # abs(value) versions = client.get_app_versions(app_id, value + 1) if len(versions) <= value: # We don't have enough versions. Return an error. msg = "Application {!r} only has {!r} version(s)." raise DCOSException(msg.format(app_id, len(versions), value)) else: return versions[value] else: raise DCOSException( 'Relative versions must be negative: {}'.format(version)) def _cli_config_schema(): """ :returns: schema for marathon cli config :rtype: dict """ return json.loads( pkg_resources.resource_string( 'dcoscli', 'data/config-schema/marathon.json').decode('utf-8')) PKցFHj?O)dcoscli/data/universe-schema/package.json{ "$schema": "http://json-schema.org/schema#", "type": "object", "properties": { "name": { "type": "string" }, "version": { "type": "string" }, "scm": { "type": "string" }, "maintainer": { "type": "string" }, "website": { "type": "string" }, "description": { "type": "string" }, "framework": { "type": "boolean", "default": false, "description": "True if this package installs a new Mesos framework." }, "preInstallNotes": { "type": "string", "description": "Pre installation notes that would be useful to the user of this package." }, "postInstallNotes": { "type": "string", "description": "Post installation notes that would be useful to the user of this package." }, "postUninstallNotes": { "type": "string", "description": "Post uninstallation notes that would be useful to the user of this package." }, "tags": { "type": "array", "items": { "type": "string", "pattern": "^[^\\s]+$" } }, "licenses": { "type": "array", "items": { "type": "object", "properties": { "name": { "type": "string", "description": "The name of the license. For example one of [Apache License Version 2.0 | MIT License | BSD License | Proprietary]" }, "url": { "type": "string", "pattern": "((?<=\\()[A-Za-z][A-Za-z0-9\\+\\.\\-]*:([A-Za-z0-9\\.\\-_~:/\\?#\\[\\]@!\\$&'\\(\\)\\*\\+,;=]|%[A-Fa-f0-9]{2})+(?=\\)))|([A-Za-z][A-Za-z0-9\\+\\.\\-]*:([A-Za-z0-9\\.\\-_~:/\\?#\\[\\]@!\\$&'\\(\\)\\*\\+,;=]|%[A-Fa-f0-9]{2})+)", "description": "The URL where the license can be accessed" } }, "additionalProperties": false, "required": [ "name", "url" ] } } }, "required": [ "name", "version", "maintainer", "description", "tags" ], "additionalProperties": false } PKցFHt,,)dcoscli/data/universe-schema/command.json{ "$schema": "http://json-schema.org/schema#", "oneOf": [ { "type": "object", "properties": { "pip": { "type": "array", "items": { "type": "string" }, "title": "Embedded Requirements File", "description": "An array of strings representing of the requirements file to use for installing the subcommand for Pip. Each item is interpreted as a line in the requirements file." } }, "additionalProperties": false, "required": ["pip"] } ] } PKցFHN(dcoscli/data/universe-schema/config.json{ "id": "http://json-schema.org/draft-04/schema#", "$schema": "http://json-schema.org/draft-04/schema#", "description": "Core schema meta-schema", "definitions": { "schemaArray": { "type": "array", "minItems": 1, "items": { "$ref": "#" } }, "positiveInteger": { "type": "integer", "minimum": 0 }, "positiveIntegerDefault0": { "allOf": [ { "$ref": "#/definitions/positiveInteger" }, { "default": 0 } ] }, "simpleTypes": { "enum": [ "array", "boolean", "integer", "null", "number", "object", "string" ] }, "stringArray": { "type": "array", "items": { "type": "string" }, "minItems": 1, "uniqueItems": true } }, "type": "object", "properties": { "id": { "type": "string", "format": "uri" }, "$schema": { "type": "string", "format": "uri" }, "title": { "type": "string" }, "description": { "type": "string" }, "default": {}, "multipleOf": { "type": "number", "minimum": 0, "exclusiveMinimum": true }, "maximum": { "type": "number" }, "exclusiveMaximum": { "type": "boolean", "default": false }, "minimum": { "type": "number" }, "exclusiveMinimum": { "type": "boolean", "default": false }, "maxLength": { "$ref": "#/definitions/positiveInteger" }, "minLength": { "$ref": "#/definitions/positiveIntegerDefault0" }, "pattern": { "type": "string", "format": "regex" }, "additionalItems": { "anyOf": [ { "type": "boolean" }, { "$ref": "#" } ], "default": {} }, "items": { "anyOf": [ { "$ref": "#" }, { "$ref": "#/definitions/schemaArray" } ], "default": {} }, "maxItems": { "$ref": "#/definitions/positiveInteger" }, "minItems": { "$ref": "#/definitions/positiveIntegerDefault0" }, "uniqueItems": { "type": "boolean", "default": false }, "maxProperties": { "$ref": "#/definitions/positiveInteger" }, "minProperties": { "$ref": "#/definitions/positiveIntegerDefault0" }, "required": { "$ref": "#/definitions/stringArray" }, "additionalProperties": { "anyOf": [ { "type": "boolean" }, { "$ref": "#" } ], "default": {} }, "definitions": { "type": "object", "additionalProperties": { "$ref": "#" }, "default": {} }, "properties": { "type": "object", "additionalProperties": { "$ref": "#" }, "default": {} }, "patternProperties": { "type": "object", "additionalProperties": { "$ref": "#" }, "default": {} }, "dependencies": { "type": "object", "additionalProperties": { "anyOf": [ { "$ref": "#" }, { "$ref": "#/definitions/stringArray" } ] } }, "enum": { "type": "array", "minItems": 1, "uniqueItems": true }, "type": { "anyOf": [ { "$ref": "#/definitions/simpleTypes" }, { "type": "array", "items": { "$ref": "#/definitions/simpleTypes" }, "minItems": 1, "uniqueItems": true } ] }, "allOf": { "$ref": "#/definitions/schemaArray" }, "anyOf": { "$ref": "#/definitions/schemaArray" }, "oneOf": { "$ref": "#/definitions/schemaArray" }, "not": { "$ref": "#" } }, "dependencies": { "exclusiveMaximum": [ "maximum" ], "exclusiveMinimum": [ "minimum" ] }, "default": {} } PKցFH 'dcoscli/data/config-schema/package.json{ "$schema": "http://json-schema.org/schema#", "type": "object", "properties": { "sources": { "type": "array", "items": { "type": "string", "pattern": "^((?:(?:(https?|file))://)(?:(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\\.?)+(?:[a-zA-Z]{2,6}\\.?|[a-zA-Z0-9-]{2,}\\.?)?|\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})?(?::\\d+)?(?:/?|[/?]\\S+)|((git|ssh|https?)|(git@[\\w\\.]+))(:(//)?)([\\w\\.@\\:/\\-~]+)(\\.git)(/)?)$" }, "title": "Package sources", "description": "The list of package source in search order", "default": [ "git://github.com/mesosphere/universe.git" ], "additionalItems": false, "uniqueItems": true }, "cache": { "type": "string", "title": "Package cache directory", "description": "Path to the local package cache directory", "default": "/tmp/cache" } }, "additionalProperties": false, "required": ["sources", "cache"] } PKցFHV-zz(dcoscli/data/config-schema/marathon.json{ "$schema": "http://json-schema.org/schema#", "type": "object", "properties": { "url": { "type": "string", "format": "uri", "title": "Marathon base URL", "description": "Base URL for talking to Marathon. It overwrites the value specified in core.dcos_url", "default": "http://localhost:8080" } }, "additionalProperties": false } PKցFH+#$dcoscli/data/config-schema/core.json{ "$schema": "http://json-schema.org/schema#", "additionalProperties": false, "properties": { "dcos_url": { "description": "The URL to the location of the DCOS", "format": "uri", "title": "DCOS URL", "type": "string" }, "email": { "description": "Your email address", "title": "Your email address", "type": "string" }, "mesos_master_url": { "description": "Mesos Master URL. Must be of the format: \"http://host:port\"", "format": "uri", "title": "Mesos Master URL", "type": "string" }, "refresh_token": { "description": "Your OAuth refresh token", "title": "Your OAuth refresh token", "type": "string" }, "reporting": { "default": true, "description": "Whether to report usage events to Mesosphere", "title": "Usage Reporting", "type": "boolean" }, "timeout": { "default": 5, "description": "Request timeout in seconds", "minimum": 1, "title": "Request timeout in seconds", "type": "integer" }, "token": { "description": "Your OAuth access token", "title": "Your OAuth access token", "type": "string" }, "ssl_verify": { "type": "string", "default": "false", "title": "SSL Verification", "description": "Whether or not to verify SSL certs for HTTPS or path to cert(s)" } }, "type": "object" } PKցFH%dcoscli/data/help/package.txtInstall and manage DCOS packages Usage: dcos package --config-schema dcos package --info dcos package describe [--app --cli --config] [--render] [--package-versions] [--options=] [--package-version=] dcos package install [--cli | [--app --app-id=]] [--package-version=] [--options=] [--yes] dcos package list [--json --endpoints --app-id= ] dcos package search [--json ] dcos package sources dcos package uninstall [--cli | [--app --app-id= --all]] dcos package update [--validate] Options: --all Apply the operation to all matching packages --app Apply the operation only to the package's Marathon application --app-id= The application id --cli Apply the operation only to the package's CLI command --config Print the package's config.json, which contains the configurable properties for marathon.json and command.json -h, --help Show this screen --info Show a short description of this subcommand --options= Path to a JSON file containing package installation options --package-version= Package version to install --package-versions Print all versions for this package --render Render the package's marathon.json or command.json template with the values from config.json and --options. If not provided, print the raw templates. --validate Validate package content when updating sources --version Show version --yes Assume "yes" is the answer to all prompts and run non-interactively Positional Arguments: Name of the DCOS package Pattern to use for searching for package PKցFH)1dcoscli/data/help/config.txtGet and set DCOS CLI configuration properties Usage: dcos config --info dcos config append dcos config prepend dcos config set dcos config show [] dcos config unset [--index=] dcos config validate Options: -h, --help Show this screen --info Show a short description of this subcommand --version Show version --index= Index into the list. The first element in the list has an index of zero Positional Arguments: The name of the property The value of the property PKցFH/2dcoscli/data/help/dcos.txtCommand line utility for the Mesosphere Datacenter Operating System (DCOS) 'dcos help' lists all available subcommands. See 'dcos --help' to read about a specific subcommand. Usage: dcos [options] [] [...] Options: --help Show this screen --version Show version --log-level= If set then print supplementary messages to stderr at or above this level. The severity levels in the order of severity are: debug, info, warning, error, and critical. E.g. Setting the option to warning will print warning, error and critical messages to stderr. Note: that this does not affect the output sent to stdout by the command. --debug If set then enable further debug messages which are sent to stdout. Environment Variables: DCOS_LOG_LEVEL If set then it specifies that message should be printed to stderr at or above this level. See the --log-level option for details. DCOS_CONFIG This environment variable points to the location of the DCOS configuration file. [default: ~/.dcos/dcos.toml] DCOS_DEBUG If set then enable further debug messages which are sent to stdout. DCOS_SSL_VERIFY If set, specifies whether to verify SSL certs for HTTPS, or the path to the certificate(s). Can also be configured by setting `core.ssl_config` in the config. PKցFHVOOdcoscli/data/help/task.txtManage DCOS tasks Usage: dcos task --info dcos task [--completed --json ] dcos task log [--completed --follow --lines=N] [] dcos task ls [--long] [] Options: -h, --help Show this screen --info Show a short description of this subcommand --completed Include completed tasks as well --follow Print data as the file grows --json Print json-formatted tasks --lines=N Print the last N lines [default: 10] --long Use a long listing format --version Show version Positional Arguments: Print this file. [default: stdout] List this directory. [default: '.'] Only match tasks whose ID matches . may be a substring of the ID, or a unix glob pattern. PKցFHy\\dcoscli/data/help/service.txtManage DCOS services Usage: dcos service --info dcos service [--completed --inactive --json] dcos service log [--follow --lines=N --ssh-config-file=] [] dcos service shutdown Options: -h, --help Show this screen --completed Show completed services in addition to active ones. Completed services are those that have been disconnected from master, and have reached their failover timeout, or have been explicitly shutdown via the /shutdown endpoint. --inactive Show inactive services in addition to active ones. Inactive services are those that have been disconnected from master, but haven't yet reached their failover timeout. --info Show a short description of this subcommand --follow Print data as the file grows --json Print json-formatted services --lines=N Print the last N lines [default: 10] --ssh-config-file= Path to SSH config file. Used to access marathon logs. --version Show version Positional Arguments: Output this file. [default: stdout] The DCOS Service name. The DCOS Service ID PKցFHkfdcoscli/data/help/help.txtDisplay command line usage information Usage: dcos help dcos help --info dcos help Options: --help Show this screen --info Show a short description of this subcommand --version Show version PKցFH|dcoscli/data/help/node.txtManage DCOS nodes Usage: dcos node --info dcos node [--json] dcos node log [--follow --lines=N --master --slave=] dcos node ssh [--option SSHOPT=VAL ...] [--config-file=] [--user=] [--master-proxy] (--master | --slave=) Options: -h, --help Show this screen --info Show a short description of this subcommand --json Print json-formatted nodes --follow Print data as the file grows --lines=N Print the last N lines [default: 10] --master Access the leading master --master-proxy Proxy the SSH connection through a master node. This can be useful when accessing DCOS from a separate network. For example, in the default AWS configuration, the private slaves are unreachable from the public internet. You can access them using this option, which will first hop from the publicly available master. --slave= Access the slave with the provided ID --option SSHOPT=VAL SSH option (see `man ssh_config`) --config-file= Path to SSH config file --user= SSH user [default: core] --version Show version PKցFH**dcoscli/data/help/marathon.txtDeploy and manage applications on the DCOS Usage: dcos marathon --config-schema dcos marathon --info dcos marathon about dcos marathon app add [] dcos marathon app list [--json] dcos marathon app remove [--force] dcos marathon app restart [--force] dcos marathon app show [--app-version=] dcos marathon app start [--force] [] dcos marathon app stop [--force] dcos marathon app kill [--scale] [--host=] dcos marathon app update [--force] [...] dcos marathon app version list [--max-count=] dcos marathon deployment list [--json ] dcos marathon deployment rollback dcos marathon deployment stop dcos marathon deployment watch [--max-count=] [--interval=] dcos marathon task list [--json ] dcos marathon task show dcos marathon group add [] dcos marathon group list [--json] dcos marathon group scale [--force] dcos marathon group show [--group-version=] dcos marathon group remove [--force] dcos marathon group update [--force] [...] Options: -h, --help Show this screen --info Show a short description of this subcommand --json Print json-formatted tasks --version Show version --force This flag disable checks in Marathon during update operations --app-version= This flag specifies the application version to use for the command. The application version () can be specified as an absolute value or as relative value. Absolute version values must be in ISO8601 date format. Relative values must be specified as a negative integer and they represent the version from the currently deployed application definition --group-version= This flag specifies the group version to use for the command. The group version () can be specified as an absolute value or as relative value. Absolute version values must be in ISO8601 date format. Relative values must be specified as a negative integer and they represent the version from the currently deployed group definition --config-schema Show the configuration schema for the Marathon subcommand --max-count= Maximum number of entries to try to fetch and return --interval= Number of seconds to wait between actions --scale Scale the app down after performing the the operation. --host= The host name to isolate your command to Positional Arguments: The application id Path to a file or HTTP(S) URL containing the app's JSON definition. If omitted, the definition is read from stdin. For a detailed description see (https://mesosphere.github.io/ marathon/docs/rest-api.html#post-/v2/apps). The deployment id The group id Path to a file or HTTP(S) URL containing the group's JSON definition. If omitted, the definition is read from stdin. For a detailed description see (https://mesosphere.github.io/ marathon/docs/rest-api.html#post-/v2/groups). The number of instances to start Must be of the format =. E.g. cpus=2.0. If omitted, properties are read from stdin. The task id The factor to scale an application group by PKցFHdcoscli/node/__init__.pyPKցFHg 6VVdcoscli/node/main.pyimport os import subprocess import dcoscli import docopt import pkg_resources from dcos import cmds, emitting, errors, mesos, util from dcos.errors import DCOSException, DefaultError from dcoscli import log, tables from dcoscli.main import decorate_docopt_usage logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() def main(): try: return _main() except DCOSException as e: emitter.publish(e) return 1 @decorate_docopt_usage def _main(): util.configure_process_from_environ() args = docopt.docopt( _doc(), version="dcos-node version {}".format(dcoscli.version)) return cmds.execute(_cmds(), args) def _doc(): """ :rtype: str """ return pkg_resources.resource_string( 'dcoscli', 'data/help/node.txt').decode('utf-8') def _cmds(): """ :returns: All of the supported commands :rtype: [Command] """ return [ cmds.Command( hierarchy=['node', '--info'], arg_keys=[], function=_info), cmds.Command( hierarchy=['node', 'log'], arg_keys=['--follow', '--lines', '--master', '--slave'], function=_log), cmds.Command( hierarchy=['node', 'ssh'], arg_keys=['--master', '--slave', '--option', '--config-file', '--user', '--master-proxy'], function=_ssh), cmds.Command( hierarchy=['node'], arg_keys=['--json'], function=_list), ] def _info(): """Print node cli information. :returns: process return code :rtype: int """ emitter.publish(_doc().split('\n')[0]) return 0 def _list(json_): """List DCOS nodes :param json_: If true, output json. Otherwise, output a human readable table. :type json_: bool :returns: process return code :rtype: int """ client = mesos.DCOSClient() slaves = client.get_state_summary()['slaves'] if json_: emitter.publish(slaves) else: table = tables.slave_table(slaves) output = str(table) if output: emitter.publish(output) else: emitter.publish(errors.DefaultError('No slaves found.')) def _log(follow, lines, master, slave): """ Prints the contents of master and slave logs. :param follow: same as unix tail's -f :type follow: bool :param lines: number of lines to print :type lines: int :param master: whether to print the master log :type master: bool :param slave: the slave ID to print :type slave: str | None :returns: process return code :rtype: int """ if not (master or slave): raise DCOSException('You must choose one of --master or --slave.') lines = util.parse_int(lines) mesos_files = _mesos_files(master, slave) log.log_files(mesos_files, follow, lines) return 0 def _mesos_files(master, slave_id): """Returns the MesosFile objects to log :param master: whether to include the master log file :type master: bool :param slave_id: the ID of a slave. used to include a slave's log file :type slave_id: str | None :returns: MesosFile objects :rtype: [MesosFile] """ files = [] if master: files.append(mesos.MesosFile('/master/log')) if slave_id: slave = mesos.get_master().slave(slave_id) files.append(mesos.MesosFile('/slave/log', slave=slave)) return files def _ssh(master, slave, option, config_file, user, master_proxy): """SSH into a DCOS node using the IP addresses found in master's state.json :param master: True if the user has opted to SSH into the leading master :type master: bool | None :param slave: The slave ID if the user has opted to SSH into a slave :type slave: str | None :param option: SSH option :type option: [str] :param config_file: SSH config file :type config_file: str | None :param user: SSH user :type user: str | None :param master_proxy: If True, SSH-hop from a master :type master_proxy: bool | None :rtype: int :returns: process return code """ ssh_options = util.get_ssh_options(config_file, option) dcos_client = mesos.DCOSClient() if master: host = mesos.MesosDNSClient().hosts('leader.mesos.')[0]['ip'] else: summary = dcos_client.get_state_summary() slave_obj = next((slave_ for slave_ in summary['slaves'] if slave_['id'] == slave), None) if slave_obj: host = mesos.parse_pid(slave_obj['pid'])[1] else: raise DCOSException('No slave found with ID [{}]'.format(slave)) master_public_ip = dcos_client.metadata().get('PUBLIC_IPV4') if master_proxy: if not os.environ.get('SSH_AUTH_SOCK'): raise DCOSException( "There is no SSH_AUTH_SOCK env variable, which likely means " "you aren't running `ssh-agent`. `dcos node ssh " "--master-proxy` depends on `ssh-agent` to safely use your " "private key to hop between nodes in your cluster. Please " "run `ssh-agent`, then add your private key with `ssh-add`.") if not master_public_ip: raise DCOSException(("Cannot use --master-proxy. Failed to find " "'PUBLIC_IPV4' at {}").format( dcos_client.get_dcos_url('metadata'))) cmd = "ssh -A -t {0}{1}@{2} ssh -A -t {1}@{3}".format( ssh_options, user, master_public_ip, host) else: cmd = "ssh -t {0}{1}@{2}".format( ssh_options, user, host) emitter.publish(DefaultError("Running `{}`".format(cmd))) if (not master_proxy) and master_public_ip: emitter.publish( DefaultError("If you are running this command from a separate " "network than DCOS, consider using `--master-proxy`")) return subprocess.call(cmd, shell=True) PKցFHdcoscli/task/__init__.pyPKցFHʓԗdcoscli/task/main.pyimport posixpath import dcoscli import docopt import pkg_resources from dcos import cmds, emitting, mesos, util from dcos.errors import DCOSException, DCOSHTTPException, DefaultError from dcoscli import log, tables from dcoscli.main import decorate_docopt_usage logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() def main(): try: return _main() except DCOSException as e: emitter.publish(e) return 1 @decorate_docopt_usage def _main(): util.configure_process_from_environ() args = docopt.docopt( _doc(), version="dcos-task version {}".format(dcoscli.version)) return cmds.execute(_cmds(), args) def _doc(): """ :rtype: str """ return pkg_resources.resource_string( 'dcoscli', 'data/help/task.txt').decode('utf-8') def _cmds(): """ :returns: All of the supported commands :rtype: [Command] """ return [ cmds.Command( hierarchy=['task', '--info'], arg_keys=[], function=_info), cmds.Command( hierarchy=['task', 'log'], arg_keys=['--follow', '--completed', '--lines', '', ''], function=_log), cmds.Command( hierarchy=['task', 'ls'], arg_keys=['', '', '--long'], function=_ls), cmds.Command( hierarchy=['task'], arg_keys=['', '--completed', '--json'], function=_task), ] def _info(): """Print task cli information. :returns: process return code :rtype: int """ emitter.publish(_doc().split('\n')[0]) return 0 def _task(fltr, completed, json_): """List DCOS tasks :param fltr: task id filter :type fltr: str :param completed: If True, include completed tasks :type completed: bool :param json_: If True, output json. Otherwise, output a human readable table. :type json_: bool :returns: process return code """ if fltr is None: fltr = "" tasks = sorted(mesos.get_master().tasks(completed=completed, fltr=fltr), key=lambda task: task['name']) if json_: emitter.publish([task.dict() for task in tasks]) else: table = tables.task_table(tasks) output = str(table) if output: emitter.publish(output) return 0 def _log(follow, completed, lines, task, file_): """ Tail a file in the task's sandbox. :param follow: same as unix tail's -f :type follow: bool :param completed: whether to include completed tasks :type completed: bool :param lines: number of lines to print :type lines: int :param task: task pattern to match :type task: str :param file_: file path to read :type file_: str :returns: process return code :rtype: int """ if task is None: fltr = "" else: fltr = task if file_ is None: file_ = 'stdout' lines = util.parse_int(lines) # get tasks client = mesos.DCOSClient() master = mesos.Master(client.get_master_state()) tasks = master.tasks(completed=completed, fltr=fltr) if not tasks: if not completed: completed_tasks = master.tasks(completed=True, fltr=fltr) if completed_tasks: msg = 'No running tasks match ID [{}]; however, there '.format( fltr) if len(completed_tasks) > 1: msg += 'are {} matching completed tasks. '.format( len(completed_tasks)) else: msg += 'is 1 matching completed task. ' msg += 'Run with --completed to see these logs.' raise DCOSException(msg) raise DCOSException('No matching tasks. Exiting.') mesos_files = _mesos_files(tasks, file_, client) if not mesos_files: raise DCOSException('No matching tasks. Exiting.') log.log_files(mesos_files, follow, lines) return 0 def _ls(task, path, long_): """ List files in a task's sandbox. :param task: task pattern to match :type task: str :param path: file path to read :type path: str :param long_: whether to use a long listing format :type long_: bool :returns: process return code :rtype: int """ if path is None: path = '.' if path.startswith('/'): path = path[1:] dcos_client = mesos.DCOSClient() task_obj = mesos.get_master(dcos_client).task(task) dir_ = posixpath.join(task_obj.directory(), path) try: files = dcos_client.browse(task_obj.slave(), dir_) except DCOSHTTPException as e: if e.response.status_code == 404: raise DCOSException( 'Cannot access [{}]: No such file or directory'.format(path)) else: raise if files: if long_: emitter.publish(tables.ls_long_table(files)) else: emitter.publish( ' '.join(posixpath.basename(file_['path']) for file_ in files)) def _mesos_files(tasks, file_, client): """Return MesosFile objects for the specified tasks and file name. Only include files that satisfy all of the following: a) belong to an available slave b) have an executor entry on the slave :param tasks: tasks on which files reside :type tasks: [mesos.Task] :param file_: file path to read :type file_: str :param client: DCOS client :type client: mesos.DCOSClient :returns: MesosFile objects :rtype: [mesos.MesosFile] """ # load slave state in parallel slaves = _load_slaves_state([task.slave() for task in tasks]) # some completed tasks may have entries on the master, but none on # the slave. since we need the slave entry to get the executor # sandbox, we only include files with an executor entry. available_tasks = [task for task in tasks if task.slave() in slaves and task.executor()] # create files. return [mesos.MesosFile(file_, task=task, dcos_client=client) for task in available_tasks] def _load_slaves_state(slaves): """Fetch each slave's state.json in parallel, and return the reachable slaves. :param slaves: slaves to fetch :type slaves: [MesosSlave] :returns: MesosSlave objects that were successfully reached :rtype: [MesosSlave] """ reachable_slaves = [] for job, slave in util.stream(lambda slave: slave.state(), slaves): try: job.result() reachable_slaves.append(slave) except DCOSException as e: emitter.publish( DefaultError('Error accessing slave: {0}'.format(e))) return reachable_slaves PKցFHdcoscli/package/__init__.pyPKցFHz\Z\Zdcoscli/package/main.pyimport hashlib import json import os import sys import tempfile import zipfile from collections import defaultdict import dcoscli import docopt import pkg_resources from dcos import (cmds, emitting, errors, http, marathon, options, package, subcommand, util) from dcos.errors import DCOSException from dcoscli import tables from dcoscli.main import decorate_docopt_usage from six import iteritems logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() def main(): try: return _main() except DCOSException as e: emitter.publish(e) return 1 def _doc(): return pkg_resources.resource_string( 'dcoscli', 'data/help/package.txt').decode('utf-8') @decorate_docopt_usage def _main(): util.configure_process_from_environ() args = docopt.docopt( _doc(), version='dcos-package version {}'.format(dcoscli.version)) http.silence_requests_warnings() return cmds.execute(_cmds(), args) def _cmds(): """ :returns: All of the supported commands :rtype: dcos.cmds.Command """ return [ cmds.Command( hierarchy=['package', 'sources'], arg_keys=[], function=_list_sources), cmds.Command( hierarchy=['package', 'update'], arg_keys=['--validate'], function=_update), cmds.Command( hierarchy=['package', 'describe'], arg_keys=['', '--app', '--cli', '--options', '--render', '--package-versions', '--package-version', '--config'], function=_describe), cmds.Command( hierarchy=['package', 'install'], arg_keys=['', '--package-version', '--options', '--app-id', '--cli', '--app', '--yes'], function=_install), cmds.Command( hierarchy=['package', 'list'], arg_keys=['--json', '--endpoints', '--app-id', ''], function=_list), cmds.Command( hierarchy=['package', 'search'], arg_keys=['--json', ''], function=_search), cmds.Command( hierarchy=['package', 'uninstall'], arg_keys=['', '--all', '--app-id', '--cli', '--app'], function=_uninstall), cmds.Command( hierarchy=['package'], arg_keys=['--config-schema', '--info'], function=_package), ] def _package(config_schema, info): """ :param config_schema: Whether to output the config schema :type config_schema: boolean :param info: Whether to output a description of this subcommand :type info: boolean :returns: Process status :rtype: int """ if config_schema: schema = json.loads( pkg_resources.resource_string( 'dcoscli', 'data/config-schema/package.json').decode('utf-8')) emitter.publish(schema) elif info: _info() else: emitter.publish(options.make_generic_usage_message(_doc())) return 1 return 0 def _info(): """Print package cli information. :returns: Process status :rtype: int """ emitter.publish(_doc().split('\n')[0]) return 0 def _list_sources(): """List configured package sources. :returns: Process status :rtype: int """ config = util.get_config() sources = package.list_sources(config) for source in sources: emitter.publish("{} {}".format(source.hash(), source.url)) return 0 def _update(validate): """Update local package definitions from sources. :param validate: Whether to validate package content when updating sources. :type validate: bool :returns: Process status :rtype: int """ config = util.get_config() package.update_sources(config, validate) return 0 def _describe(package_name, app, cli, options_path, render, package_versions, package_version, config): """Describe the specified package. :param package_name: The package to describe :type package_name: str :param app: If True, marathon.json will be printed :type app: boolean :param cli: If True, command.json should be printed :type cli: boolean :param options_path: Path to json file with options to override config.json defaults. :type options_path: str :param render: If True, marathon.json and/or command.json templates will be rendered :type render: boolean :param package_versions: If True, a list of all package versions will be printed :type package_versions: boolean :param package_version: package version :type package_version: str | None :param config: If True, config.json will be printed :type config: boolean :returns: Process status :rtype: int """ # If the user supplied template options, they definitely want to # render the template if options_path: render = True if package_versions and \ (app or cli or options_path or render or package_version or config): raise DCOSException( 'If --package-versions is provided, no other option can be ' 'provided') pkg = package.resolve_package(package_name) if pkg is None: raise DCOSException("Package [{}] not found".format(package_name)) pkg_revision = pkg.latest_package_revision(package_version) if pkg_revision is None: raise DCOSException("Version {} of package [{}] is not available". format(package_version, package_name)) pkg_json = pkg.package_json(pkg_revision) if package_version is None: revision_map = pkg.package_revisions_map() pkg_versions = list(revision_map.values()) del pkg_json['version'] pkg_json['versions'] = pkg_versions if package_versions: emitter.publish('\n'.join(pkg_json['versions'])) elif cli or app or config: user_options = _user_options(options_path) options = pkg.options(pkg_revision, user_options) if cli: if render: cli_output = pkg.command_json(pkg_revision, options) else: cli_output = pkg.command_template(pkg_revision) if cli_output and cli_output[-1] == '\n': cli_output = cli_output[:-1] emitter.publish(cli_output) if app: if render: app_output = pkg.marathon_json(pkg_revision, options) else: app_output = pkg.marathon_template(pkg_revision) if app_output and app_output[-1] == '\n': app_output = app_output[:-1] emitter.publish(app_output) if config: config_output = pkg.config_json(pkg_revision) emitter.publish(config_output) else: pkg_json = pkg.package_json(pkg_revision) emitter.publish(pkg_json) return 0 def _user_options(path): """ Read the options at the given file path. :param path: file path :type path: str :returns: options :rtype: dict """ if path is None: return {} else: with util.open_file(path) as options_file: return util.load_json(options_file) def _confirm(prompt, yes): """ :param prompt: message to display to the terminal :type prompt: str :param yes: whether to assume that the user responded with yes :type yes: bool :returns: True if the user responded with yes; False otherwise :rtype: bool """ if yes: return True else: while True: sys.stdout.write('{} [yes/no] '.format(prompt)) sys.stdout.flush() response = sys.stdin.readline().strip().lower() if response == 'yes' or response == 'y': return True elif response == 'no' or response == 'n': return False else: emitter.publish( "'{}' is not a valid response.".format(response)) def _install(package_name, package_version, options_path, app_id, cli, app, yes): """Install the specified package. :param package_name: the package to install :type package_name: str :param package_version: package version to install :type package_version: str :param options_path: path to file containing option values :type options_path: str :param app_id: app ID for installation of this package :type app_id: str :param cli: indicates if the cli should be installed :type cli: bool :param app: indicate if the application should be installed :type app: bool :param yes: automatically assume yes to all prompts :type yes: bool :returns: process status :rtype: int """ if cli is False and app is False: # Install both if neither flag is specified cli = app = True config = util.get_config() pkg = package.resolve_package(package_name, config) if pkg is None: msg = "Package [{}] not found\n".format(package_name) + \ "You may need to run 'dcos package update' to update your " + \ "repositories" raise DCOSException(msg) pkg_revision = pkg.latest_package_revision(package_version) if pkg_revision is None: if package_version is not None: msg = "Version {} of package [{}] is not available".format( package_version, package_name) else: msg = "Package [{}] not available".format(package_name) raise DCOSException(msg) user_options = _user_options(options_path) pkg_json = pkg.package_json(pkg_revision) pre_install_notes = pkg_json.get('preInstallNotes') if pre_install_notes: emitter.publish(pre_install_notes) if not _confirm('Continue installing?', yes): emitter.publish('Exiting installation.') return 0 options = pkg.options(pkg_revision, user_options) revision_map = pkg.package_revisions_map() package_version = revision_map.get(pkg_revision) if app and (pkg.has_marathon_definition(pkg_revision) or pkg.has_marathon_mustache_definition(pkg_revision)): # Install in Marathon msg = 'Installing Marathon app for package [{}] version [{}]'.format( pkg.name(), package_version) if app_id is not None: msg += ' with app id [{}]'.format(app_id) emitter.publish(msg) init_client = marathon.create_client(config) package.install_app( pkg, pkg_revision, init_client, options, app_id) if cli and pkg.has_command_definition(pkg_revision): # Install subcommand msg = 'Installing CLI subcommand for package [{}] version [{}]'.format( pkg.name(), package_version) emitter.publish(msg) subcommand.install(pkg, pkg_revision, options) subcommand_paths = subcommand.get_package_commands(package_name) new_commands = [os.path.basename(p).replace('-', ' ', 1) for p in subcommand_paths] if new_commands: commands = ', '.join(new_commands) plural = "s" if len(new_commands) > 1 else "" emitter.publish("New command{} available: {}".format(plural, commands)) post_install_notes = pkg_json.get('postInstallNotes') if post_install_notes: emitter.publish(post_install_notes) return 0 def _list(json_, endpoints, app_id, package_name): """List installed apps :param json_: output json if True :type json_: bool :param endpoints: Whether to include a list of endpoints as port-host pairs :type endpoints: boolean :param app_id: App ID of app to show :type app_id: str :param package_name: The package to show :type package_name: str :returns: process return code :rtype: int """ config = util.get_config() init_client = marathon.create_client(config) installed = package.installed_packages(init_client, endpoints) # only emit those packages that match the provided package_name and app_id results = [] for pkg in installed: pkg_info = pkg.dict() if (_matches_package_name(package_name, pkg_info) and _matches_app_id(app_id, pkg_info)): if app_id: # if the user is asking a specific id then only show that id pkg_info['apps'] = [ app for app in pkg_info['apps'] if app == app_id ] results.append(pkg_info) if results or json_: emitting.publish_table(emitter, results, tables.package_table, json_) else: msg = ("There are currently no installed packages. " "Please use `dcos package install` to install a package.") raise DCOSException(msg) return 0 def _matches_package_name(name, pkg_info): """ :param name: the name of the package :type name: str :param pkg_info: the package description :type pkg_info: dict :returns: True if the name is not defined or the package matches that name; False otherwise :rtype: bool """ return name is None or pkg_info['name'] == name def _matches_app_id(app_id, pkg_info): """ :param app_id: the application id :type app_id: str :param pkg_info: the package description :type pkg_info: dict :returns: True if the app id is not defined or the package matches that app id; False otherwize :rtype: bool """ return app_id is None or app_id in pkg_info.get('apps') def _search(json_, query): """Search for matching packages. :param json_: output json if True :type json_: bool :param query: The search term :type query: str :returns: Process status :rtype: int """ if not query: query = '' config = util.get_config() results = [index_entry.as_dict() for index_entry in package.search(query, config)] if any(result['packages'] for result in results) or json_: emitting.publish_table(emitter, results, tables.package_search_table, json_) else: raise DCOSException('No packages found.') return 0 def _uninstall(package_name, remove_all, app_id, cli, app): """Uninstall the specified package. :param package_name: The package to uninstall :type package_name: str :param remove_all: Whether to remove all instances of the named package :type remove_all: boolean :param app_id: App ID of the package instance to uninstall :type app_id: str :returns: Process status :rtype: int """ err = package.uninstall(package_name, remove_all, app_id, cli, app) if err is not None: emitter.publish(err) return 1 return 0 def _bundle(package_directory, output_directory): """ :param package_directory: directory containing the package :type package_directory: str :param output_directory: directory where to save the package zip file :type output_directory: str :returns: process status :rtype: int """ if output_directory is None: output_directory = os.getcwd() logger.debug('Using [%s] as the ouput directory', output_directory) # Find package.json file and parse it if not os.path.exists(os.path.join(package_directory, 'package.json')): raise DCOSException( ('The file package.json is required in the package directory ' '[{}]').format(package_directory)) package_json = _validate_json_file( os.path.join(package_directory, 'package.json')) with tempfile.NamedTemporaryFile() as temp_file: with zipfile.ZipFile( temp_file.name, mode='w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zip_file: # list through package directory and add files zip archive for filename in sorted(os.listdir(package_directory)): fullpath = os.path.join(package_directory, filename) if filename == 'marathon.json.mustache': zip_file.write(fullpath, arcname=filename) elif filename in ['config.json', 'command.json', 'package.json']: # schema check the config and command json file _validate_json_file(fullpath) zip_file.write(fullpath, arcname=filename) elif filename == 'assets' and os.path.isdir(fullpath): _bundle_assets(fullpath, zip_file) elif filename == 'images' and os.path.isdir(fullpath): _bundle_images(fullpath, zip_file) else: # anything else is an error raise DCOSException( ('Error bundling package. Extra file in package ' 'directory [{}]').format(fullpath)) # Compute the name of the package file zip_file_name = os.path.join( output_directory, '{}-{}-{}.zip'.format( package_json['name'], package_json['version'], _hashfile(temp_file.name))) if os.path.exists(zip_file_name): raise DCOSException( 'Output file [{}] already exists'.format( zip_file_name)) # rename with digest util.sh_copy(temp_file.name, zip_file_name) # Print the full path to the file emitter.publish( errors.DefaultError( 'Created DCOS Universe package [{}].'.format(zip_file_name))) return 0 def _validate_json_file(fullpath): """Validates the content of the file against its schema. Throws an exception if the file is not valid. :param fullpath: full path to the file. :type fullpath: str :return: json object if it is a special file :rtype: dict """ filename = os.path.basename(fullpath) if filename in ['command.json', 'config.json', 'package.json']: schema_path = 'data/universe-schema/{}'.format(filename) else: raise DCOSException( ('Error bundling package. Unknown file in package ' 'directory [{}]').format(fullpath)) special_schema = util.load_jsons( pkg_resources.resource_string('dcoscli', schema_path).decode('utf-8')) with util.open_file(fullpath) as special_file: special_json = util.load_json(special_file) errs = util.validate_json(special_json, special_schema) if errs: emitter.publish( errors.DefaultError( 'Error validating JSON file [{}]'.format(fullpath))) raise DCOSException(util.list_to_err(errs)) return special_json def _hashfile(filename): """Calculates the sha256 of a file :param filename: path to the file to sum :type filename: str :returns: digest in hexadecimal :rtype: str """ hasher = hashlib.sha256() with open(filename, 'rb') as f: for chunk in iter(lambda: f.read(4096), b''): hasher.update(chunk) return hasher.hexdigest() def _bundle_assets(assets_directory, zip_file): """Bundle the assets directory :param assets_directory: path to the assets directory :type assets_directory: str :param zip_file: zip file object :type zip_file: zipfile.ZipFile :rtype: None """ for filename in sorted(os.listdir(assets_directory)): fullpath = os.path.join(assets_directory, filename) if filename == 'uris' and os.path.isdir(fullpath): _bundle_uris(fullpath, zip_file) else: # anything else is an error raise DCOSException( ('Error bundling package. Extra file in package ' 'directory [{}]').format(fullpath)) def _bundle_uris(uris_directory, zip_file): """Bundle the uris directory :param uris_directory: path to the uris directory :type uris_directory: str :param zip_file: zip file object :type zip_file: zipfile.ZipFile :rtype: None """ uris = sorted(os.listdir(uris_directory)) # these uris will be found through a mustache template from the property # name so make sure each name is unique. uri_properties = defaultdict(list) for name in uris: uri_properties[name.replace('.', '-')].append(name) collisions = [uri_list for (prop, uri_list) in iteritems(uri_properties) if len(uri_list) > 1] if collisions: raise DCOSException( 'Error bundling package. Multiple assets map to the same property ' 'name (periods [.] are replaced with dashes [-]): {}'.format( collisions)) for filename in uris: fullpath = os.path.join(uris_directory, filename) zip_file.write(fullpath, arcname='assets/uris/{}'.format(filename)) def _bundle_images(images_directory, zip_file): """Bundle the images directory :param images_directory: path to the images directory :type images_directory: str :param zip_file: zip file object :type zip_file: zipfile.ZipFile :rtype: None """ for filename in sorted(os.listdir(images_directory)): fullpath = os.path.join(images_directory, filename) if (filename == 'icon-small.png' or filename == 'icon-medium.png' or filename == 'icon-large.png'): util.validate_png(fullpath) zip_file.write(fullpath, arcname='images/{}'.format(filename)) elif filename == 'screenshots' and os.path.isdir(fullpath): _bundle_screenshots(fullpath, zip_file) else: # anything else is an error raise DCOSException( ('Error bundling package. Extra file in package ' 'directory [{}]').format(fullpath)) def _bundle_screenshots(screenshot_directory, zip_file): """Bundle the screenshots directory :param screenshot_directory: path to the screenshots directory :type screenshot_directory: str :param zip_file: zip file object :type zip_file: zipfile.ZipFile :rtype: None """ for filename in sorted(os.listdir(screenshot_directory)): fullpath = os.path.join(screenshot_directory, filename) util.validate_png(fullpath) zip_file.write( fullpath, arcname='images/screenshots/{}'.format(filename)) PKցFHdcoscli/service/__init__.pyPKցFHP#沋dcoscli/service/main.pyimport subprocess import dcoscli import docopt import pkg_resources from dcos import cmds, emitting, marathon, mesos, package, util from dcos.errors import DCOSException, DefaultError from dcoscli import log, tables from dcoscli.main import decorate_docopt_usage logger = util.get_logger(__name__) emitter = emitting.FlatEmitter() def main(): try: return _main() except DCOSException as e: emitter.publish(e) return 1 @decorate_docopt_usage def _main(): util.configure_process_from_environ() args = docopt.docopt( _doc(), version="dcos-service version {}".format(dcoscli.version)) return cmds.execute(_cmds(), args) def _doc(): """ :rtype: str """ return pkg_resources.resource_string( 'dcoscli', 'data/help/service.txt').decode('utf-8') def _cmds(): """ :returns: All of the supported commands :rtype: [Command] """ return [ cmds.Command( hierarchy=['service', 'log'], arg_keys=['--follow', '--lines', '--ssh-config-file', '', ''], function=_log), cmds.Command( hierarchy=['service', 'shutdown'], arg_keys=[''], function=_shutdown), cmds.Command( hierarchy=['service', '--info'], arg_keys=[], function=_info), cmds.Command( hierarchy=['service'], arg_keys=['--inactive', '--completed', '--json'], function=_service), ] def _info(): """Print services cli information. :returns: process return code :rtype: int """ emitter.publish(_doc().split('\n')[0]) return 0 def _service(inactive, completed, is_json): """List dcos services :param inactive: If True, include completed tasks :type inactive: bool :param is_json: If true, output json. Otherwise, output a human readable table. :type is_json: bool :returns: process return code :rtype: int """ services = mesos.get_master().frameworks( inactive=inactive, completed=completed) if is_json: emitter.publish([service.dict() for service in services]) else: table = tables.service_table(services) output = str(table) if output: emitter.publish(output) return 0 def _shutdown(service_id): """Shuts down a service :param service_id: the id for the service :type service_id: str :returns: process return code :rtype: int """ mesos.DCOSClient().shutdown_framework(service_id) return 0 def _log(follow, lines, ssh_config_file, service, file_): """Prints the contents of the logs for a given service. The service task is located by first identifying the marathon app running a framework named `service`. :param follow: same as unix tail's -f :type follow: bool :param lines: number of lines to print :type lines: int :param ssh_config_file: SSH config file. Used for marathon. :type ssh_config_file: str | None :param service: service name :type service: str :param file_: file path to read :type file_: str :returns: process return code :rtype: int """ lines = util.parse_int(lines) if service == 'marathon': if file_: raise DCOSException('The argument is invalid for marathon.' ' The systemd journal is always used for the' ' marathon log.') return _log_marathon(follow, lines, ssh_config_file) else: if ssh_config_file: raise DCOSException( 'The `--ssh-config-file` argument is invalid for non-marathon ' 'services. SSH is not used.') return _log_service(follow, lines, service, file_) def _log_service(follow, lines, service, file_): """Prints the contents of the logs for a given service. Used for non-marathon services. :param follow: same as unix tail's -f :type follow: bool :param lines: number of lines to print :type lines: int :param service: service name :type service: str :param file_: file path to read :type file_: str :returns: process return code :rtype: int """ if file_ is None: file_ = 'stdout' task = _get_service_task(service) return _log_task(task['id'], follow, lines, file_) def _log_task(task_id, follow, lines, file_): """Prints the contents of the logs for a given task ID. :param task_id: task ID :type task_id: str :param follow: same as unix tail's -f :type follow: bool :param lines: number of lines to print :type lines: int :param file_: file path to read :type file_: str :returns: process return code :rtype: int """ dcos_client = mesos.DCOSClient() task = mesos.get_master(dcos_client).task(task_id) mesos_file = mesos.MesosFile(file_, task=task, dcos_client=dcos_client) return log.log_files([mesos_file], follow, lines) def _get_service_task(service_name): """Gets the task running the given service. If there is more than one such task, throws an exception. :param service_name: service name :type service_name: str :returns: The marathon task dict :rtype: dict """ marathon_client = marathon.create_client() app = _get_service_app(marathon_client, service_name) tasks = marathon_client.get_app(app['id'])['tasks'] if len(tasks) != 1: raise DCOSException( ('We expected marathon app [{}] to be running 1 task, but we ' + 'instead found {} tasks').format(app['id'], len(tasks))) return tasks[0] def _get_service_app(marathon_client, service_name): """Gets the marathon app running the given service. If there is not exactly one such app, throws an exception. :param marathon_client: marathon client :type marathon_client: marathon.Client :param service_name: service name :type service_name: str :returns: marathon app :rtype: dict """ apps = package.get_apps_for_framework(service_name, marathon_client) if len(apps) > 1: raise DCOSException( 'Multiple marathon apps found for service name [{}]: {}'.format( service_name, ', '.join('[{}]'.format(app['id']) for app in apps))) elif len(apps) == 0: raise DCOSException( 'No marathon apps found for service [{}]'.format(service_name)) else: return apps[0] def _log_marathon(follow, lines, ssh_config_file): """Prints the contents of the marathon logs. :param follow: same as unix tail's -f :type follow: bool :param lines: number of lines to print :type lines: int :param ssh_config_file: SSH config file. :type ssh_config_file: str | None ;:returns: process return code :rtype: int """ ssh_options = util.get_ssh_options(ssh_config_file, []) journalctl_args = '' if follow: journalctl_args += '-f ' if lines: journalctl_args += '-n {} '.format(lines) leader_ip = marathon.create_client().get_leader().split(':')[0] cmd = ("ssh {0}core@{1} " + "journalctl {2}-u dcos-marathon").format( ssh_options, leader_ip, journalctl_args) emitter.publish(DefaultError("Running `{}`".format(cmd))) return subprocess.call(cmd, shell=True) PKցFHdcoscli/config/__init__.pyPKցFH8a,,dcoscli/config/main.pyimport collections import copy import json import dcoscli import docopt import pkg_resources import six from dcos import cmds, config, emitting, http, jsonitem, subcommand, util from dcos.errors import DCOSException from dcoscli import analytics from dcoscli.main import decorate_docopt_usage emitter = emitting.FlatEmitter() logger = util.get_logger(__name__) def main(): try: return _main() except DCOSException as e: emitter.publish(e) return 1 @decorate_docopt_usage def _main(): util.configure_process_from_environ() args = docopt.docopt( _doc(), version='dcos-config version {}'.format(dcoscli.version)) http.silence_requests_warnings() return cmds.execute(_cmds(), args) def _doc(): """ :rtype: str """ return pkg_resources.resource_string( 'dcoscli', 'data/help/config.txt').decode('utf-8') def _check_config(toml_config_pre, toml_config_post): """ :param toml_config_pre: dictionary for the value before change :type toml_config_pre: dcos.api.config.Toml :param toml_config_post: dictionary for the value with change :type toml_config_post: dcos.api.config.Toml :returns: process status :rtype: int """ errors_pre = util.validate_json(toml_config_pre._dictionary, _generate_root_schema(toml_config_pre)) errors_post = util.validate_json(toml_config_post._dictionary, _generate_root_schema(toml_config_post)) logger.info('Comparing changes in the configuration...') logger.info('Errors before the config command: %r', errors_pre) logger.info('Errors after the config command: %r', errors_post) if len(errors_post) != 0: if len(errors_pre) == 0: raise DCOSException(util.list_to_err(errors_post)) def _errs(errs): return set([e.split('\n')[0] for e in errs]) diff_errors = _errs(errors_post) - _errs(errors_pre) if len(diff_errors) != 0: raise DCOSException(util.list_to_err(errors_post)) def _cmds(): """ :returns: all the supported commands :rtype: list of dcos.cmds.Command """ return [ cmds.Command( hierarchy=['config', 'set'], arg_keys=['', ''], function=_set), cmds.Command( hierarchy=['config', 'append'], arg_keys=['', ''], function=_append), cmds.Command( hierarchy=['config', 'prepend'], arg_keys=['', ''], function=_prepend), cmds.Command( hierarchy=['config', 'unset'], arg_keys=['', '--index'], function=_unset), cmds.Command( hierarchy=['config', 'show'], arg_keys=[''], function=_show), cmds.Command( hierarchy=['config', 'validate'], arg_keys=[], function=_validate), cmds.Command( hierarchy=['config'], arg_keys=['--info'], function=_info), ] def _info(info): """ :param info: Whether to output a description of this subcommand :type info: boolean :returns: process status :rtype: int """ emitter.publish(_doc().split('\n')[0]) return 0 def _set(name, value): """ :returns: process status :rtype: int """ toml_config = util.get_config(True) section, subkey = _split_key(name) config_schema = _get_config_schema(section) new_value = jsonitem.parse_json_value(subkey, value, config_schema) toml_config_pre = copy.deepcopy(toml_config) if section not in toml_config_pre._dictionary: toml_config_pre._dictionary[section] = {} value_exists = name in toml_config old_value = toml_config.get(name) toml_config[name] = new_value if (name == 'core.reporting' and new_value is True) or \ (name == 'core.email'): analytics.segment_identify(toml_config) _check_config(toml_config_pre, toml_config) config.save(toml_config) if not value_exists: emitter.publish("[{}]: set to '{}'".format(name, new_value)) elif old_value == new_value: emitter.publish("[{}]: already set to '{}'".format(name, old_value)) else: emitter.publish( "[{}]: changed from '{}' to '{}'".format( name, old_value, new_value)) return 0 def _append(name, value): """ :returns: process status :rtype: int """ toml_config = util.get_config(True) python_value = _parse_array_item(name, value) toml_config_pre = copy.deepcopy(toml_config) section = name.split(".", 1)[0] if section not in toml_config_pre._dictionary: toml_config_pre._dictionary[section] = {} toml_config[name] = toml_config.get(name, []) + python_value _check_config(toml_config_pre, toml_config) config.save(toml_config) return 0 def _prepend(name, value): """ :returns: process status :rtype: int """ toml_config = util.get_config(True) python_value = _parse_array_item(name, value) toml_config_pre = copy.deepcopy(toml_config) section = name.split(".", 1)[0] if section not in toml_config_pre._dictionary: toml_config_pre._dictionary[section] = {} toml_config[name] = python_value + toml_config.get(name, []) _check_config(toml_config_pre, toml_config) config.save(toml_config) return 0 def _unset(name, index): """ :returns: process status :rtype: int """ toml_config = util.get_config(True) toml_config_pre = copy.deepcopy(toml_config) section = name.split(".", 1)[0] if section not in toml_config_pre._dictionary: toml_config_pre._dictionary[section] = {} value = toml_config.pop(name, None) if value is None: raise DCOSException("Property {!r} doesn't exist".format(name)) elif isinstance(value, collections.Mapping): raise DCOSException(_generate_choice_msg(name, value)) elif ((isinstance(value, collections.Sequence) and not isinstance(value, six.string_types)) and index is not None): index = util.parse_int(index) if not value: raise DCOSException( 'Index ({}) is out of bounds - [{}] is empty'.format( index, name)) if index < 0 or index >= len(value): raise DCOSException( 'Index ({}) is out of bounds - possible values are ' 'between {} and {}'.format(index, 0, len(value) - 1)) popped_value = value.pop(index) emitter.publish( "[{}]: removed element '{}' at index '{}'".format( name, popped_value, index)) toml_config[name] = value config.save(toml_config) return 0 elif index is not None: raise DCOSException( 'Unsetting based on an index is only supported for lists') else: emitter.publish("Removed [{}]".format(name)) config.save(toml_config) return 0 def _show(name): """ :returns: process status :rtype: int """ toml_config = util.get_config(True) if name is not None: value = toml_config.get(name) if value is None: raise DCOSException("Property {!r} doesn't exist".format(name)) elif isinstance(value, collections.Mapping): raise DCOSException(_generate_choice_msg(name, value)) else: emitter.publish(value) else: # Let's list all of the values for key, value in sorted(toml_config.property_items()): emitter.publish('{}={}'.format(key, value)) return 0 def _validate(): """ :returns: process status :rtype: int """ toml_config = util.get_config(True) errs = util.validate_json(toml_config._dictionary, _generate_root_schema(toml_config)) if len(errs) != 0: emitter.publish(util.list_to_err(errs)) return 1 emitter.publish("Congratulations, your configuration is valid!") return 0 def _generate_root_schema(toml_config): """ :param toml_configs: dictionary of values :type toml_config: TomlConfig :returns: configuration_schema :rtype: jsonschema """ root_schema = { '$schema': 'http://json-schema.org/schema#', 'type': 'object', 'properties': {}, 'additionalProperties': False, } # Load the config schema from all the subsections into the root schema for section in toml_config.keys(): config_schema = _get_config_schema(section) root_schema['properties'][section] = config_schema return root_schema def _generate_choice_msg(name, value): """ :param name: name of the property :type name: str :param value: dictionary for the value :type value: dcos.config.Toml :returns: an error message for top level properties :rtype: str """ message = ("Property {!r} doesn't fully specify a value - " "possible properties are:").format(name) for key, _ in sorted(value.property_items()): message += '\n{}.{}'.format(name, key) return message def _get_config_schema(command): """ :param command: the subcommand name :type command: str :returns: the subcommand's configuration schema :rtype: dict """ # core.* config variables are special. They're valid, but don't # correspond to any particular subcommand, so we must handle them # separately. if command == "core": return json.loads( pkg_resources.resource_string( 'dcoscli', 'data/config-schema/core.json').decode('utf-8')) executable = subcommand.command_executables(command) return subcommand.config_schema(executable) def _split_key(name): """ :param name: the full property path - e.g. marathon.url :type name: str :returns: the section and property name :rtype: (str, str) """ terms = name.split('.', 1) if len(terms) != 2: raise DCOSException('Property name must have both a section and ' 'key:
. - E.g. marathon.url') return (terms[0], terms[1]) def _parse_array_item(name, value): """ :param name: the name of the property :type name: str :param value: the value to parse :type value: str :returns: the parsed value as an array with one element :rtype: (list of any, dcos.errors.Error) where any is string, int, float, bool, array or dict """ section, subkey = _split_key(name) config_schema = _get_config_schema(section) parser = jsonitem.find_parser(subkey, config_schema) if parser.schema['type'] != 'array': raise DCOSException( "Append/Prepend not supported on '{0}' properties - use 'dcos " "config set {0} {1}'".format(name, value)) if ('items' in parser.schema and parser.schema['items']['type'] == 'string'): value = '["' + value + '"]' else: # We are going to assume that wrapping it in an array is enough value = '[' + value + ']' return parser(value) PKցFHdcoscli/help/__init__.pyPKցFHh dcoscli/help/main.pyimport subprocess import dcoscli import docopt import pkg_resources from concurrent.futures import ThreadPoolExecutor from dcos import cmds, emitting, options, subcommand, util from dcos.errors import DCOSException from dcoscli.main import decorate_docopt_usage emitter = emitting.FlatEmitter() logger = util.get_logger(__name__) def main(): try: return _main() except DCOSException as e: emitter.publish(e) return 1 @decorate_docopt_usage def _main(): util.configure_process_from_environ() args = docopt.docopt( _doc(), version='dcos-help version {}'.format(dcoscli.version)) return cmds.execute(_cmds(), args) def _doc(): """ :rtype: str """ return pkg_resources.resource_string( 'dcoscli', 'data/help/help.txt').decode('utf-8') def _cmds(): """ :returns: All of the supported commands :rtype: list of dcos.cmds.Command """ return [ cmds.Command( hierarchy=['help', '--info'], arg_keys=[], function=_info), cmds.Command( hierarchy=['help'], arg_keys=[''], function=_help), ] def _info(): """ :returns: process return code :rtype: int """ emitter.publish(_doc().split('\n')[0]) return 0 def _help(command): """ :param command: the command name for which you want to see a help :type command: str :returns: process return code :rtype: int """ if command is not None: _help_command(command) else: logger.debug("DCOS bin path: {!r}".format(util.dcos_bin_path())) paths = subcommand.list_paths() with ThreadPoolExecutor(max_workers=len(paths)) as executor: results = executor.map(subcommand.documentation, paths) commands_message = options\ .make_command_summary_string(sorted(results)) emitter.publish( "Command line utility for the Mesosphere Datacenter Operating\n" "System (DCOS). The Mesosphere DCOS is a distributed operating\n" "system built around Apache Mesos. This utility provides tools\n" "for easy management of a DCOS installation.\n") emitter.publish("Available DCOS commands:") emitter.publish(commands_message) emitter.publish( "\nGet detailed command description with 'dcos --help'.") return 0 def _help_command(command): """ :param command: the command name for which you want to see a help :type command: str :returns: process return code :rtype: int """ executable = subcommand.command_executables(command) return subprocess.call([executable, command, '--help']) PKցFH h##$dcoscli-0.3.1.data/scripts/env-setupif [ -n "$BASH_SOURCE" ] ; then BIN_DIR=$(dirname "$BASH_SOURCE") elif [ $(basename -- "$0") = "env-setup" ]; then BIN_DIR=$(dirname "$0") else BIN_DIR=$PWD/bin fi # real, absolute path to BIN_DIR FULL_BIN_PATH=$(python -c "import os; print(os.path.realpath('$BIN_DIR'))") # ensure BIN_DIR is prepended to PATH expr "$PATH" : "${FULL_BIN_PATH}.*" > /dev/null || export PATH=$FULL_BIN_PATH:$PATH export DCOS_CONFIG=~/.dcos/dcos.toml if [ ! -f "$DCOS_CONFIG" ]; then mkdir -p $(dirname "$DCOS_CONFIG") touch "$DCOS_CONFIG" fi PKFHX'dcoscli-0.3.1.dist-info/DESCRIPTION.rstDCOS Command Line Interface =========================== The DCOS Command Line Interface (CLI) is a command line utility that provides a user-friendly yet powerful way to manage DCOS installations. This project is open source. Please see GitHub_ to access source code and to contribute. Full documentation is available for the DCOS CLI on the `Mesosphere docs website`_. .. _GitHub: https://github.com/mesosphere/dcos-cli .. _Mesosphere docs website: http://docs.mesosphere.com/using/cli/ PKFH'$999(dcoscli-0.3.1.dist-info/entry_points.txt[console_scripts] dcos = dcoscli.main:main dcos-config = dcoscli.config.main:main dcos-help = dcoscli.help.main:main dcos-marathon = dcoscli.marathon.main:main dcos-node = dcoscli.node.main:main dcos-package = dcoscli.package.main:main dcos-service = dcoscli.service.main:main dcos-task = dcoscli.task.main:main PKFHn;;%dcoscli-0.3.1.dist-info/metadata.json{"classifiers": ["Development Status :: 4 - Beta", "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Topic :: Software Development :: User Interfaces", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.4"], "extensions": {"python.commands": {"wrap_console": {"dcos": "dcoscli.main:main", "dcos-config": "dcoscli.config.main:main", "dcos-help": "dcoscli.help.main:main", "dcos-marathon": "dcoscli.marathon.main:main", "dcos-node": "dcoscli.node.main:main", "dcos-package": "dcoscli.package.main:main", "dcos-service": "dcoscli.service.main:main", "dcos-task": "dcoscli.task.main:main"}}, "python.details": {"contacts": [{"email": "team@mesosphere.io", "name": "Mesosphere, Inc.", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}, "project_urls": {"Home": "https://github.com/mesosphere/dcos-cli"}}, "python.exports": {"console_scripts": {"dcos": "dcoscli.main:main", "dcos-config": "dcoscli.config.main:main", "dcos-help": "dcoscli.help.main:main", "dcos-marathon": "dcoscli.marathon.main:main", "dcos-node": "dcoscli.node.main:main", "dcos-package": "dcoscli.package.main:main", "dcos-service": "dcoscli.service.main:main", "dcos-task": "dcoscli.task.main:main"}}}, "extras": [], "generator": "bdist_wheel (0.28.0)", "keywords": ["mesos", "apache", "marathon", "mesosphere", "command", "datacenter"], "metadata_version": "2.0", "name": "dcoscli", "run_requires": [{"requires": ["dcos (==0.3.1)", "docopt (>=0.6,<1.0)", "futures (>=3.0,<4.0)", "oauth2client (>=1.4,<2.0)", "pkginfo (>=1.2,<2.0)", "rollbar (>=0.9,<1.0)", "toml (>=0.9,<1.0)", "virtualenv (>=13.0,<14.0)"]}], "summary": "DCOS Command Line Interface", "version": "0.3.1"}PKFH̚r.%dcoscli-0.3.1.dist-info/top_level.txtdcoscli tests PKFH>nndcoscli-0.3.1.dist-info/WHEELWheel-Version: 1.0 Generator: bdist_wheel (0.28.0) Root-Is-Purelib: true Tag: py2-none-any Tag: py3-none-any PKFHS dcoscli-0.3.1.dist-info/METADATAMetadata-Version: 2.0 Name: dcoscli Version: 0.3.1 Summary: DCOS Command Line Interface Home-page: https://github.com/mesosphere/dcos-cli Author: Mesosphere, Inc. Author-email: team@mesosphere.io License: UNKNOWN Keywords: mesos apache marathon mesosphere command datacenter Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Intended Audience :: Developers Classifier: Intended Audience :: Information Technology Classifier: Topic :: Software Development :: User Interfaces Classifier: License :: OSI Approved :: Apache Software License Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.4 Requires-Dist: dcos (==0.3.1) Requires-Dist: docopt (>=0.6,<1.0) Requires-Dist: futures (>=3.0,<4.0) Requires-Dist: oauth2client (>=1.4,<2.0) Requires-Dist: pkginfo (>=1.2,<2.0) Requires-Dist: rollbar (>=0.9,<1.0) Requires-Dist: toml (>=0.9,<1.0) Requires-Dist: virtualenv (>=13.0,<14.0) DCOS Command Line Interface =========================== The DCOS Command Line Interface (CLI) is a command line utility that provides a user-friendly yet powerful way to manage DCOS installations. This project is open source. Please see GitHub_ to access source code and to contribute. Full documentation is available for the DCOS CLI on the `Mesosphere docs website`_. .. _GitHub: https://github.com/mesosphere/dcos-cli .. _Mesosphere docs website: http://docs.mesosphere.com/using/cli/ PKFHkGdcoscli-0.3.1.dist-info/RECORDdcoscli/__init__.py,sha256=oWC-kZOwfKFeiqeLK6vnEAO35OhC5ja2-xXugHPFmHo,131 dcoscli/analytics.py,sha256=kliwvT6YKi7sPXFN5nGSEaFWml9picC9OV17MzLS1hs,6906 dcoscli/common.py,sha256=2dLcwgx5Juu-VSxBI1JPDUA1fVkAxrWcETTlZBB4xAc,757 dcoscli/constants.py,sha256=sKXUkycwr2ogNhP_TAVGTa5kIIjW5kzAsxhUY8HXDi4,288 dcoscli/log.py,sha256=LTrWovDwPoHLuclS_Uw1U5CNmXmgD1_EDyaFMAmm0HI,5954 dcoscli/main.py,sha256=y1Dwp9tnBEcFelKEtnJtDycXK7KDS-zvMjyTJUmeDJo,3233 dcoscli/tables.py,sha256=z8odgz7JAY7hLhH8FGhBZqejRrkF1ppa7NUGTkGbmWI,11360 dcoscli/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 dcoscli/config/main.py,sha256=Qvriyo01kVI-yONWpdyICPieNrXBD2u0_4SeCpoqLHs,11434 dcoscli/data/config-schema/core.json,sha256=3Pdx2x1h3lV_zWSzcQgHwBf2LjlrLrVXgxWyVAfjDrs,1702 dcoscli/data/config-schema/marathon.json,sha256=zL2vXhOnzhUzAvQHOR-XJI9eU9E2RLQpeqp-YPtsQXY,378 dcoscli/data/config-schema/package.json,sha256=fDfR4eJoSl1vUmVYlRn26bQzkIoTDGF9yrHJmtqCjZ4,962 dcoscli/data/help/config.txt,sha256=FkR8YsxSay5b6IAzGU_f-LiqlhHjQQ4QbbW_cUAr3q8,668 dcoscli/data/help/dcos.txt,sha256=PaRtNsXJWEfaTUpZXvwgIgCJ6SbBKgx45P-p1OanVo8,1943 dcoscli/data/help/help.txt,sha256=kk6AEmf3n4TkdE6r7z9SWGX1zvd9xCgR-ntjz5IUH4c,235 dcoscli/data/help/marathon.txt,sha256=5qqHe0XCf46kVHt54hbDn5lsTRLBTV7at-y8w5Mf2EU,5162 dcoscli/data/help/node.txt,sha256=7DQoe2nC8Wqu9ymm0LQJ-OBk2hU-gcpHJnMpkBNliRY,1429 dcoscli/data/help/package.txt,sha256=KmpbiwPgVJEjmQsmh6KBLnLKnvBvp3h3veGwow0ziS0,2189 dcoscli/data/help/service.txt,sha256=tF8tKKvidEguBFkliot7s4ol-ZIwU9tz2nZtzs30SgY,1628 dcoscli/data/help/task.txt,sha256=1LvA8zoiHKOg1ytlXwuUTdj6Tq4JQ-cqC4s88O4wXH0,847 dcoscli/data/universe-schema/command.json,sha256=1jhnkr4y_vIr8CrZfyV67k9UrJIjeDFdVRa-NJubk-w,556 dcoscli/data/universe-schema/config.json,sha256=c1P_E_qpeQJ4E7lcizX5kgV4ImlqcMQ8ebO_tySedrQ,4375 dcoscli/data/universe-schema/package.json,sha256=RIHVCjuVB_gEvI7PxLDavduH2as0V5IBlY4pqhz05U8,2073 dcoscli/help/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 dcoscli/help/main.py,sha256=k3_LbV-tqg2e3rkW-8g0nvK1ld3rSygKgLhQ56TdwVY,2790 dcoscli/marathon/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 dcoscli/marathon/main.py,sha256=HRW9B59MUbeNp64AQIqFfYg8HMNFxv1ZtCmGm3gSnSo,22581 dcoscli/node/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 dcoscli/node/main.py,sha256=fAAZAn0nGpwUd_BkaIHwv5xXtamf-bUk5m5LxEhU82w,6230 dcoscli/package/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 dcoscli/package/main.py,sha256=Ox8yvwPGLex3zBwOl-BG46GcPDOZ5fGF4DHixBkpUJc,23132 dcoscli/service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 dcoscli/service/main.py,sha256=DFblQ5FDJ80fMTnAGcLIqT23oQWU16GutzRji7nJ2P4,7563 dcoscli/task/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 dcoscli/task/main.py,sha256=3ukZ3VKMSHrfQC8aFSJQjGhCVlZGTxrjzKS9rIDlKbM,6898 dcoscli-0.3.1.data/scripts/env-setup,sha256=wzA8UAARTmxyJ0MG3kRqm_IVIge3jc55Y86SIYdNmEw,547 dcoscli-0.3.1.dist-info/DESCRIPTION.rst,sha256=HUKLQlLswxgRXIXS5kUn8aan-ty5duxRFGPuvKQGOXY,493 dcoscli-0.3.1.dist-info/METADATA,sha256=vnSbEC5jw2saGdyV-Z-FyLbJ4yWHbe1wEkEgWiNDHcs,1538 dcoscli-0.3.1.dist-info/RECORD,, dcoscli-0.3.1.dist-info/WHEEL,sha256=c5du820PMLPXFYzXDp0SSjIjJ-7MmVRpJa1kKfTaqlc,110 dcoscli-0.3.1.dist-info/entry_points.txt,sha256=6WxeZ3gZx0G0ex_Hs8jINH0riRcbsE1wu9Hn_71Hkaw,313 dcoscli-0.3.1.dist-info/metadata.json,sha256=Fx34dvYVBf3FA0LwljLWiOH0eoR2oIceeeDPUPk8o-o,1851 dcoscli-0.3.1.dist-info/top_level.txt,sha256=QaJSU3GTh6ZeWfudTXSBbNByMISDtDu44SFo2m39FVc,14 tests/fixtures/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 tests/fixtures/marathon.py,sha256=iVzJUQ5Zu7MFfqC3hdMAWs1nvWN-gT-aRle34uHN0Ek,4248 tests/fixtures/node.py,sha256=EUbqcGJ5gpcWjp5AR1JT1_l2XcHWXgv64Gk1sN3Z4d8,1383 tests/fixtures/package.py,sha256=TwVazqEaIENL67qTNn3NavzxnVrqbPtric76RLnAFec,4485 tests/fixtures/service.py,sha256=KBfOSRszFmzgnd8ShmSvEzLcEyuXJrpYVAOSI5nGlFY,1275 tests/fixtures/task.py,sha256=O41U7PVgVRgxSv-WwW9rguMmaMZgglrIL7wVTE__GZQ,2211 tests/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 tests/integrations/common.py,sha256=UgDDULupeH7tH5C1W3YdLOp-d5_ABc5FsNNWC1scrqE,15208 tests/integrations/test_analytics.py,sha256=Gli3vDxIi1VsKV8AfaNrHBdWSKiB6Jl24Bg8qTTjte8,5336 tests/integrations/test_auth.py,sha256=A0pYo7ghX0-sADB1EL2jNjZ3qVVI-662BwYE9UBcKqM,1623 tests/integrations/test_config.py,sha256=dC0VQUVzqrlH9ssnDAEukvF6IuBInFLc_bh-tLrAa1g,16618 tests/integrations/test_dcos.py,sha256=2RL2PABgNeoGxv5Vr_921mqCkOFqs0fTh87GcAtQnOc,1995 tests/integrations/test_help.py,sha256=U6FCoS6e7l_hgG7xuztdqwlTxLn3yfmWx9dwdwMkiIs,2459 tests/integrations/test_http_auth.py,sha256=5lkMFPMIM6C_Tbatd5d2JV8iLYvyFdzsG1Xzx_ITHjI,4829 tests/integrations/test_marathon.py,sha256=xCe1FlI8AMQMXT67_4YLWYqzXtyqm-JNNOPjHws16to,23331 tests/integrations/test_marathon_groups.py,sha256=SK-8L4gTjhDQAuDUfBAop9Ft9S-8WjQnG1iG-rUTtSY,6176 tests/integrations/test_node.py,sha256=77xuOL1nFGR9T2bdSgtWAtbTa09cDz5ntD7BMbYxGC0,5865 tests/integrations/test_package.py,sha256=WvOmIjwMoLQ474nSv4m167KxIYfhomNFCj5F7INRbV4,29626 tests/integrations/test_service.py,sha256=riu1nK_VmZNGjtDTRCAK_KZ0SStNLHGnZkwfHhEl5F4,7959 tests/integrations/test_ssl.py,sha256=03CV2_38xg3b5gfVBGYtETEkcz0mo8Nbed4-E02cUB8,3092 tests/integrations/test_task.py,sha256=DS61wnoWEZkxagKSARVnTlKlx2w6IFWMXDHaG6jOedQ,9303 tests/unit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 tests/unit/test_tables.py,sha256=g_Zk4od-XtCS39113KbLLNb1fMecSruM8-phDYhmJiM,2332 PKցFHtests/unit/__init__.pyPKցFHc  4tests/unit/test_tables.pyPKցFH tests/integrations/__init__.pyPKցFH[`E@@! tests/integrations/test_config.pyPKցFHD$Jtests/integrations/test_dcos.pyPKցFHlnh;h;Rtests/integrations/common.pyPKցFHWWtests/integrations/test_auth.pyPKցFHJ$  *tests/integrations/test_ssl.pyPKցFHiIF  *ztests/integrations/test_marathon_groups.pyPKցFH$tests/integrations/test_http_auth.pyPKցFHM$tests/integrations/test_analytics.pyPKցFH2 tests/integrations/test_help.pyPKցFHU#tests/integrations/test_node.pyPKցFH,|W$W$tests/integrations/test_task.pyPKցFHp"'tests/integrations/test_service.pyPKցFH}ss"Gtests/integrations/test_package.pyPKցFH{۵#[#[#tests/integrations/test_marathon.pyPKցFHbtests/fixtures/__init__.pyPKցFHKǘtests/fixtures/marathon.pyPKցFHj'tests/fixtures/task.pyPKցFH _ggA0tests/fixtures/node.pyPKցFH,5tests/fixtures/service.pyPKցFH=O1?;tests/fixtures/package.pyPKFH]rڃLdcoscli/__init__.pyPKցFHmBB~Mdcoscli/log.pyPKցFHȺddcoscli/common.pyPKցFH`,`,hdcoscli/tables.pyPKցFHÆ  dcoscli/constants.pyPKցFH+4dcoscli/analytics.pyPKցFH dcoscli/main.pyPKցFHdcoscli/marathon/__init__.pyPKցFH8 5X5X%dcoscli/marathon/main.pyPKցFHj?O)dcoscli/data/universe-schema/package.jsonPKցFHt,,)dcoscli/data/universe-schema/command.jsonPKցFHN(c!dcoscli/data/universe-schema/config.jsonPKցFH '2dcoscli/data/config-schema/package.jsonPKցFHV-zz(6dcoscli/data/config-schema/marathon.jsonPKցFH+#$8dcoscli/data/config-schema/core.jsonPKցFH%o?dcoscli/data/help/package.txtPKցFH)17Hdcoscli/data/help/config.txtPKցFH/2 Kdcoscli/data/help/dcos.txtPKցFHVOORdcoscli/data/help/task.txtPKցFHy\\cVdcoscli/data/help/service.txtPKցFHkf\dcoscli/data/help/help.txtPKցFH|^dcoscli/data/help/node.txtPKցFH**cdcoscli/data/help/marathon.txtPKցFHPxdcoscli/node/__init__.pyPKցFHg 6VVxdcoscli/node/main.pyPKցFHdcoscli/task/__init__.pyPKցFHʓԗDdcoscli/task/main.pyPKցFHhdcoscli/package/__init__.pyPKցFHz\Z\Zdcoscli/package/main.pyPKցFH2dcoscli/service/__init__.pyPKցFHP#沋kdcoscli/service/main.pyPKցFH+%dcoscli/config/__init__.pyPKցFH8a,,c%dcoscli/config/main.pyPKցFHARdcoscli/help/__init__.pyPKցFHh wRdcoscli/help/main.pyPKցFH h##$]dcoscli-0.3.1.data/scripts/env-setupPKFHX'_dcoscli-0.3.1.dist-info/DESCRIPTION.rstPKFH'$999(&bdcoscli-0.3.1.dist-info/entry_points.txtPKFHn;;%cdcoscli-0.3.1.dist-info/metadata.jsonPKFH̚r.%#kdcoscli-0.3.1.dist-info/top_level.txtPKFH>nntkdcoscli-0.3.1.dist-info/WHEELPKFHS ldcoscli-0.3.1.dist-info/METADATAPKFHkG]rdcoscli-0.3.1.dist-info/RECORDPKBB1