PK3mG&>n>n>shutit_util.py#!/usr/bin/env pythen """ShutIt utility functions. """ #The MIT License (MIT) # #Copyright (C) 2014 OpenBet Limited # #Permission is hereby granted, free of charge, to any person obtaining a copy of #this software and associated documentation files (the "Software"), to deal in #the Software without restriction, including without limitation the rights to #use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies #of the Software, and to permit persons to whom the Software is furnished to do #so, subject to the following conditions: # #The above copyright notice and this permission notice shall be included in all #copies or substantial portions of the Software. # #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR #IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, #ITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL #THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER #LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE #SOFTWARE. import sys import argparse import os import stat from ConfigParser import RawConfigParser import time import re import imp import shutit_global from shutit_module import ShutItModule import pexpect import socket import textwrap import json import binascii import base64 import subprocess import getpass import StringIO import glob import hashlib import urlparse import urllib2 import shutil from shutit_module import ShutItFailException import operator import threading import string import random _default_cnf = ''' ################################################################################ # Default core config file for ShutIt. ################################################################################ # Details relating to the target you are building to (container, ssh or bash) [target] # Root password for the target - replace with your chosen password # If left blank, you will be prompted for a password password: # Hostname for the target - replace with your chosen target hostname # (where applicable, eg docker container) hostname: locale:en_US.UTF-8 # space separated list of ports to expose # e.g. "ports:2222:22 8080:80" would expose container ports 22 and 80 as the # host's 2222 and 8080 (where applicable) ports: # volume arguments, eg /tmp/postgres:/var/lib/postgres:ro volumes: # volumes-from arguments volumes_from: # Name to give the docker container (where applicable). # Empty means "let docker default a name". name: # Whether to remove the docker container when finished (where applicable). rm:no # Information specific to the host on which the build runs. [host] # Ask the user if they want shutit on their path add_shutit_to_path: yes # Folder with files you want to copy from in your build. # Often a good idea to have a central folder for this per host # in your /path/to/shutit/configs/`hostname`_`username`.cnf # If set to blank, then defaults to /path/to/shutit/artifacts (preferred) # If set to "artifacts", then defaults to the artifacts folder in the cwd. artifacts_dir: # Docker executable on your host machine docker_executable:docker # space separated list of dns servers to use dns: # Password for the username above on the host (only needed if sudo is needed) password: # Log file - will be set to 0600 perms, and defaults to /tmp/_shutit_log_ # A timestamp will be added to the end of the filename. logfile: # ShutIt paths to look up modules in separated by ":", eg /path1/here:/opt/path2/there shutit_module_path:. # Repository information [repository] # Whether to tag tag:yes # Whether to suffix the date to the tag suffix_date:no # Suffix format (default is epoch seconds (%s), but %Y%m%d_%H%M%S is an option if the length is ok with the index) suffix_format:%s # tag name name:my_module # Whether to tar up the docker image exported export:no # Whether to tar up the docker image saved save:no # Whether to push to the server push:no # User on registry to namespace repo - can be set to blank if not docker.io user: #Must be set if push is true/yes and user is not blank password:YOUR_INDEX_PASSWORD_OR_BLANK #Must be set if push is true/yes and user is not blank email:YOUR_INDEX_EMAIL_OR_BLANK # repository server # make blank if you want this to be sent to the main docker index on docker.io server: # tag suffix, defaults to "latest", eg registry/username/repository:latest. # empty is also "latest" repo_name: tag_name:latest # Root setup script # Each module should set these in a config [shutit.tk.setup] shutit.core.module.build:yes # Modules may rely on the below settings, only change for debugging. do_update:yes [shutit.tk.conn_bash] # None [shutit.tk.conn_ssh] # Required ssh_host: # All other configs are optional ssh_port: ssh_user: password: ssh_key: # (what to execute on the target to get a root shell) ssh_cmd: # Aspects of build process [build] build_log:yes # How to connect to target conn_module:shutit.tk.conn_docker # Run any docker container in privileged mode privileged:no # lxc-conf arg, eg #lxc_conf:lxc.aa_profile=unconfined lxc_conf: # Base image can be over-ridden by --image_tag defaults to this. base_image:ubuntu:14.04 # Whether to perform tests. dotest:yes # --net argument to docker, eg "bridge", "none", "container:" or "host". Empty means use default (bridge). net: ''' class LayerConfigParser(RawConfigParser): def __init__(self): RawConfigParser.__init__(self) self.layers = [] def read(self, filenames): if type(filenames) is not list: filenames = [filenames] for filename in filenames: cp = RawConfigParser() cp.read(filename) self.layers.append((cp, filename, None)) return RawConfigParser.read(self, filenames) def readfp(self, fp, filename=None): cp = RawConfigParser() fp.seek(0) cp.readfp(fp, filename) self.layers.append((cp, filename, fp)) fp.seek(0) ret = RawConfigParser.readfp(self, fp, filename) return ret def whereset(self, section, option): for cp, filename, fp in reversed(self.layers): if cp.has_option(section, option): return filename raise ShutItFailException('[%s]/%s was never set' % (section, option)) def get_config_set(self, section, option): """Returns a set with each value per config file in it. """ values = set() for cp, filename, fp in self.layers: if cp.has_option(section, option): values.add(cp.get(section, option)) return values def reload(self): """ Re-reads all layers again. In theory this should overwrite all the old values with any newer ones. It assumes we never delete a config item before reload. """ oldlayers = self.layers self.layers = [] for cp, filename, fp in oldlayers: if fp is None: self.read(filename) else: self.readfp(fp, filename) def remove_section(self, *args, **kwargs): raise NotImplementedError('Layer config parsers aren\'t directly mutable') def remove_option(self, *args, **kwargs): raise NotImplementedError('Layer config parsers aren\'t directly mutable') def set(self, *args, **kwargs): raise NotImplementedError('Layer config parsers aren\'t directly mutable') def is_file_secure(file_name): """Returns false if file is considered insecure, true if secure. If file doesn't exist, it's considered secure! """ if not os.path.isfile(file_name): return True file_mode = os.stat(file_name).st_mode if file_mode & (stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH): return False return True def colour(code, msg): """Colourize the given string for a terminal. """ return '\033[%sm%s\033[0m' % (code, msg) def get_configs(shutit, configs): """Reads config files in, checking their security first (in case passwords/sensitive info is in them). """ cfg = shutit.cfg cp = LayerConfigParser() fail_str = '' files = [] for config_file in configs: if type(config_file) is tuple: continue if not is_file_secure(config_file): fail_str = fail_str + '\nchmod 0600 ' + config_file files.append(config_file) if fail_str != '': if cfg['build']['interactive'] > 0: fail_str = 'Files are not secure, mode should be 0600. Running the following commands to correct:\n' + fail_str + '\n' # Actually show this to the user before failing... shutit.log(fail_str, force_stdout=True) shutit.log('\n\nDo you want me to run this for you? (input y/n)\n', force_stdout=True) if cfg['build']['interactive'] == 0 or util_raw_input(shutit=shutit,default='y') == 'y': for f in files: shutit.log('Correcting insecure file permissions on: ' + f, force_stdout=True) os.chmod(f,0600) # recurse return get_configs(shutit, configs) shutit.fail(fail_str) for config in configs: if type(config) is tuple: cp.readfp(config[1], filename=config[0]) else: cp.read(config) # Treat allowed_images as a special, additive case cfg['build']['shutit.core.module.allowed_images'] = cp.get_config_set('build', 'shutit.core.module.allowed_images') return cp def issue_warning(msg, wait): """Issues a warning to stderr. """ print >> sys.stderr, msg time.sleep(wait) def random_id(size=8, chars=string.ascii_letters + string.digits): """Generates a random string of given size from the given chars. @param size: The size of the random string. @param chars: Constituent pool of characters to draw random characters from. @type size: number @type chars: string @rtype: string @return: The string of random characters. """ return ''.join(random.choice(chars) for _ in range(size)) def random_word(size=6): """Returns a random word in lower case. """ word_file = find_asset('words') words = open(word_file).read().splitlines() word = '' while len(word) != 6 or word.find("'") > -1: word = words[int(random.random() * (len(words) - 1))] return word.lower() def find_asset(filename): dirs = ['/usr/share/dict', sys.prefix, os.path.join(sys.prefix,'local'), shutit_global.shutit_main_dir, os.path.join(shutit_global.shutit_main_dir,'../../..'), shutit_global.shutit.cfg['host']['shutit_path'], '/usr/local'] dirs = dirs + sys.path for iter_dir in dirs: if os.access(os.path.join(iter_dir,filename),os.F_OK): return os.path.join(iter_dir,filename) if os.access(os.path.join(os.path.join(iter_dir,'assets'),filename),os.F_OK): return os.path.join(os.path.join(iter_dir,'assets'),filename) if os.access(os.path.join(os.path.join(iter_dir,'shutit_assets'),filename),os.F_OK): return os.path.join(os.path.join(iter_dir,'shutit_assets'),filename) return filename # Manage config settings, returning a dict representing the settings # that have been sanity-checked. def get_base_config(cfg, cfg_parser): """Responsible for getting core configuration from config files. """ cfg['config_parser'] = cp = cfg_parser # BEGIN Read from config files # build - details relating to the build cfg['build']['privileged'] = cp.getboolean('build', 'privileged') cfg['build']['lxc_conf'] = cp.get('build', 'lxc_conf') cfg['build']['build_log'] = cp.getboolean('build', 'build_log') cfg['build']['base_image'] = cp.get('build', 'base_image') cfg['build']['dotest'] = cp.get('build', 'dotest') cfg['build']['net'] = cp.get('build', 'net') cfg['build']['completed'] = False cfg['build']['step_through'] = False cfg['build']['ctrlc_stop'] = False cfg['build']['check_exit'] = True # Width of terminal to set up on login and assume for other cases. cfg['build']['stty_cols'] = 320 # Take a command-line arg if given, else default. if cfg['build']['conn_module'] == None: cfg['build']['conn_module'] = cp.get('build', 'conn_module') # Track logins in a stack and details in logins. cfg['build']['login_stack'] = [] cfg['build']['logins'] = {} # Whether to accept default configs cfg['build']['accept_defaults'] = None # See shutit_global.check_environment cfg['build']['current_environment_id'] = None # target - the target of the build, ie the container cfg['target']['hostname'] = cp.get('target', 'hostname') cfg['target']['locale'] = cp.get('target', 'locale') cfg['target']['ports'] = cp.get('target', 'ports') cfg['target']['volumes'] = cp.get('target', 'volumes') cfg['target']['volumes_from'] = cp.get('target', 'volumes_from') cfg['target']['name'] = cp.get('target', 'name') cfg['target']['rm'] = cp.getboolean('target', 'rm') # host - the host on which the shutit script is run cfg['host']['add_shutit_to_path'] = cp.getboolean('host', 'add_shutit_to_path') cfg['host']['artifacts_dir'] = cp.get('host', 'artifacts_dir') cfg['host']['docker_executable'] = cp.get('host', 'docker_executable') cfg['host']['dns'] = cp.get('host', 'dns') cfg['host']['password'] = cp.get('host', 'password') cfg['host']['logfile'] = cp.get('host', 'logfile') cfg['host']['shutit_module_path'] = cp.get('host', 'shutit_module_path').split(':') # repository - information relating to repository/registry cfg['repository']['name'] = cp.get('repository', 'name') cfg['repository']['server'] = cp.get('repository', 'server') cfg['repository']['push'] = cp.getboolean('repository', 'push') cfg['repository']['tag'] = cp.getboolean('repository', 'tag') cfg['repository']['export'] = cp.getboolean('repository', 'export') cfg['repository']['save'] = cp.getboolean('repository', 'save') cfg['repository']['suffix_date'] = cp.getboolean('repository', 'suffix_date') cfg['repository']['suffix_format'] = cp.get('repository', 'suffix_format') cfg['repository']['user'] = cp.get('repository', 'user') cfg['repository']['password'] = cp.get('repository', 'password') cfg['repository']['email'] = cp.get('repository', 'email') cfg['repository']['tag_name'] = cp.get('repository', 'tag_name') # END Read from config files # BEGIN Standard expects # It's important that these have '.*' in them at the start, so that the matched data is reliably 'after' in the # child object. Use these where possible to make things more consistent. # Attempt to capture any starting prompt (when starting) with this regexp. cfg['expect_prompts']['base_prompt'] = '\r\n.*[@#$] ' # END Standard expects # BEGIN tidy configs up if cfg['host']['artifacts_dir'] == 'artifacts': cfg['host']['artifacts_dir'] = os.path.join(shutit_global.cwd, 'artifacts') elif cfg['host']['artifacts_dir'] == '': cfg['host']['artifacts_dir'] = os.path.join(shutit_global.shutit_main_dir, 'artifacts') if cfg['host']['logfile'] == '': if not os.access(cfg['build']['shutit_state_dir_base'],os.F_OK): os.mkdir(cfg['build']['shutit_state_dir_base']) if not os.access(cfg['build']['shutit_state_dir'],os.F_OK): os.mkdir(cfg['build']['shutit_state_dir']) os.chmod(cfg['build']['shutit_state_dir_base'],0777) os.chmod(cfg['build']['shutit_state_dir'],0777) logfile = os.path.join(cfg['build']['shutit_state_dir'], 'shutit_build.log') else: logfile = cfg['host']['logfile'] + '_' + cfg['build']['build_id'] cfg['host']['logfile'] = logfile if cfg['build']['build_log']: cfg['build']['build_log_file'] = open(logfile, 'a') # Lock it down to the running user. os.chmod(logfile,0600) # delivery method bash and image_tag make no sense if cfg['build']['delivery'] in ('bash','ssh'): if cfg['target']['docker_image'] != '': print('delivery method specified (' + cfg['build']['delivery'] + ') and image_tag argument make no sense') sys.exit(1) if cfg['target']['docker_image'] == '': cfg['target']['docker_image'] = cfg['build']['base_image'] # END tidy configs up # BEGIN warnings # Warn if something appears not to have been overridden warn = '' # FAILS begins # rm is incompatible with repository actions if cfg['target']['rm'] and (cfg['repository']['tag'] or cfg['repository']['push'] or cfg['repository']['save'] or cfg['repository']['export']): print("Can't have [target]/rm and [repository]/(push/save/export) set to true") sys.exit(1) if warn != '' and cfg['build']['debug']: issue_warning('Showing config as read in. This can also be done by calling with list_configs:',2) shutit_global.shutit.log(print_config(cfg), force_stdout=True, code='32') time.sleep(1) if cfg['target']['hostname'] != '' and cfg['build']['net'] != '' and cfg['build']['net'] != 'bridge': print('\n\ntarget/hostname or build/net configs must be blank\n\n') sys.exit(1) # FAILS ends # Returns the config dict def parse_args(shutit): """Responsible for parsing arguments. TODO: precedence of configs documented Environment variables: SHUTIT_OPTIONS: Loads command line options from the environment (if set). Behaves like GREP_OPTIONS: - space separated list of arguments - backslash before a space escapes the space separation - backslash before a backslash is interpreted as a single backslash - all other backslashes are treated literally eg ' a\ b c\\ \\d \\\e\' becomes '', 'a b', 'c\', '\d', '\\e\' SHUTIT_OPTIONS is ignored if we are creating a skeleton """ cfg = shutit.cfg cfg['host']['real_user_id'] = pexpect.run('id -u ' + cfg['host']['real_user']).strip() # These are in order of their creation actions = ['build', 'list_configs', 'list_modules', 'list_deps', 'serve', 'skeleton'] # COMPAT 2014-05-15 - build is the default if there is no action specified # and we've not asked for help and we've called via 'shutit_main.py' if len(sys.argv) == 1 or (len(sys.argv) > 1 and sys.argv[1] not in actions and '-h' not in sys.argv and '--help' not in sys.argv and os.path.basename(sys.argv[0]) == 'shutit_main.py'): sys.argv.insert(1, 'build') # Pexpect documentation says systems have issues with pauses < 0.05 def check_pause(value): ivalue = float(value) if ivalue < 0.05: raise argparse.ArgumentTypeError( "%s is an invalid pause (must be >= 0.05)" % value) return ivalue parser = argparse.ArgumentParser(description='ShutIt - a tool for managing complex Docker deployments.\n\nTo view help for a specific subcommand, type ./shutit -h',prog="ShutIt") parser.add_argument('--version', action='version', version='%(prog)s 0.7') subparsers = parser.add_subparsers(dest='action', help='''Action to perform - build=deploy to target, serve=run a shutit web server, skeleton=construct a skeleton module, list_configs=show configuration as read in, list_modules=show modules available, list_deps=show dep graph ready for graphviz. Defaults to 'build'.''') sub_parsers = dict() for action in actions: sub_parsers[action] = subparsers.add_parser(action) sub_parsers['skeleton'].add_argument('--module_directory', help='Absolute path to new directory for module',default='') sub_parsers['skeleton'].add_argument('--module_name', help='Name for your module. Single word and lower case, eg: mymysql',default='') sub_parsers['skeleton'].add_argument('--domain', help='Arbitrary but unique domain for namespacing your module, eg com.mycorp',default='') sub_parsers['skeleton'].add_argument('--depends', help='Module id to depend on, default shutit.tk.setup (optional)', default='shutit.tk.setup') sub_parsers['skeleton'].add_argument('--base_image', help='FROM image, default ubuntu:14.04 (optional)', default='ubuntu:14.04') sub_parsers['skeleton'].add_argument('--script', help='Pre-existing shell script to integrate into module (optional)', nargs='?', default=None) sub_parsers['skeleton'].add_argument('--output_dir', help='Just output the created directory', default=False, const=True, action='store_const') sub_parsers['skeleton'].add_argument('--dockerfile', default=None) sub_parsers['skeleton'].add_argument('--delivery', help='Delivery method, aka target. "docker" container (default), configured "ssh" connection, "bash" session', default=None, choices=('docker','dockerfile','ssh','bash')) sub_parsers['build'].add_argument('--export', help='Perform docker export to a tar file', const=True, default=False, action='store_const') sub_parsers['build'].add_argument('--save', help='Perform docker save to a tar file', const=True, default=False, action='store_const') sub_parsers['build'].add_argument('--push', help='Push to a repo', const=True, default=False, action='store_const') sub_parsers['build'].add_argument('--distro', help='Specify the distro type', default='', choices=('ubuntu','debian','alpine','steamos','red hat','centos','fedora','shutit')) sub_parsers['build'].add_argument('--mount_docker', help='Mount the docker socket', default=False, action='store_const', const=True) sub_parsers['build'].add_argument('-w','--walkthrough', help='Run in walkthrough mode', default=False, action='store_const', const=True) sub_parsers['build'].add_argument('--video', help='Run in video mode. Same as walkthrough, but waits n seconds rather than for input', nargs=1, default=-1) sub_parsers['list_configs'].add_argument('--history', help='Show config with history', const=True, default=False, action='store_const') sub_parsers['list_modules'].add_argument('--long', help='Show extended module info, including ordering', const=True, default=False, action='store_const') sub_parsers['list_modules'].add_argument('--sort', help='Order the modules seen, default to module id', default='id', choices=('id','run_order')) for action in ['build', 'serve', 'list_configs', 'list_modules', 'list_deps']: sub_parsers[action].add_argument('--config', help='Config file for setup config. Must be with perms 0600. Multiple arguments allowed; config files considered in order.', default=[], action='append') sub_parsers[action].add_argument('-d','--delivery', help='Delivery method, aka target. "docker" container (default), configured "ssh" connection, "bash" session', default=None, choices=('docker','dockerfile','ssh','bash')) sub_parsers[action].add_argument('-s', '--set', help='Override a config item, e.g. "-s target rm no". Can be specified multiple times.', default=[], action='append', nargs=3, metavar=('SEC', 'KEY', 'VAL')) sub_parsers[action].add_argument('--image_tag', help='Build container from specified image - if there is a symbolic reference, please use that, eg localhost.localdomain:5000/myref', default='') sub_parsers[action].add_argument('--tag_modules', help='''Tag each module after it's successfully built regardless of the module config and based on the repository config.''', default=False, const=True, action='store_const') sub_parsers[action].add_argument('-m', '--shutit_module_path', default=None, help='List of shutit module paths, separated by colons. ShutIt registers modules by running all .py files in these directories.') sub_parsers[action].add_argument('--pause', help='Pause between commands to avoid race conditions.', default='0.05', type=check_pause) sub_parsers[action].add_argument('--debug', help='Show debug.', default=False, const=True, action='store_const') sub_parsers[action].add_argument('--trace', help='Trace function calls', const=True, default=False, action='store_const') sub_parsers[action].add_argument('--interactive', help='Level of interactive. 0 = none, 1 = honour pause points and config prompting, 2 = query user on each module, 3 = tutorial mode', default='1') sub_parsers[action].add_argument('--ignorestop', help='Ignore STOP files', const=True, default=False, action='store_const') sub_parsers[action].add_argument('--ignoreimage', help='Ignore disallowed images', const=True, default=None, action='store_const') sub_parsers[action].add_argument('--imageerrorok', help='Exit without error if allowed images fails (used for test scripts)', const=True, default=False, action='store_const') sub_parsers[action].add_argument('--deps_only', help='build deps only, tag with suffix "_deps"', const=True, default=False, action='store_const') args_list = sys.argv[1:] if os.environ.get('SHUTIT_OPTIONS', None) and args_list[0] != 'skeleton': env_args = os.environ['SHUTIT_OPTIONS'].strip() # Split escaped backslashes env_args_split = re.split(r'(\\\\)', env_args) # Split non-escaped spaces env_args_split = [re.split(r'(?= 0: cfg['build']['walkthrough'] = True cfg['build']['walkthrough_wait'] = float(args.video[0]) elif cfg['action']['list_configs']: cfg['list_configs']['cfghistory'] = args.history elif cfg['action']['list_modules']: cfg['list_modules']['long'] = args.long cfg['list_modules']['sort'] = args.sort # What are we building on? Convert arg to conn_module we use. if args.delivery == 'docker' or args.delivery == None: cfg['build']['conn_module'] = 'shutit.tk.conn_docker' cfg['build']['delivery'] = 'docker' elif args.delivery == 'ssh': cfg['build']['conn_module'] = 'shutit.tk.conn_ssh' cfg['build']['delivery'] = 'ssh' elif args.delivery == 'bash' or args.delivery == 'dockerfile': cfg['build']['conn_module'] = 'shutit.tk.conn_bash' cfg['build']['delivery'] = args.delivery # If the image_tag has been set then ride roughshod over the ignoreimage value if not supplied if args.image_tag != '' and args.ignoreimage == None: args.ignoreimage = True # If ignoreimage is still not set, then default it to False if args.ignoreimage == None: args.ignoreimage = False # Get these early for this part of the build. # These should never be config arguments, since they are needed before config is passed in. if args.shutit_module_path is not None: module_paths = args.shutit_module_path.split(':') if '.' not in module_paths: if cfg['build']['debug']: shutit_global.shutit.log('Working directory path not included, adding...') time.sleep(1) module_paths.append('.') args.set.append(('host', 'shutit_module_path', ':'.join(module_paths))) cfg['build']['debug'] = args.debug cfg['build']['trace'] = args.trace cfg['build']['interactive'] = int(args.interactive) cfg['build']['command_pause'] = float(args.pause) cfg['build']['extra_configs'] = args.config cfg['build']['config_overrides'] = args.set cfg['build']['ignorestop'] = args.ignorestop cfg['build']['ignoreimage'] = args.ignoreimage cfg['build']['imageerrorok'] = args.imageerrorok cfg['build']['tag_modules'] = args.tag_modules cfg['build']['deps_only'] = args.deps_only cfg['target']['docker_image'] = args.image_tag # Finished parsing args. # Sort out config path if cfg['build']['interactive'] >= 3 or cfg['action']['list_configs'] or cfg['action']['list_modules'] or cfg['action']['list_deps'] or cfg['build']['debug']: cfg['build']['log_config_path'] = cfg['build']['shutit_state_dir'] + '/config/' + cfg['build']['build_id'] if os.path.exists(cfg['build']['log_config_path']): print(cfg['build']['log_config_path'] + ' exists. Please move and re-run.') sys.exit(1) os.makedirs(cfg['build']['log_config_path']) os.chmod(cfg['build']['log_config_path'],0777) # Tutorial stuff. if cfg['build']['interactive'] >= 3: print textwrap.dedent("""\ ================================================================================ SHUTIT - INTRODUCTION ================================================================================ ShutIt is a script that allows the building of static target environments. allowing a high degree of flexibility and easy conversion from other build methods (eg bash scripts) It is configured through command-line arguments (see --help) and .cnf files. ================================================================================ ================================================================================ CONFIG ================================================================================ The config is read in the following order: ================================================================================ ~/.shutit/config - Host- and username-specific config for this host. /path/to/this/shutit/module/configs/build.cnf - Config specifying what should be built when this module is invoked. /your/path/to/.cnf - Passed-in config (via --config, see --help) command-line overrides, eg -s com.mycorp.mymodule.module name value ================================================================================ Config items look like this: [section] name:value or as command-line overrides: -s section name value ================================================================================ """ + colour('32', '\n[Hit return to continue]')) util_raw_input() print textwrap.dedent("""\ ================================================================================ MODULES ================================================================================ Each module (which is a .py file) has a lifecycle, "module_id" and "run_order". The lifecycle (briefly) is as follows: foreach module: remove all modules config'd for removal foreach module: build tag stop all modules already started do repository work configured start all modules that were stopped start foreach module: test module stop all modules already started foreach module: finalize module and these stages are run from the module code, returning True or False as appropriate. The module_id is a string that uniquely identifies the module. The run_order is a float that defines the order in which the module should be run relative to other modules. This guarantees a deterministic ordering of the modules run. See shutit_module.py for more detailed documentation on these. ================================================================================ """ + colour('32', '\n[Hit return to continue]')) util_raw_input() print textwrap.dedent("""\ ================================================================================ PAUSE POINTS ================================================================================ Pause points can be placed within the build, which is useful for debugging. This is used throughout this tutorial. When debugging, pause_points will output your keyboard input before you finish. This can help you build your build, as these commands can be pasted into the module you are developing easily. To escape a pause point when it happens, hit the "CTRL" and the "]" key simultaneously. ================================================================================ """ + colour('32', '\n[Hit return to continue]')) util_raw_input() # Set up trace as fast as possible. if cfg['build']['trace']: def tracefunc(frame, event, arg, indent=[0]): if event == "call": shutit.log("-> call function: " + frame.f_code.co_name + " " + str(frame.f_code.co_varnames),force_stdout=True) elif event == "return": shutit.log("<- exit function: " + frame.f_code.co_name,force_stdout=True) return tracefunc sys.settrace(tracefunc) def load_configs(shutit): """Responsible for loading config files into ShutIt. Recurses down from configured shutit module paths. """ cfg = shutit.cfg # Get root default config. configs = [('defaults', StringIO.StringIO(_default_cnf))] # Add the shutit global host- and user-specific config file. configs.append(os.path.join(shutit.shutit_main_dir, 'configs/' + socket.gethostname() + '_' + cfg['host']['real_user'] + '.cnf')) configs.append(os.path.join(cfg['shutit_home'], 'config')) # Add the local build.cnf configs.append('configs/build.cnf') # Get passed-in config(s) for config_file_name in cfg['build']['extra_configs']: run_config_file = os.path.expanduser(config_file_name) if not os.path.isfile(run_config_file): print('Did not recognise ' + run_config_file + ' as a file - do you need to touch ' + run_config_file + '?') sys.exit() configs.append(run_config_file) # Image to use to start off. The script should be idempotent, so running it # on an already built image should be ok, and is advised to reduce diff space required. if cfg['build']['interactive'] >= 3 or cfg['action']['list_configs'] or cfg['build']['debug']: msg = '' print textwrap.dedent("""\n""") + textwrap.dedent("""Looking at config files in the following order:""") for c in configs: if type(c) is tuple: c = c[0] msg = msg + ' \n' + c shutit.log(' ' + c) if cfg['build']['interactive'] >= 3: print textwrap.dedent("""\n""") + msg + textwrap.dedent(colour('32', '\n\n[Hit return to continue]')) util_raw_input(shutit=shutit) if cfg['action']['list_configs'] or cfg['build']['debug']: f = file(cfg['build']['log_config_path'] + '/config_file_order.txt','w') f.write(msg) f.close() # Interpret any config overrides, write to a file and add them to the # list of configs to be interpreted if cfg['build']['config_overrides']: # We don't need layers, this is a temporary configparser override_cp = RawConfigParser() for o_sec, o_key, o_val in cfg['build']['config_overrides']: if not override_cp.has_section(o_sec): override_cp.add_section(o_sec) override_cp.set(o_sec, o_key, o_val) override_fd = StringIO.StringIO() override_cp.write(override_fd) override_fd.seek(0) configs.append(('overrides', override_fd)) cfg_parser = get_configs(shutit, configs) get_base_config(cfg, cfg_parser) if cfg['build']['debug']: # Set up the manhole. try: import manhole manhole.install( verbose=True, patch_fork=True, activate_on=None, oneshot_on=None, sigmask=manhole.ALL_SIGNALS, socket_path=None, reinstall_delay=0.5, locals=None ) except Exception: shutit.log('No manhole package available, skipping import') pass def load_shutit_modules(shutit): """Responsible for loading the shutit modules based on the configured module paths. """ cfg = shutit.cfg if cfg['build']['debug']: shutit.log('ShutIt module paths now: ') shutit.log(cfg['host']['shutit_module_path']) time.sleep(1) for shutit_module_path in cfg['host']['shutit_module_path']: load_all_from_path(shutit, shutit_module_path) def list_modules(shutit): """Display a list of loaded modules. Config items: - ['list_modules']['long'] If set, also print each module's run order value - ['list_modules']['sort'] Select the column by which the list is ordered: - id: sort the list by module id - run_order: sort the list by module run order The output is also saved to ['build']['log_config_path']/module_order.txt Dependencies: texttable, operator """ cfg = shutit.cfg # list of module ids and other details # will also contain column headers table_list = [] if cfg['list_modules']['long']: # --long table: sort modules by run order table_list.append(["Order","Module ID","Description","Run Order"]) else: # "short" table ==> sort module by module_id table_list.append(["Module ID","Description"]) if cfg['list_modules']['sort'] == 'run_order': a = {} for m in shutit.shutit_modules: a.update({m.module_id:m.run_order}) # sort dict by run_order; see http://stackoverflow.com/questions/613183/sort-a-python-dictionary-by-value b = sorted(a.items(), key=operator.itemgetter(1)) count = 0 # now b is a list of tuples (module_id, run_order) for pair in b: # module_id is the first item of the tuple k = pair[0] for m in shutit.shutit_modules: if m.module_id == k: count = count + 1 if cfg['list_modules']['long']: table_list.append([str(count),m.module_id,m.description,str(m.run_order)]) else: table_list.append([m.module_id,m.description]) elif cfg['list_modules']['sort'] == 'id': a = [] for m in shutit.shutit_modules: a.append(m.module_id) a.sort() count = 0 for k in a: for m in shutit.shutit_modules: if m.module_id == k: count = count + 1 if cfg['list_modules']['long']: table_list.append([str(count),m.module_id,m.description,str(m.run_order)]) else: table_list.append([m.module_id,m.description]) # format table for display import texttable table = texttable.Texttable() table.add_rows(table_list) msg = table.draw() print msg f = file(cfg['build']['log_config_path'] + '/module_order.txt','w') f.write(msg) f.close() def print_config(cfg, hide_password=True, history=False): """Returns a string representing the config of this ShutIt run. """ cp = cfg['config_parser'] s = '' keys1 = cfg.keys() if keys1: keys1.sort() for k in keys1: if type(k) == str and type(cfg[k]) == dict: s += '\n[' + k + ']\n' keys2 = cfg[k].keys() if keys2: keys2.sort() for k1 in keys2: line = '' line += k1 + ':' # If we want to hide passwords, we do so using a sha512 # done an aritrary number of times (27). if hide_password and (k1 == 'password' or k1 == 'passphrase'): p = hashlib.sha512(cfg[k][k1]).hexdigest() i = 27 while i > 0: i = i - 1 p = hashlib.sha512(s).hexdigest() line += p else: if type(cfg[k][k1] == bool): line += str(cfg[k][k1]) elif type(cfg[k][k1] == str): line += cfg[k][k1] if history: try: line += (30-len(line)) * ' ' + ' # ' + cp.whereset(k, k1) except Exception: # Assume this is because it was never set by a config parser. line += (30-len(line)) * ' ' + ' # ' + "defaults in code" s += line + '\n' return s def set_pexpect_child(key, child): """Set a pexpect child in the global dictionary by key. """ shutit_global.pexpect_children.update({key:child}) def get_pexpect_child(key): """Get a pexpect child in the global dictionary by key. """ return shutit_global.pexpect_children[key] def load_all_from_path(shutit, path): """Dynamically imports files within the same directory (in the end, the path). """ #111: handle expanded paths path = os.path.abspath(path) #http://stackoverflow.com/questions/301134/dynamic-module-import-in-python if os.path.abspath(path) == shutit.shutit_main_dir: return if not os.path.exists(path): return if os.path.exists(path + '/STOPBUILD') and not cfg['build']['ignorestop']: shutit.log('Ignoring directory: ' + path + ' as it has a STOPBUILD file in it. Pass --ignorestop to shutit run to override.', force_stdout=True) return for sub in glob.glob(os.path.join(path, '*')): subpath = os.path.join(path, sub) if os.path.isfile(subpath): load_mod_from_file(shutit, subpath) elif os.path.isdir(subpath): load_all_from_path(shutit, subpath) def load_mod_from_file(shutit, fpath): """Loads modules from a .py file into ShutIt if there are no modules from this file already. We expect to have a callable 'module/0' which returns one or more module objects. If this doesn't exist we assume that the .py file works in the old style (automatically inserting the module into shutit_global) or it's not a shutit module. """ cfg = shutit.cfg fpath = os.path.abspath(fpath) file_ext = os.path.splitext(os.path.split(fpath)[-1])[-1] if file_ext.lower() != '.py': return if re.match(shutit_global.cwd + '\/context\/.*',fpath): shutit.log('Ignoring file: "' + fpath + '" as this appears to be part of the context directory') return # Do we already have modules from this file? If so we know we can skip. # Note that this attribute will only be set for 'new style' module loading, # this should be ok because 'old style' loading checks for duplicate # existing modules. # TODO: this is quadratic complexity existingmodules = [ m for m in shutit.shutit_modules if getattr(m, '__module_file', None) == fpath ] if len(existingmodules) > 0: return # Looks like it's ok to load this file if cfg['build']['debug']: shutit.log('Loading source for: ' + fpath) # Add this directory to the python path iff not already there. directory = os.path.dirname(fpath) if directory not in sys.path: sys.path.append(os.path.dirname(fpath)) mod_name = base64.b32encode(fpath).replace('=', '') pymod = imp.load_source(mod_name, fpath) # Got the python module, now time to pull the shutit module(s) out of it. targets = [ ('module', shutit.shutit_modules), ('conn_module', shutit.conn_modules) ] cfg['build']['source'] = {} for attr, target in targets: modulefunc = getattr(pymod, attr, None) # Old style or not a shutit module, nothing else to do if not callable(modulefunc): return modules = modulefunc() if type(modules) is not list: modules = [modules] for module in modules: setattr(module, '__module_file', fpath) ShutItModule.register(module.__class__) target.add(module) cfg['build']['source'][fpath] = open(fpath).read() # Build report def build_report(shutit, msg=''): """Resposible for constructing a report to be output as part of the build. Retrurns report as a string. """ cfg = shutit.cfg s = '' s += '################################################################################\n' s += '# COMMAND HISTORY BEGIN ' + shutit_global.cfg['build']['build_id'] + '\n' s += get_commands(shutit) s += '# COMMAND HISTORY END ' + shutit_global.cfg['build']['build_id'] + '\n' s += '################################################################################\n' s += '################################################################################\n' s += '# BUILD REPORT FOR BUILD BEGIN ' + shutit_global.cfg['build']['build_id'] + '\n' s += '# ' + msg + '\n' if shutit_global.cfg['build']['report'] != '': s += shutit_global.cfg['build']['report'] + '\n' else: s += '# Nothing to report\n' if 'container_id' in cfg['target']: s += '# CONTAINER_ID: ' + cfg['target']['container_id'] + '\n' s += '# BUILD REPORT FOR BUILD END ' + shutit_global.cfg['build']['build_id'] + '\n' s += '###############################################################################\n' return s def get_commands(shutit): """Gets command that have been run and have not been redacted. """ s = '' for c in shutit.shutit_command_history: if type(c) == str: #Ignore commands with leading spaces if c[0] != ' ': s += c + '\n' return s def get_hash(string): """Helper function to get preceding integer eg com.openbet == 1003189494 >>> import binascii >>> abs(binascii.crc32('shutit.tk')) 782914092 Recommended means of determining run order integer part. """ return abs(binascii.crc32(string)) def create_skeleton(shutit): """Helper function to create a standard module directory ready to run and tinker with. """ cfg = shutit.cfg shutit_dir = sys.path[0] # Set up local directories skel_path = cfg['skeleton']['path'] skel_module_name = cfg['skeleton']['module_name'] skel_domain = cfg['skeleton']['domain'] skel_domain_hash = cfg['skeleton']['domainhash'] skel_depends = cfg['skeleton']['depends'] skel_base_image = cfg['skeleton']['base_image'] skel_script = cfg['skeleton']['script'] skel_dockerfile = cfg['skeleton']['dockerfile'] skel_output_dir = cfg['skeleton']['output_dir'] skel_delivery = cfg['skeleton']['delivery'] # Set up dockerfile cfg cfg['dockerfile']['base_image'] = skel_base_image cfg['dockerfile']['cmd'] = """/bin/sh -c 'sleep infinity'""" cfg['dockerfile']['user'] = '' cfg['dockerfile']['maintainer'] = '' cfg['dockerfile']['entrypoint'] = '' cfg['dockerfile']['expose'] = [] cfg['dockerfile']['env'] = [] cfg['dockerfile']['volume'] = [] cfg['dockerfile']['onbuild'] = [] cfg['dockerfile']['script'] = [] # Check setup if len(skel_path) == 0 or skel_path[0] != '/': shutit.fail('Must supply a directory and it must be absolute') if os.path.exists(skel_path): shutit.fail(skel_path + ' already exists') if len(skel_module_name) == 0: shutit.fail('Must supply a name for your module, eg mymodulename') if not re.match('^[a-zA-z_][0-9a-zA-Z_]+$', skel_module_name): shutit.fail('Module names must comply with python classname standards: cf: http://stackoverflow.com/questions/10120295/valid-characters-in-a-python-class-name') if len(skel_domain) == 0: shutit.fail('Must supply a domain for your module, eg com.yourname.madeupdomainsuffix') os.makedirs(skel_path) os.mkdir(os.path.join(skel_path, 'configs')) os.mkdir(os.path.join(skel_path, 'bin')) if skel_delivery != 'bash': os.mkdir(os.path.join(skel_path, 'context')) os.mkdir(os.path.join(skel_path, 'haproxy')) templatemodule_path = os.path.join(skel_path, skel_module_name + '.py') readme_path = os.path.join(skel_path, 'README.md') buildsh_path = os.path.join(skel_path, 'bin', 'build.sh') testsh_path = os.path.join(skel_path, 'bin', 'test.sh') runsh_path = os.path.join(skel_path, 'bin', 'run.sh') phoenixsh_path = os.path.join(skel_path, 'bin', 'phoenix.sh') buildpushsh_path = os.path.join(skel_path, 'bin', 'build_and_push.sh') buildcnf_path = os.path.join(skel_path, 'configs', 'build.cnf') pushcnf_path = os.path.join(skel_path, 'configs', 'push.cnf') builddockerfile_path = os.path.join(skel_path, 'Dockerfile') if skel_delivery != 'bash': haproxycnf_path = os.path.join(skel_path, 'haproxy', 'haproxy.cfg') haproxydockerfile_path = os.path.join(skel_path, 'haproxy', 'Dockerfile') if skel_dockerfile: if os.path.basename(skel_dockerfile) != 'Dockerfile': skel_dockerfile += '/Dockerfile' if not os.path.exists(skel_dockerfile): if urlparse.urlparse(skel_dockerfile)[0] == '': shutit.fail('Dockerfile "' + skel_dockerfile + '" must exist') dockerfile_contents = urllib2.urlopen(skel_dockerfile).read() dockerfile_dirname = None else: dockerfile_contents = open(skel_dockerfile).read() dockerfile_dirname = os.path.dirname(skel_dockerfile) if dockerfile_dirname == '': dockerfile_dirname = './' if os.path.exists(dockerfile_dirname): shutil.rmtree(skel_path + '/context') shutil.copytree(dockerfile_dirname, skel_path + '/context') # Remove Dockerfile as it's not part of the context. if os.path.isfile(skel_path + '/context/Dockerfile'): os.remove(skel_path + '/context/Dockerfile') # Change to this context os.chdir(dockerfile_dirname) # Wipe the command as we expect one in the file. cfg['dockerfile']['cmd'] = '' dockerfile_list = parse_dockerfile(shutit, dockerfile_contents) # Set defaults from given dockerfile for item in dockerfile_list: # These items are not order-dependent and don't affect the build, so we collect them here: docker_command = item[0].upper() if docker_command == 'FROM': # Should be only one of these cfg['dockerfile']['base_image'] = item[1] elif docker_command == "ONBUILD": # Maps to finalize :) - can we have more than one of these? assume yes # This contains within it one of the above commands, so we need to abstract this out. cfg['dockerfile']['onbuild'].append(item[1]) elif docker_command == "MAINTAINER": cfg['dockerfile']['maintainer'] = item[1] elif docker_command == "VOLUME": # Put in the run.sh. try: cfg['dockerfile']['volume'].append(' '.join(json.loads(item[1]))) except Exception: cfg['dockerfile']['volume'].append(item[1]) elif docker_command == 'EXPOSE': # Put in the run.sh. cfg['dockerfile']['expose'].append(item[1]) elif docker_command == "ENTRYPOINT": # Put in the run.sh? Yes, if it exists it goes at the front of cmd try: cfg['dockerfile']['entrypoint'] = ' '.join(json.loads(item[1])) except Exception: cfg['dockerfile']['entrypoint'] = item[1] elif docker_command == "CMD": # Put in the run.sh try: cfg['dockerfile']['cmd'] = ' '.join(json.loads(item[1])) except Exception: cfg['dockerfile']['cmd'] = item[1] # Other items to be run through sequentially (as they are part of the script) if docker_command == "USER": # Put in the start script as well as su'ing from here - assuming order dependent? cfg['dockerfile']['script'].append((docker_command, item[1])) # We assume the last one seen is the one we use for the image. # Put this in the default start script. cfg['dockerfile']['user'] = item[1] elif docker_command == 'ENV': # Put in the run.sh. cfg['dockerfile']['script'].append((docker_command, item[1])) # Set in the build cfg['dockerfile']['env'].append(item[1]) elif docker_command == "RUN": # Only handle simple commands for now and ignore the fact that Dockerfiles run # with /bin/sh -c rather than bash. try: cfg['dockerfile']['script'].append((docker_command, ' '.join(json.loads(item[1])))) except Exception: cfg['dockerfile']['script'].append((docker_command, item[1])) elif docker_command == "ADD": # Send file - is this potentially got from the web? Is that the difference between this and COPY? cfg['dockerfile']['script'].append((docker_command, item[1])) elif docker_command == "COPY": # Send file cfg['dockerfile']['script'].append((docker_command, item[1])) elif docker_command == "WORKDIR": # Push and pop cfg['dockerfile']['script'].append((docker_command, item[1])) elif docker_command == "COMMENT": # Push and pop cfg['dockerfile']['script'].append((docker_command, item[1])) # We now have the script, so let's construct it inline here templatemodule = '' # Header. templatemodule += ''' # Created from dockerfile: ''' + skel_dockerfile + ''' # Maintainer: ''' + cfg['dockerfile']['maintainer'] + ''' from shutit_module import ShutItModule class template(ShutItModule): def is_installed(self, shutit): return False ''' # build build = '' numpushes = 0 wgetgot = False for item in cfg['dockerfile']['script']: dockerfile_command = item[0].upper() dockerfile_args = item[1].split() cmd = ' '.join(dockerfile_args).replace("'", "\\'") if dockerfile_command == 'RUN': build += """\n\t\tshutit.send('""" + cmd + """')""" elif dockerfile_command == 'WORKDIR': build += """\n\t\tshutit.send('pushd """ + cmd + """')""" numpushes = numpushes + 1 elif dockerfile_command == 'COPY' or dockerfile_command == 'ADD': # The path must be inside the context of the build; you cannot COPY ../something /something, because the first step of a docker build is to send the context directory (and subdirectories) to the docker daemon. if dockerfile_args[0][0:1] == '..' or dockerfile_args[0][0] == '/' or dockerfile_args[0][0] == '~': shutit.fail('Invalid line: ' + str(dockerfile_args) + ' file must be in local subdirectory') if dockerfile_args[1][-1] == '/': # Dir we're COPYing or ADDing to destdir = dockerfile_args[1] # File/dir we're COPYing or ADDing from fromfile = dockerfile_args[0] # Final file/dir outfile = destdir + fromfile if os.path.isfile(fromfile): outfiledir = os.path.dirname(fromfile) build += """\n\t\tshutit.send('mkdir -p """ + destdir + '/' + outfiledir + """')""" elif os.path.isdir(fromfile): build += """\n\t\tshutit.send('mkdir -p """ + destdir + fromfile + """')""" else: outfile = dockerfile_args[1] # If this is something we have to wget: if dockerfile_command == 'ADD' and urlparse.urlparse(dockerfile_args[0])[0] != '': if not wgetgot: build += """\n\t\tshutit.install('wget')""" wgetgot = True if dockerfile_args[1][-1] == '/': destdir = destdir[0:-1] outpath = urlparse.urlparse(dockerfile_args[0])[2] outpathdir = os.path.dirname(outpath) build += """\n\t\tshutit.send('mkdir -p """ + destdir + outpathdir + """')""" build += """\n\t\tshutit.send('wget -O """ + destdir + outpath + ' ' + dockerfile_args[0] + """')""" else: outpath = dockerfile_args[1] destdir = os.path.dirname(dockerfile_args[1]) build += """\n\t\tshutit.send('mkdir -p """ + destdir + """')""" build += """\n\t\tshutit.send('wget -O """ + outpath + ' ' + dockerfile_args[0] + """')""" else: # From the local filesystem on construction: localfile = dockerfile_args[0] # Local file location on build: buildstagefile = 'context/' + dockerfile_args[0] #if localfile[-4:] == '.tar': # build += """\n\t\tshutit.send_file('""" + outfile + '/' + localfile + """')""" #elif localfile[-4:] == '.bz2': #elif localfile[-3:] == '.gz': #elif localfile[-3:] == '.xz': if os.path.isdir(localfile): build += """\n\t\tshutit.send_host_dir('""" + outfile + """', '""" + buildstagefile + """')""" else: build += """\n\t\tshutit.send_host_file('""" + outfile + """', '""" + buildstagefile + """')""" elif dockerfile_command == 'ENV': cmd = '='.join(dockerfile_args).replace("'", "\\'") build += """\n\t\tshutit.send('export """ + '='.join(dockerfile_args) + """')""" elif dockerfile_command == 'COMMENT': build += """\n\t\t# """ + ' '.join(dockerfile_args) while numpushes > 0: build += """\n\t\tshutit.send('popd')""" numpushes = numpushes - 1 templatemodule += ''' def build(self, shutit):''' + build + ''' # Some useful API calls for reference. See shutit's docs for more info and options: # # ISSUING BASH COMMANDS # shutit.send(send,expect=) - Send a command, wait for expect (string or compiled regexp) # to be seen before continuing. By default this is managed # by ShutIt with shell prompts. # shutit.multisend(send,send_dict) - Send a command, dict contains {expect1:response1,expect2:response2,...} # shutit.send_and_get_output(send) - Returns the output of the sent command # shutit.send_and_match_output(send, matches) # - Returns True if any lines in output match any of # the regexp strings in the matches list # shutit.send_until(send,regexps) - Send command over and over until one of the regexps seen in the output. # shutit.run_script(script) - Run the passed-in string as a script # shutit.install(package) - Install a package # shutit.remove(package) - Remove a package # shutit.login(user='root', command='su -') # - Log user in with given command, and set up prompt and expects. # Use this if your env (or more specifically, prompt) changes at all, # eg reboot, bash, ssh # shutit.logout(command='exit') - Clean up from a login. # # COMMAND HELPER FUNCTIONS # shutit.add_to_bashrc(line) - Add a line to bashrc # shutit.get_url(fname, locations) - Get a file via url from locations specified in a list # shutit.get_ip_address() - Returns the ip address of the target # shutit.command_available(command) - Returns true if the command is available to run # # LOGGING AND DEBUG # shutit.log(msg,add_final_message=False) - # Send a message to the log. add_final_message adds message to # output at end of build # shutit.pause_point(msg='') - Give control of the terminal to the user # shutit.step_through(msg='') - Give control to the user and allow them to step through commands # # SENDING FILES/TEXT # shutit.send_file(path, contents) - Send file to path on target with given contents as a string # shutit.send_host_file(path, hostfilepath) # - Send file from host machine to path on the target # shutit.send_host_dir(path, hostfilepath) # - Send directory and contents to path on the target # shutit.insert_text(text, fname, pattern) # - Insert text into file fname after the first occurrence of # regexp pattern. # shutit.delete_text(text, fname, pattern) # - Delete text from file fname after the first occurrence of # regexp pattern. # shutit.replace_text(text, fname, pattern) # - Replace text from file fname after the first occurrence of # regexp pattern. # ENVIRONMENT QUERYING # shutit.host_file_exists(filename, directory=False) # - Returns True if file exists on host # shutit.file_exists(filename, directory=False) # - Returns True if file exists on target # shutit.user_exists(user) - Returns True if the user exists on the target # shutit.package_installed(package) - Returns True if the package exists on the target # shutit.set_password(password, user='') # - Set password for a given user on target # # USER INTERACTION # shutit.get_input(msg,default,valid[],boolean?,ispass?) # - Get input from user and return output # shutit.fail(msg) - Fail the program and exit with status 1 # ''' # Gather and place finalize bit finalize = '' for line in cfg['dockerfile']['onbuild']: finalize += '\n\t\tshutit.send(\'' + line + '\'' templatemodule += ''' def finalize(self, shutit):''' + finalize + ''' return True def test(self, shutit): return True def is_installed(self, shutit): return False def get_config(self, shutit): # CONFIGURATION # shutit.get_config(module_id,option,default=None,boolean=False) # - Get configuration value, boolean indicates whether the item is # a boolean type, eg get the config with: # shutit.get_config(self.module_id, 'myconfig', default='a value') # and reference in your code with: # shutit.cfg[self.module_id]['myconfig'] return True ''' templatemodule += """ def module(): return template( """ + """\'%s.%s.%s\'""" % (skel_domain, skel_module_name, skel_module_name) + """, """ + skel_domain_hash + ".00" + """, description='', delivery_methods=[('""" + skel_delivery + """')], maintainer='""" + cfg['dockerfile']['maintainer'] + """', depends=['%s""" % (skel_depends) + """'] ) """ # Return program to main shutit_dir if dockerfile_dirname: os.chdir(shutit_dir) else: templatemodule = open(find_asset('shutit_module_template_bare.py')).read() templatemodule = (templatemodule ).replace('template', skel_module_name ).replace('GLOBALLY_UNIQUE_STRING', '\'%s.%s.%s\'' % (skel_domain, skel_module_name, skel_module_name) ).replace('FLOAT', skel_domain_hash + '.00' ).replace('DEPENDS', skel_depends ).replace('DELIVERY', skel_delivery ) readme = skel_module_name + ': description of module directory in here' buildsh = textwrap.dedent('''\ #!/bin/bash [[ -z "$SHUTIT" ]] && SHUTIT="$1/shutit" [[ ! -a "$SHUTIT" ]] || [[ -z "$SHUTIT" ]] && SHUTIT="$(which shutit)" if [[ ! -a "$SHUTIT" ]] then echo "Must have shutit on path, eg export PATH=$PATH:/path/to/shutit_dir" exit 1 fi pushd .. $SHUTIT build -d ''' + skel_delivery + ''' "$@" if [[ $? != 0 ]] then popd exit 1 fi popd ''') testsh = textwrap.dedent('''\ #!/bin/bash # Test the building of this module if [ $0 != test.sh ] && [ $0 != ./test.sh ] then echo echo "Called as: $0" echo "Must be run as test.sh or ./test.sh" exit fi ./build.sh "$@" ''') volumes_arg = '' for varg in cfg['dockerfile']['volume']: volumes_arg += ' -v ' + varg + ':' + varg ports_arg = '' if type(cfg['dockerfile']['expose']) == str: for parg in cfg['dockerfile']['expose']: ports_arg += ' -p ' + parg + ':' + parg else: for parg in cfg['dockerfile']['expose']: for port in parg.split(): ports_arg += ' -p ' + port + ':' + port env_arg = '' for earg in cfg['dockerfile']['env']: env_arg += ' -e ' + earg.split()[0] + ':' + earg.split()[1] runsh = textwrap.dedent('''\ #!/bin/bash # Example for running DOCKER=${DOCKER:-docker} IMAGE_NAME=%s CONTAINER_NAME=$IMAGE_NAME DOCKER_ARGS='' while getopts "i:c:a:" opt do case "$opt" in i) IMAGE_NAME=$OPTARG ;; c) CONTAINER_NAME=$OPTARG ;; a) DOCKER_ARGS=$OPTARG ;; esac done ${DOCKER} run -d --name ${CONTAINER_NAME}''' % (skel_module_name,) + ports_arg + volumes_arg + env_arg + ' ${DOCKER_ARGS} ${IMAGE_NAME} ' + cfg['dockerfile']['entrypoint'] + ' ' + cfg['dockerfile']['cmd'] + '\n') buildpushsh = textwrap.dedent('''\ export SHUTIT_OPTIONS="$SHUTIT_OPTIONS --config configs/push.cnf -s repository push yes" ./build.sh "$@" ''') buildcnf = textwrap.dedent('''\ ############################################################################### # PLEASE NOTE: This file should be changed only by the maintainer. # PLEASE NOTE: This file is only sourced if the "shutit build" command is run # and this file is in the relative path: configs/build.cnf # This is to ensure it is only sourced if _this_ module is the # target. ############################################################################### # When this module is the one being built, which modules should be built along with it by default? # This feeds into automated testing of each module. [''' + '%s.%s.%s' % (skel_domain, skel_module_name, skel_module_name) + '''] shutit.core.module.build:yes # Allowed images as a regexp, eg ["ubuntu:12.*"], or [".*"], or ["centos"]. # It's recommended this is locked down as far as possible. shutit.core.module.allowed_images:["''' + cfg['dockerfile']['base_image'] + '''"] # Aspects of build process [build] base_image:''' + cfg['dockerfile']['base_image'] + ''' # Volume arguments wanted as part of the build [target] volumes: [repository] name:''' + skel_module_name + ''' ''') phoenixsh = textwrap.dedent('''\ #!/bin/bash set -e DOCKER=${DOCKER:-docker} CONTAINER_BASE_NAME=${CONTAINER_BASE_NAME:-%s} # haproxy image suffix # Sent on to: # HA_BACKEND_PORT_A # + # | # +------------------+ | +----------------+ # | | | | Container A | # | +---v----> Open on port: | # | HAProxy | | CONTAINER_PORT| # | Container | | | # | | +----------------+ #Request+---->received | # |on port: | +----------------+ # |HA_PROXY_PORT | | Container B | # | +---+----> Open on port: | # | | ^ | CONTAINER_PORT| # | | | | | # +------------------+ | +----------------+ # | # + # Sent on to: # HA_BACKEND_PORT_B # HA_PROXY_CONTAINER_SUFFIX=${HA_PROXY_CONTAINER_SUFFIX:-haproxy} # The port on which your haproxy image is configured to receive requests from inside HA_PROXY_PORT=${HA_PROXY_PORT:-8080} # The port on which your backend 'a' is configured to receive requests on the host HA_BACKEND_PORT_A=${HA_BACKEND_PORT_A:-8081} # The port on which your backend 'b' is configured to receive requests on the host HA_BACKEND_PORT_B=${HA_BACKEND_PORT_B:-8082} # The port on which your service container receives requests CONTAINER_PORT=${CONTAINER_PORT:-80} # Set up haproxy. # Remove proxy if it's died. If it doesn't exist, rebuild it first. HAPROXY=$($DOCKER ps --filter=name=${CONTAINER_BASE_NAME}_${HA_PROXY_CONTAINER_SUFFIX} -q) if [[ $HAPROXY = '' ]] then HAPROXY=$($DOCKER ps --filter=name=${CONTAINER_BASE_NAME}_${HA_PROXY_CONTAINER_SUFFIX} -q -a) if [[ $HAPROXY != '' ]] then $DOCKER rm -f ${CONTAINER_BASE_NAME}_${HA_PROXY_CONTAINER_SUFFIX} fi pushd ../haproxy sed "s/HA_PROXY_PORT/${HA_PROXY_PORT}/g;s/HA_BACKEND_PORT_A/${HA_BACKEND_PORT_A}/g;s/HA_BACKEND_PORT_B/${HA_BACKEND_PORT_B}/g" haproxy.cfg.template > haproxy.cfg $DOCKER build -t ${CONTAINER_BASE_NAME}_${HA_PROXY_CONTAINER_SUFFIX} . $DOCKER run -d --net=host --name ${CONTAINER_BASE_NAME}_${HA_PROXY_CONTAINER_SUFFIX} ${CONTAINER_BASE_NAME}_${HA_PROXY_CONTAINER_SUFFIX} popd fi # Cleanup any left-over containers, build the new one, rename the old one, # rename the new one, delete the old one. $DOCKER rm -f ${CONTAINER_BASE_NAME}_old > /dev/null 2>&1 || /bin/true ./build.sh -s repository tag yes -s repository name ${CONTAINER_BASE_NAME} # If there's a running instance, gather the used port, and move any old container USED_PORT='' NEW_PORT=${HA_BACKEND_PORT_A} if [[ $($DOCKER ps --filter=name="${CONTAINER_BASE_NAME}$" -q -a) != '' ]] then $DOCKER rm -f ${CONTAINER_BASE_NAME}_old > /dev/null 2>&1 || /bin/true USED_PORT=$($DOCKER inspect -f '{{range $p, $conf := .NetworkSettings.Ports}}{{(index $conf 0).HostPort}} {{end}}' $CONTAINER_BASE_NAME) # Decide which port to use if [[ "$USED_PORT" -eq "${HA_BACKEND_PORT_A}" ]] then NEW_PORT=${HA_BACKEND_PORT_B} fi $DOCKER rename ${CONTAINER_BASE_NAME} ${CONTAINER_BASE_NAME}_old fi # The random id is required - suspected docker bug RANDOM_ID=$RANDOM ./run.sh -i "${CONTAINER_BASE_NAME}" -c "${CONTAINER_BASE_NAME}_${RANDOM_ID}" -a "-p ${NEW_PORT}:${CONTAINER_PORT}" $DOCKER rm -f ${CONTAINER_BASE_NAME}_old > /dev/null 2>&1 || /bin/true $DOCKER rename ${CONTAINER_BASE_NAME}_${RANDOM_ID} ${CONTAINER_BASE_NAME}''' % (skel_module_name)) pushcnf = textwrap.dedent('''\ ############################################################################### # PLEASE NOTE: This file should be changed only by the maintainer. # PLEASE NOTE: IF YOU WANT TO CHANGE THE CONFIG, PASS IN # --config configfilename # OR ADD DETAILS TO YOUR # ~/.shutit/config # FILE ############################################################################### [target] rm:false [repository] # COPY THESE TO YOUR ~/.shutit/config FILE AND FILL OUT ITEMS IN CAPS #user:YOUR_USERNAME ## Fill these out in server- and username-specific config (also in this directory) #password:YOUR_REGISTRY_PASSWORD_OR_BLANK ## Fill these out in server- and username-specific config (also in this directory) #email:YOUR_REGISTRY_EMAIL_OR_BLANK #tag:no #push:yes #save:no #export:no ##server:REMOVE_ME_FOR_DOCKER_INDEX ## tag suffix, defaults to "latest", eg registry/username/repository:latest. ## empty is also "latest" #tag_name:latest #suffix_date:no #suffix_format:%s ''') haproxycnf = textwrap.dedent('''\ global maxconn 256 defaults mode tcp frontend front_door bind *:HA_PROXY_PORT default_backend nodes timeout client 10m backend nodes timeout connect 2s timeout server 10m server server1 127.0.0.1:HA_BACKEND_PORT_A maxconn 32 check server server2 127.0.0.1:HA_BACKEND_PORT_B maxconn 32 check''') haproxydockerfile = textwrap.dedent('''\ FROM haproxy:1.5 COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg''') builddockerfile = textwrap.dedent('''\ FROM ''' + cfg['dockerfile']['base_image'] + ''' RUN apt-get update RUN apt-get install -y -qq git python-pip RUN pip install shutit WORKDIR /opt # Change the next two lines to build your ShutIt module. RUN git clone https://github.com/yourname/yourshutitproject.git WORKDIR /opt/yourshutitproject RUN shutit build --delivery dockerfile CMD ["/bin/bash"] ''') open(templatemodule_path, 'w').write(templatemodule) open(buildsh_path, 'w').write(buildsh) os.chmod(buildsh_path, os.stat(buildsh_path).st_mode | 0111) # chmod +x open(buildcnf_path, 'w').write(buildcnf) os.chmod(buildcnf_path, 0400) if skel_delivery != 'bash': open(buildpushsh_path, 'w').write(buildpushsh) os.chmod(buildpushsh_path, os.stat(buildpushsh_path).st_mode | 0111) # chmod +x # build.cnf should be read-only (maintainer changes only) open(pushcnf_path, 'w').write(pushcnf) os.chmod(pushcnf_path, 0600) open(testsh_path, 'w').write(testsh) os.chmod(testsh_path, os.stat(testsh_path).st_mode | 0111) # chmod +x open(builddockerfile_path, 'w').write(builddockerfile) open(readme_path, 'w').write(readme) open(runsh_path, 'w').write(runsh) os.chmod(runsh_path, os.stat(runsh_path).st_mode | 0111) # chmod +x open(phoenixsh_path, 'w').write(phoenixsh) os.chmod(phoenixsh_path, os.stat(phoenixsh_path).st_mode | 0111) # chmod +x open(haproxycnf_path, 'w').write(haproxycnf) open(haproxycnf_path + '.template', 'w').write(haproxycnf) open(haproxydockerfile_path, 'w').write(haproxydockerfile) if skel_script is not None: print textwrap.dedent('''\ ================================================================================ Please note that your bash script in: ''' + skel_script + ''' should be a simple set of one-liners that return to the prompt. Anything fancy with ifs, backslashes or other multi-line commands need to be handled more carefully. ================================================================================''') sbsi = cfg['build']['shutit_state_dir'] + '/shutit_bash_script_include_' + str(int(time.time())) skel_mod_path = os.path.join(skel_path, skel_module_name + '.py') # Read in new file script_list = open(skel_script).read().splitlines() skel_mod_path_list = open(skel_mod_path).read().splitlines() new_script = [] for line in script_list: # remove leading space line = line.strip() # ignore empty lines # ignore lines with leading # if len(line) == 0 or line[0] == '#': continue # surround with send command and space line = "\t\tshutit.send('''" + line + "''')" # double quotes (?) new_script.append(line) # insert code into relevant part of skel_mod_path final_script = '' def_build_found = False # Go through each line of the base file for line in skel_mod_path_list: # Set trip switch to on once we find def build if string.find(line,'def build') != -1: def_build_found = True # If we're in the build method, and at the return line.... if def_build_found and string.find(line,'return True') != -1: # ...script in for new_script_line in new_script: final_script += new_script_line + '\r\n' # Set trip switch back to off def_build_found = False # Add line to final script final_script += line + '\r\n' open(skel_mod_path,'w').write(final_script) # Are we creating a new folder inside an existing git repo? if subprocess.call(['git', 'status'], stderr=open(os.devnull, 'wb'), stdout=open(os.devnull, 'wb')) != 0: subprocess.check_call(['git', 'init'], cwd=skel_path, stderr=open(os.devnull, 'wb'), stdout=open(os.devnull, 'wb')) try: subprocess.check_call([ 'cp', find_asset('.gitignore'), '.gitignore' ], cwd=skel_path) except Exception: #gitignore is not essential pass if skel_output_dir: print skel_path else: print textwrap.dedent('''\ ================================================================================ Run: cd ''' + skel_path + '''/bin && ./build.sh to build. An image called ''' + skel_module_name + ''' will be created and can be run with the run.sh command in bin/. ================================================================================''') # Parses the dockerfile (passed in as a string) # and info to extract, and returns a list with the information in a more canonical form, still ordered. def parse_dockerfile(shutit, contents): ret = [] full_line = '' for l in contents.split('\n'): # Handle continuations if len(l) > 0: if l[-1] == '\\': full_line += l[0:-1] pass else: full_line += l m = re.match("^[\s]*([A-Za-z]+)[\s]*(.*)$", full_line) m1 = None if m: ret.append([m.group(1), m.group(2)]) else: m1 = re.match("^#(..*)$", full_line) if m1: ret.append(['COMMENT', m1.group(1)]) else: shutit.log("Ignored line in parse_dockerfile: " + l) full_line = '' return ret def util_raw_input(shutit=None, prompt='', default=None, ispass=False): """Handles raw_input calls, and switches off interactivity if there is apparently no controlling terminal (or there are any other problems) """ msg = '' prompt = '\n' + prompt + '\n' if shutit and shutit.cfg['build']['interactive'] == 0: return default if not determine_interactive(shutit): return default try: if ispass: print prompt return getpass.getpass() else: resp = raw_input(prompt).strip() if resp == '': return default else: return resp except Exception: msg = 'Problems getting raw input, assuming no controlling terminal.' if shutit: set_noninteractive(shutit,msg=msg) return default def determine_interactive(shutit=None): """Determine whether we're in an interactive context. Sets interactivity off if appropriate. cf http://stackoverflow.com/questions/24861351/how-to-detect-if-python-script-is-being-run-as-a-background-process """ try: if not sys.stdout.isatty() or os.getpgrp() != os.tcgetpgrp(sys.stdout.fileno()): if shutit != None: set_noninteractive(shutit) return False except Exception: if shutit != None: set_noninteractive(shutit,msg='Problems determining interactivity, assuming not.') return False return True def set_noninteractive(shutit,msg="setting non-interactive"): cfg = shutit.cfg shutit.log(msg) cfg['build']['interactive'] = 0 return def print_stack_trace(): print '================================================================================' print 'Strack trace was:\n================================================================================' import traceback (a,b,c) = sys.exc_info() traceback.print_tb(c) print '================================================================================' # get the ordinal for a given char, in a friendly way def get_wide_hex(char): if len(char) != 2: return r'\x' + hex(ord(char))[2:] return r'\u' + hex(0x10000 + (ord(char[0]) - 0xD800) * 0x400 + (ord(char[1]) - 0xDC00))[2:] in_ctrlc = False def ctrlc_background(): global in_ctrlc in_ctrlc = True time.sleep(1) in_ctrlc = False def ctrl_c_signal_handler(signal, frame): """CTRL-c signal handler - enters a pause point if it can. """ if in_ctrlc: print "CTRL-c quit!" # Unfortunately we have 'except' blocks catching all exceptions, # so we can't use sys.exit os._exit(1) shutit_frame = get_shutit_frame(frame) print '\n' + '*' * 80 print "CTRL-c caught" if shutit_frame: shutit = shutit_frame.f_locals['shutit'] shutit.cfg['build']['ctrlc_stop'] = True print "You may need to wait for the command to complete for a pause point" print "CTRL-c twice to quit." print '*' * 80 t = threading.Thread(target=ctrlc_background) t.daemon = True t.start() def get_shutit_frame(frame): if not frame.f_back: return None else: if 'shutit' in frame.f_locals: return frame return get_shutit_frame(frame.f_back) def print_frame_recurse(frame): if not frame.f_back: return else: print '=============================================================================' print frame.f_locals print_frame_recurse(frame.f_back) def check_regexp(regex): if regex == None: # Is this ok? return True try: re.compile(regex); result = True except re.error: result = False return result PKBGnèeZZshutit_main.py#!/usr/bin/env python #The MIT License (MIT) # #Copyright (C) 2014 OpenBet Limited # #Permission is hereby granted, free of charge, to any person obtaining a copy of #this software and associated documentation files (the "Software"), to deal in #the Software without restriction, including without limitation the rights to #use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies #of the Software, and to permit persons to whom the Software is furnished to do #so, subject to the following conditions: # #The above copyright notice and this permission notice shall be included in all #copies or substantial portions of the Software. # #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR #IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, #ITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL #THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER #LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE #SOFTWARE. """ShutIt is a means of building stateless target hosts in a flexible and predictable way. """ from shutit_module import ShutItModule, ShutItException, ShutItFailException import ConfigParser import shutit_util import urllib import shutit_global import sys import os import json import re import signal from distutils import spawn def module_ids(shutit, rev=False): """Gets a list of module ids guaranteed to be sorted by run_order, ignoring conn modules (run order < 0). """ ids = sorted(shutit.shutit_map.keys(),key=lambda module_id: shutit.shutit_map[module_id].run_order) if rev: return list(reversed(ids)) else: return ids def allowed_module_ids(shutit, rev=False): """Gets a list of module ids that are allowed to be run, guaranteed to be sorted by run_order, ignoring conn modules (run order < 0). """ module_ids_list = module_ids(shutit,rev) allowed_module_ids = [] for module_id in module_ids_list: if allowed_image(shutit,module_id): allowed_module_ids.append(module_id) return allowed_module_ids def disallowed_module_ids(shutit, rev=False): """Gets a list of disallowed module ids that are not allowed to be run, guaranteed to be sorted by run_order, ignoring conn modules (run order < 0). """ module_ids_list = module_ids(shutit,rev) disallowed_module_ids = [] for module_id in module_ids_list: if not allowed_image(shutit,module_id): disallowed_module_ids.append(module_id) return disallowed_module_ids def print_modules(shutit): """Returns a string table representing the modules in the ShutIt module map. """ cfg = shutit.cfg string = '' string = string + 'Modules: \n' string = string + ' Run order Build Remove Module ID\n' for module_id in module_ids(shutit): string = string + (' ' + str(shutit.shutit_map[module_id].run_order) + ' ' + str(cfg[module_id]['shutit.core.module.build']) + ' ' + str(cfg[module_id]['shutit.core.module.remove']) + ' ' + module_id + '\n') return string # run_order of -1 means 'stop everything' def stop_all(shutit, run_order=-1): """Runs stop method on all modules less than the passed-in run_order. Used when target is exporting itself mid-build, so we clean up state before committing run files etc. """ cfg = shutit.cfg if cfg['build']['interactive'] >= 3: print('\nRunning stop on all modules' + \ shutit_util.colour('32', '\n\n[Hit return to continue]')) shutit_util.util_raw_input(shutit=shutit) # sort them so they're stopped in reverse order for module_id in module_ids(shutit, rev=True): shutit_module_obj = shutit.shutit_map[module_id] if run_order == -1 or shutit_module_obj.run_order <= run_order: if is_installed(shutit, shutit_module_obj): if not shutit_module_obj.stop(shutit): shutit.fail('failed to stop: ' + \ module_id, child=shutit.pexpect_children['target_child']) # Start all apps less than the supplied run_order def start_all(shutit, run_order=-1): """Runs start method on all modules less than the passed-in run_order. Used when target is exporting itself mid-build, so we can export a clean target and still depended-on modules running if necessary. """ cfg = shutit.cfg if cfg['build']['interactive'] >= 3: print('\nRunning start on all modules' + shutit_util.colour('32', '\n\n[Hit return to continue]\n')) shutit_util.util_raw_input(shutit=shutit) # sort them so they're started in order for module_id in module_ids(shutit): shutit_module_obj = shutit.shutit_map[module_id] if run_order == -1 or shutit_module_obj.run_order <= run_order: if is_installed(shutit, shutit_module_obj): if not shutit_module_obj.start(shutit): shutit.fail('failed to start: ' + module_id, \ child=shutit.pexpect_children['target_child']) def is_installed(shutit, shutit_module_obj): """Returns true if this module is installed. Uses cache where possible. """ cfg = shutit.cfg # Cache first cfg = shutit.cfg if shutit_module_obj.module_id in cfg['environment'][cfg['build']['current_environment_id']]['modules_installed']: return True if shutit_module_obj.module_id in cfg['environment'][cfg['build']['current_environment_id']]['modules_not_installed']: return False # Is it installed? if shutit_module_obj.is_installed(shutit): cfg['environment'][cfg['build']['current_environment_id']]['modules_installed'].append(shutit_module_obj.module_id) return True # If not installed, and not in cache, add it. else: if shutit_module_obj.module_id not in cfg['environment'][cfg['build']['current_environment_id']]['modules_not_installed']: cfg['environment'][cfg['build']['current_environment_id']]['modules_not_installed'].append(shutit_module_obj.module_id) return False def is_to_be_built_or_is_installed(shutit, shutit_module_obj): """Returns true if this module is configured to be built, or if it is already installed. """ cfg = shutit.cfg if cfg[shutit_module_obj.module_id]['shutit.core.module.build']: return True return is_installed(shutit, shutit_module_obj) def is_ready(shutit, shutit_module_obj): """Returns true if this module is ready to be built. Caches the result (as it's assumed not to change during the build). """ cfg = shutit.cfg if shutit_module_obj.module_id in cfg['environment'][cfg['build']['current_environment_id']]['modules_ready']: shutit.log('is_ready: returning True from cache') return True ready = shutit_module_obj.check_ready(shutit) if ready: cfg['environment'][cfg['build']['current_environment_id']]['modules_ready'].append(shutit_module_obj.module_id) return True else: return False def init_shutit_map(shutit): """Initializes the module map of shutit based on the modules we have gathered. Checks we have core modules Checks for duplicate module details. Sets up common config. Sets up map of modules. """ cfg = shutit.cfg modules = shutit.shutit_modules # Have we got anything to process outside of special modules? if len([mod for mod in modules if mod.run_order > 0]) < 1: shutit.log(modules) path = ':'.join(cfg['host']['shutit_module_path']) shutit.log('\nIf you are new to ShutIt, see:\n\n\thttp://ianmiell.github.io/shutit/\n\nor try running\n\n\tshutit skeleton\n\n',code=31,prefix=False,force_stdout=True) if path == '': shutit.fail('No ShutIt modules aside from core ones found and no ShutIt' + ' module path given. ' + '\nDid you set --shutit_module_path/-m wrongly?\n') elif path == '.': shutit.fail('No modules aside from core ones found and no ShutIt' + ' module path given apart from default (.).\n\n- Did you' + ' set --shutit_module_path/-m?\n- Is there a STOP* file' + ' in your . dir?\n') else: shutit.fail('No modules aside from core ones found and no ShutIt ' + 'modules in path:\n\n' + path + '\n\nor their subfolders. Check your ' + '--shutit_module_path/-m setting and check that there are ' + 'ShutIt modules below without STOP* files in any relevant ' + 'directories.\n') shutit.log('PHASE: base setup', code='32') if cfg['build']['interactive'] >= 3: shutit.log('\nChecking to see whether there are duplicate module ids ' + 'or run orders in the visible modules.', force_stdout=True) shutit.log('\nModules I see are:\n', force_stdout=True) for module in modules: shutit.log(module.module_id, force_stdout=True, code='32') shutit.log('\n', force_stdout=True) run_orders = {} has_core_module = False for module in modules: assert isinstance(module, ShutItModule) if module.module_id in shutit.shutit_map: shutit.fail('Duplicated module id: ' + module.module_id + '\n\nYou may want to check your --shutit_module_path setting') if module.run_order in run_orders: shutit.fail('Duplicate run order: ' + str(module.run_order) + ' for ' + module.module_id + ' and ' + run_orders[module.run_order].module_id + '\n\nYou may want to check your --shutit_module_path setting') if module.run_order == 0: has_core_module = True shutit.shutit_map[module.module_id] = run_orders[module.run_order] = module if not has_core_module: shutit.fail('No module with run_order=0 specified! This is required.') if cfg['build']['interactive'] >= 3: print(shutit_util.colour('32', 'Module id and run order checks OK' + '\n\n[Hit return to continue]\n')) shutit_util.util_raw_input(shutit=shutit) def config_collection(shutit): """Collect core config from config files for all seen modules. """ shutit.log('In config_collection') cfg = shutit.cfg for module_id in module_ids(shutit): # Default to None so we can interpret as ifneeded shutit.get_config(module_id, 'shutit.core.module.build', None, boolean=True, forcenone=True) shutit.get_config(module_id, 'shutit.core.module.remove', False, boolean=True) shutit.get_config(module_id, 'shutit.core.module.tag', False, boolean=True) # Default to allow any image shutit.get_config(module_id, 'shutit.core.module.allowed_images', [".*"]) module = shutit.shutit_map[module_id] cfg_file = os.path.dirname(module.__module_file) + '/configs/build.cnf' if os.path.isfile(cfg_file): # use shutit.get_config, forcing the passed-in default config_parser = ConfigParser.ConfigParser() config_parser.read(cfg_file) for section in config_parser.sections(): if section == module_id: for option in config_parser.options(section): if option == 'shutit.core.module.allowed_images': override = False for mod, opt, val in cfg['build']['config_overrides']: # skip overrides if mod == module_id and opt == option: override = True if override: continue value = config_parser.get(section,option) if option == 'shutit.core.module.allowed_images': value = json.loads(value) shutit.get_config(module_id, option, value, forcedefault=True) # ifneeded will (by default) only take effect if 'build' is not # specified. It can, however, be forced to a value, but this # should be unusual. if cfg[module_id]['shutit.core.module.build'] is None: shutit.get_config(module_id, 'shutit.core.module.build_ifneeded', True, boolean=True) cfg[module_id]['shutit.core.module.build'] = False else: shutit.get_config(module_id, 'shutit.core.module.build_ifneeded', False, boolean=True) def config_collection_for_built(shutit): """Collect configuration for modules that are being built. When this is called we should know what's being built (ie after dependency resolution). """ cfg = shutit.cfg shutit.log('In config_collection_for_built') cfg = shutit.cfg for module_id in module_ids(shutit): # Get the config even if installed or building (may be needed in other # hooks, eg test). if (is_to_be_built_or_is_installed(shutit, shutit.shutit_map[module_id]) and not shutit.shutit_map[module_id].get_config(shutit)): shutit.fail(module_id + ' failed on get_config') # Collect the build.cfg if we are building here. # If this file exists, process it. if cfg[module_id]['shutit.core.module.build']: module = shutit.shutit_map[module_id] cfg_file = os.path.dirname(module.__module_file) + '/configs/build.cnf' if os.path.isfile(cfg_file): # use shutit.get_config, forcing the passed-in default config_parser = ConfigParser.ConfigParser() config_parser.read(cfg_file) for section in config_parser.sections(): if section == module_id: for option in config_parser.options(section): override = False for mod, opt, val in cfg['build']['config_overrides']: # skip overrides if mod == module_id and opt == option: override = True if override: continue is_bool = (type(cfg[module_id][option]) == bool) if is_bool: value = config_parser.getboolean(section,option) else: value = config_parser.get(section,option) if option == 'shutit.core.module.allowed_images': value = json.loads(value) shutit.get_config(module_id, option, value, forcedefault=True) # Check the allowed_images against the base_image passed = True for module_id in module_ids(shutit): if (cfg[module_id]['shutit.core.module.build'] and (cfg[module_id]['shutit.core.module.allowed_images'] and cfg['target']['docker_image'] not in cfg[module_id]['shutit.core.module.allowed_images'])): if not allowed_image(shutit,module_id): passed = False print('\n\nWARNING!\n\nAllowed images for ' + module_id + ' are: ' + str(cfg[module_id]['shutit.core.module.allowed_images']) + ' but the configured image is: ' + cfg['target']['docker_image'] + '\n\nIs your shutit_module_path set correctly?' + '\n\nIf you want to ignore this, ' + 'pass in the --ignoreimage flag to shutit.\n\n') if not passed: if cfg['build']['imageerrorok']: # useful for test scripts print('Exiting on allowed images error, with return status 0') sys.exit(0) else: raise ShutItFailException('Allowed images checking failed') def allowed_image(shutit,module_id): """Given a module id and a shutit object, determine whether the image is allowed to be built. """ cfg = shutit.cfg shutit.log("In allowed_image: " + module_id) cfg = shutit.cfg if cfg['build']['ignoreimage']: shutit.log("ignoreimage == true, returning true" + module_id,force_stdout=True) return True shutit.log(str(cfg[module_id]['shutit.core.module.allowed_images'])) if cfg[module_id]['shutit.core.module.allowed_images']: # Try allowed images as regexps for regexp in cfg[module_id]['shutit.core.module.allowed_images']: if not shutit_util.check_regexp(regexp): shutit.fail('Illegal regexp found in allowed_images: ' + regexp) if re.match('^' + regexp + '$', cfg['target']['docker_image']): return True return False def conn_target(shutit): """Connect to the target. """ cfg = shutit.cfg conn_module = None cfg = shutit.cfg for mod in shutit.conn_modules: if mod.module_id == cfg['build']['conn_module']: conn_module = mod break if conn_module is None: shutit.fail('Couldn\'t find conn_module ' + cfg['build']['conn_module']) # Set up the target in pexpect. if cfg['build']['interactive'] >= 3: print('\nRunning the conn module (' + shutit.shutit_main_dir + '/shutit_setup.py)' + shutit_util.colour('32', '\n\n[Hit return to continue]\n')) shutit_util.util_raw_input(shutit=shutit) conn_module.get_config(shutit) conn_module.build(shutit) def finalize_target(shutit): """Finalize the target using the core finalize method. """ cfg = shutit.cfg shutit.pause_point('\nFinalizing the target module (' + shutit.shutit_main_dir + '/shutit_setup.py)', print_input=False, level=3) # Can assume conn_module exists at this point for mod in shutit.conn_modules: if mod.module_id == cfg['build']['conn_module']: conn_module = mod break conn_module.finalize(shutit) # Once we have all the modules, then we can look at dependencies. # Dependency validation begins. def resolve_dependencies(shutit, to_build, depender): """Add any required dependencies. """ shutit.log('In resolve_dependencies') cfg = shutit.cfg for dependee_id in depender.depends_on: dependee = shutit.shutit_map.get(dependee_id) # Don't care if module doesn't exist, we check this later if (dependee and dependee not in to_build and cfg[dependee_id]['shutit.core.module.build_ifneeded']): to_build.append(dependee) cfg[dependee_id]['shutit.core.module.build'] = True return True def check_dependee_exists(shutit, depender, dependee, dependee_id): """Checks whether a depended-on module is available. """ cfg = shutit.cfg # If the module id isn't there, there's a problem. if dependee == None: return ('module: \n\n' + dependee_id + '\n\nnot found in paths: ' + str(cfg['host']['shutit_module_path']) + ' but needed for ' + depender.module_id + '\nCheck your --shutit_module_path setting and ensure that ' + 'all modules configured to be built are in that path setting, ' + 'eg "--shutit_module_path /path/to/other/module/:."\n\n' + 'Also check that the module is configured to be built with ' + 'the correct module id in that module\'s configs/build.cnf file.' + '\n\nSee also help.') def check_dependee_build(shutit, depender, dependee, dependee_id): """Checks whether a depended on module is configured to be built. """ cfg = shutit.cfg # If depender is installed or will be installed, so must the dependee cfg = shutit.cfg if not (cfg[dependee.module_id]['shutit.core.module.build'] or is_to_be_built_or_is_installed(shutit,dependee)): return ('depender module id:\n\n[' + depender.module_id + ']\n\n' + 'is configured: "build:yes" or is already built ' + 'but dependee module_id:\n\n[' + dependee_id + ']\n\n' + 'is not configured: "build:yes"') def check_dependee_order(_shutit, depender, dependee, dependee_id): """Checks whether run orders are in the appropriate order. """ # If it depends on a module id, then the module id should be higher up # in the run order. if dependee.run_order > depender.run_order: return ('depender module id:\n\n' + depender.module_id + '\n\n(run order: ' + str(depender.run_order) + ') ' + 'depends on dependee module_id:\n\n' + dependee_id + '\n\n(run order: ' + str(dependee.run_order) + ') ' + 'but the latter is configured to run after the former') def make_dep_graph(depender): """Returns a digraph string fragment based on the passed-in module """ digraph = '' for dependee_id in depender.depends_on: digraph = (digraph + '"' + depender.module_id + '"->"' + dependee_id + '";\n') return digraph def check_deps(shutit): """Dependency checking phase is performed in this method. """ cfg = shutit.cfg shutit.log('PHASE: dependencies', code='32') shutit.pause_point('\nNow checking for dependencies between modules', print_input=False, level=3) # Get modules we're going to build to_build = [ shutit.shutit_map[module_id] for module_id in shutit.shutit_map if module_id in cfg and cfg[module_id]['shutit.core.module.build'] ] # Add any deps we may need by extending to_build and altering cfg for module in to_build: resolve_dependencies(shutit, to_build, module) # Dep checking def err_checker(errs, triples): """Collate error information. """ new_triples = [] for err, triple in zip(errs, triples): if not err: new_triples.append(triple) continue found_errs.append(err) return new_triples found_errs = [] triples = [] for depender in to_build: for dependee_id in depender.depends_on: triples.append((depender, shutit.shutit_map.get(dependee_id), dependee_id)) triples = err_checker([ check_dependee_exists(shutit, depender, dependee, dependee_id) for depender, dependee, dependee_id in triples ], triples) triples = err_checker([ check_dependee_build(shutit, depender, dependee, dependee_id) for depender, dependee, dependee_id in triples ], triples) triples = err_checker([ check_dependee_order(shutit, depender, dependee, dependee_id) for depender, dependee, dependee_id in triples ], triples) if found_errs: return [(err,) for err in found_errs] if cfg['build']['debug']: shutit.log('Modules configured to be built (in order) are: ', code='32') for module_id in module_ids(shutit): module = shutit.shutit_map[module_id] if cfg[module_id]['shutit.core.module.build']: shutit.log(module_id + ' ' + str(module.run_order), code='32') shutit.log('\n', code='32') return [] def check_conflicts(shutit): """Checks for any conflicts between modules configured to be built. """ cfg = shutit.cfg # Now consider conflicts shutit.log('PHASE: conflicts', code='32') errs = [] shutit.pause_point('\nNow checking for conflicts between modules', print_input=False, level=3) for module_id in module_ids(shutit): if not cfg[module_id]['shutit.core.module.build']: continue conflicter = shutit.shutit_map[module_id] for conflictee in conflicter.conflicts_with: # If the module id isn't there, there's no problem. conflictee_obj = shutit.shutit_map.get(conflictee) if conflictee_obj == None: continue if ((cfg[conflicter.module_id]['shutit.core.module.build'] or is_to_be_built_or_is_installed(shutit,conflicter)) and (cfg[conflictee_obj.module_id]['shutit.core.module.build'] or is_to_be_built_or_is_installed(shutit,conflictee_obj))): errs.append(('conflicter module id: ' + conflicter.module_id + ' is configured to be built or is already built but ' + 'conflicts with module_id: ' + conflictee_obj.module_id,)) return errs def check_ready(shutit, throw_error=True): """Check that all modules are ready to be built, calling check_ready on each of those configured to be built and not already installed (see is_installed). """ cfg = shutit.cfg shutit.log('PHASE: check_ready', code='32') errs = [] shutit.pause_point('\nNow checking whether we are ready to build modules' + ' configured to be built', print_input=False, level=3) # Find out who we are to see whether we need to log in and out or not. for module_id in module_ids(shutit): module = shutit.shutit_map[module_id] shutit.log('considering check_ready (is it ready to be built?): ' + module_id, code='32') if cfg[module_id]['shutit.core.module.build'] and module.module_id not in cfg['environment'][cfg['build']['current_environment_id']]['modules_ready'] and not is_installed(shutit,module): shutit.log('checking whether module is ready to build: ' + module_id, code='32') shutit.login(prompt_prefix=module_id,command='bash') # Move to the directory so context is correct (eg for checking for # the existence of files needed for build) revert_dir = os.getcwd() cfg['environment'][cfg['build']['current_environment_id']]['module_root_dir'] = os.path.dirname(module.__module_file) shutit.chdir(cfg['environment'][cfg['build']['current_environment_id']]['module_root_dir']) if not is_ready(shutit, module) and throw_error: errs.append((module_id + ' not ready to install.\nRead the ' + 'check_ready function in the module,\nor log ' + 'messages above to determine the issue.\n\n', shutit.pexpect_children['target_child'])) shutit.logout() shutit.chdir(revert_dir) return errs def do_remove(shutit): """Remove modules by calling remove method on those configured for removal. """ cfg = shutit.cfg # Now get the run_order keys in order and go. shutit.log('PHASE: remove', code='32') shutit.pause_point('\nNow removing any modules that need removing', print_input=False, level=3) # Login at least once to get the exports. for module_id in module_ids(shutit): module = shutit.shutit_map[module_id] shutit.log('considering whether to remove: ' + module_id, code='32') if cfg[module_id]['shutit.core.module.remove']: shutit.log('removing: ' + module_id, code='32') shutit.login(prompt_prefix=module_id,command='bash') if not module.remove(shutit): shutit.log(print_modules(shutit), code='31') shutit.fail(module_id + ' failed on remove', child=shutit.pexpect_children['target_child']) else: if cfg['build']['delivery'] in ('docker','dockerfile'): # Create a directory and files to indicate this has been removed. shutit.send(' mkdir -p ' + cfg['build']['build_db_dir'] + '/module_record/' + module.module_id + ' && rm -f ' + cfg['build']['build_db_dir'] + '/module_record/' + module.module_id + '/built && touch ' + cfg['build']['build_db_dir'] + '/module_record/' + module.module_id + '/removed') # Remove from "installed" cache if module.module_id in cfg['environment'][cfg['build']['current_environment_id']]['modules_installed']: cfg['environment'][cfg['build']['current_environment_id']]['modules_installed'].remove(module.module_id) # Add to "not installed" cache cfg['environment'][cfg['build']['current_environment_id']]['modules_not_installed'].append(module.module_id) shutit.logout() def build_module(shutit, module): """Build passed-in module. """ cfg = shutit.cfg shutit.log('building: ' + module.module_id + ' with run order: ' + str(module.run_order), code='32') cfg['build']['report'] = (cfg['build']['report'] + '\nBuilding: ' + module.module_id + ' with run order: ' + str(module.run_order)) if not module.build(shutit): shutit.fail(module.module_id + ' failed on build', child=shutit.pexpect_children['target_child']) else: if cfg['build']['delivery'] in ('docker','dockerfile'): # Create a directory and files to indicate this has been built. shutit.send(' mkdir -p ' + cfg['build']['build_db_dir'] + '/module_record/' + module.module_id + ' && touch ' + cfg['build']['build_db_dir'] + '/module_record/' + module.module_id + '/built && rm -f ' + cfg['build']['build_db_dir'] + '/module_record/' + module.module_id + '/removed') # Put it into "installed" cache cfg['environment'][cfg['build']['current_environment_id']]['modules_installed'].append(module.module_id) # Remove from "not installed" cache if module.module_id in cfg['environment'][cfg['build']['current_environment_id']]['modules_not_installed']: cfg['environment'][cfg['build']['current_environment_id']]['modules_not_installed'].remove(module.module_id) shutit.pause_point('\nPausing to allow inspect of build for: ' + module.module_id, print_input=True, level=2) cfg['build']['report'] = (cfg['build']['report'] + '\nCompleted module: ' + module.module_id) if cfg[module.module_id]['shutit.core.module.tag'] or cfg['build']['interactive'] >= 3: shutit.log(shutit_util.build_report(shutit, '#Module:' + module.module_id), code='32') if (not cfg[module.module_id]['shutit.core.module.tag'] and cfg['build']['interactive'] >= 2): shutit.log("\n\nDo you want to save state now we\'re at the " + "end of this module? (" + module.module_id + ") (input y/n)", force_stdout=True, code='32') cfg[module.module_id]['shutit.core.module.tag'] = (shutit_util.util_raw_input(shutit=shutit,default='y') == 'y') if cfg[module.module_id]['shutit.core.module.tag'] or cfg['build']['tag_modules']: shutit.log(module.module_id + ' configured to be tagged, doing repository work', force_stdout=True) # Stop all before we tag to avoid file changing errors, # and clean up pid files etc.. stop_all(shutit, module.run_order) shutit.do_repository_work(str(module.module_id) + '_' + str(module.run_order), password=cfg['host']['password'], docker_executable=cfg['host']['docker_executable'], force=True) # Start all after we tag to ensure services are up as expected. start_all(shutit, module.run_order) if cfg['build']['interactive'] >= 2: shutit.log("\n\nDo you want to stop interactive mode? (input y/n)\n", force_stdout=True,code='32') if shutit_util.util_raw_input(shutit=shutit,default='y') == 'y': cfg['build']['interactive'] = 0 def do_build(shutit): """Runs build phase, building any modules that we've determined need building. """ cfg = shutit.cfg shutit.log('PHASE: build, repository work', code='32') shutit.log(shutit_util.print_config(cfg)) if cfg['build']['interactive'] >= 3: print ('\nNow building any modules that need building' + shutit_util.colour('32', '\n\n[Hit return to continue]\n')) shutit_util.util_raw_input(shutit=shutit) module_id_list = module_ids(shutit) if cfg['build']['deps_only']: module_id_list_build_only = filter(lambda x: cfg[x]['shutit.core.module.build'], module_id_list) for module_id in module_id_list: module = shutit.shutit_map[module_id] shutit.log('considering whether to build: ' + module.module_id, code='32') if cfg[module.module_id]['shutit.core.module.build']: if cfg['build']['delivery'] not in module.ok_delivery_methods: shutit.fail('Module: ' + module.module_id + ' can only be built with one of these --delivery methods: ' + str(module.ok_delivery_methods) + '\nSee shutit build -h for more info, or try adding: --delivery to your shutit invocation') if is_installed(shutit,module): cfg['build']['report'] = (cfg['build']['report'] + '\nBuilt already: ' + module.module_id + ' with run order: ' + str(module.run_order)) else: # We move to the module directory to perform the build, returning immediately afterwards. if cfg['build']['deps_only'] and module_id == module_id_list_build_only[-1]: # If this is the last module, and we are only building deps, stop here. cfg['build']['report'] = (cfg['build']['report'] + '\nSkipping: ' + module.module_id + ' with run order: ' + str(module.run_order) + '\n\tas this is the final module and we are building dependencies only') else: revert_dir = os.getcwd() cfg['environment'][cfg['build']['current_environment_id']]['module_root_dir'] = os.path.dirname(module.__module_file) shutit.chdir(cfg['environment'][cfg['build']['current_environment_id']]['module_root_dir']) shutit.login(prompt_prefix=module_id,command='bash') build_module(shutit, module) shutit.logout() shutit.chdir(revert_dir) if is_installed(shutit, module): shutit.log('Starting module') if not module.start(shutit): shutit.fail(module.module_id + ' failed on start', child=shutit.pexpect_children['target_child']) def do_test(shutit): """Runs test phase, erroring if any return false. """ cfg = shutit.cfg if not cfg['build']['dotest']: shutit.log('Tests configured off, not running') return # Test in reverse order shutit.log('PHASE: test', code='32') if cfg['build']['interactive'] >= 3: print '\nNow doing test phase' + shutit_util.colour('32', '\n\n[Hit return to continue]\n') shutit_util.util_raw_input(shutit=shutit) stop_all(shutit) start_all(shutit) for module_id in module_ids(shutit, rev=True): module = shutit.shutit_map[module_id] # Only test if it's installed. if is_installed(shutit, shutit.shutit_map[module_id]): shutit.log('RUNNING TEST ON: ' + module_id, code='32') shutit.login(prompt_prefix=module_id,command='bash') if not shutit.shutit_map[module_id].test(shutit): shutit.fail(module_id + ' failed on test', child=shutit.pexpect_children['target_child']) shutit.logout() def do_finalize(shutit): """Runs finalize phase; run after all builds are complete and all modules have been stopped. """ cfg = shutit.cfg # Stop all the modules if cfg['build']['interactive'] >= 3: print('\nStopping all modules before finalize phase' + shutit_util.colour('32', '\n\n[Hit return to continue]\n')) shutit_util.util_raw_input(shutit=shutit) stop_all(shutit) # Finalize in reverse order shutit.log('PHASE: finalize', code='32') if cfg['build']['interactive'] >= 3: print('\nNow doing finalize phase, which we do when all builds are ' + 'complete and modules are stopped' + shutit_util.colour('32', '\n\n[Hit return to continue]\n')) shutit_util.util_raw_input(shutit=shutit) # Login at least once to get the exports. for module_id in module_ids(shutit, rev=True): # Only finalize if it's thought to be installed. if is_installed(shutit, shutit.shutit_map[module_id]): shutit.login(prompt_prefix=module_id,command='bash') if not shutit.shutit_map[module_id].finalize(shutit): shutit.fail(module_id + ' failed on finalize', child=shutit.pexpect_children['target_child']) shutit.logout() def setup_shutit_path(cfg): # try the current directory, the .. directory, or the ../shutit directory, the ~/shutit if not cfg['host']['add_shutit_to_path']: return res = shutit_util.util_raw_input(prompt='shutit appears not to be on your path - should try and we find it and add it to your ~/.bashrc (Y/n)?') if res in ['n','N']: with open(os.path.join(cfg['shutit_home'], 'config'), 'a') as f: f.write('\n[host]\nadd_shutit_to_path: no\n') return path_to_shutit = '' for d in ['.','..','~','~/shutit']: path = os.path.abspath(d + '/shutit') if not os.path.isfile(path): continue path_to_shutit = path while path_to_shutit == '': d = shutit_util.util_raw_input(prompt='cannot auto-find shutit - please input the path to your shutit dir\n') path = os.path.abspath(d + '/shutit') if not os.path.isfile(path): continue path_to_shutit = path if path_to_shutit != '': bashrc = os.path.expanduser('~/.bashrc') with open(bashrc, "a") as myfile: #http://unix.stackexchange.com/questions/26676/how-to-check-if-a-shell-is-login-interactive-batch myfile.write('\nexport PATH="$PATH:' + os.path.dirname(path_to_shutit) + '"\n') shutit_util.util_raw_input(prompt='\nPath set up - please open new terminal and re-run command\n') sys.exit() def main(): """Main ShutIt function. Handles the configured actions: - skeleton - create skeleton module - serve - run as a server - list_configs - output computed configuration - depgraph - output digraph of module dependencies """ if sys.version_info.major == 2: if sys.version_info.minor < 7: shutit_global.shutit.fail('Python version must be 2.7+') shutit = shutit_global.shutit cfg = shutit.cfg shutit_util.parse_args(shutit) if cfg['action']['skeleton']: shutit_util.create_skeleton(shutit) cfg['build']['completed'] = True return if cfg['action']['serve']: import shutit_srv cfg['build']['interactive'] = 0 revert_dir = os.getcwd() os.chdir(sys.path[0]) shutit_srv.start() os.chdir(revert_dir) return shutit_util.load_configs(shutit) # Try and ensure shutit is on the path - makes onboarding easier # Only do this if we're in a terminal if shutit_util.determine_interactive() and spawn.find_executable('shutit') is None: setup_shutit_path(cfg) shutit_util.load_mod_from_file(shutit, os.path.join(shutit.shutit_main_dir, 'shutit_setup.py')) shutit_util.load_shutit_modules(shutit) if cfg['action']['list_modules']: shutit_util.list_modules(shutit) sys.exit(0) init_shutit_map(shutit) config_collection(shutit) conn_target(shutit) errs = [] errs.extend(check_deps(shutit)) if cfg['action']['list_deps']: # Show dependency graph digraph = 'digraph depgraph {\n' digraph = digraph + '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() if module_id in cfg and cfg[module_id]['shutit.core.module.build'] ]) digraph = digraph + '\n}' f = file(cfg['build']['log_config_path'] + '/digraph.txt','w') f.write(digraph) f.close() digraph_all = 'digraph depgraph {\n' digraph_all = digraph_all + '\n'.join([ make_dep_graph(module) for module_id, module in shutit.shutit_map.items() ]) digraph_all = digraph_all + '\n}' f = file(cfg['build']['log_config_path'] + '/digraph_all.txt','w') f.write(digraph_all) f.close() shutit.log('\n================================================================================\n' + digraph_all, force_stdout=True) shutit.log('\nAbove is the digraph for all modules seen in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m mylibrary | dot -Tpng -o depgraph.png\n', force_stdout=True) shutit.log('\n================================================================================\n', force_stdout=True) shutit.log('\n\n' + digraph, force_stdout=True) shutit.log('\n================================================================================\n' + digraph, force_stdout=True) shutit.log('\nAbove is the digraph for all modules configured to be built in this shutit invocation. Use graphviz to render into an image, eg\n\n\tshutit depgraph -m mylibrary | dot -Tpng -o depgraph.png\n', force_stdout=True) shutit.log('\n================================================================================\n', force_stdout=True) # Exit now sys.exit(0) # Dependency validation done, now collect configs of those marked for build. config_collection_for_built(shutit) if False and (cfg['action']['list_configs'] or cfg['build']['debug']): shutit.log(shutit_util.print_config(cfg, history=cfg['list_configs']['cfghistory']), force_stdout=True) # Set build completed cfg['build']['completed'] = True f = file(cfg['build']['log_config_path'] + '/cfg.txt','w') f.write(shutit_util.print_config(cfg, history=cfg['list_configs']['cfghistory'])) f.close() shutit.log('================================================================================', force_stdout=True) shutit.log('Config details placed in: ' + cfg['build']['log_config_path'], force_stdout=True) shutit.log('================================================================================', force_stdout=True) shutit.log('To render the digraph of this build into an image run eg:\n\ndot -Tgv -o ' + cfg['build']['log_config_path'] + '/digraph.gv ' + cfg['build']['log_config_path'] + '/digraph.txt && dot -Tpdf -o digraph.pdf ' + cfg['build']['log_config_path'] + '/digraph.gv\n\n', force_stdout=True) shutit.log('================================================================================', force_stdout=True) shutit.log('To render the digraph of all visible modules into an image, run eg:\n\ndot -Tgv -o ' + cfg['build']['log_config_path'] + '/digraph_all.gv ' + cfg['build']['log_config_path'] + '/digraph_all.txt && dot -Tpdf -o digraph_all.pdf ' + cfg['build']['log_config_path'] + '/digraph_all.gv\n\n', force_stdout=True) shutit.log('================================================================================', force_stdout=True) shutit.log('\nConfiguration details have been written to the folder: ' + cfg['build']['log_config_path'] + '\n', force_stdout=True) shutit.log('================================================================================', force_stdout=True) if cfg['action']['list_configs']: return # Check for conflicts now. errs.extend(check_conflicts(shutit)) # Cache the results of check_ready at the start. errs.extend(check_ready(shutit, throw_error=False)) if errs: shutit.log(print_modules(shutit), code='31') child = None for err in errs: shutit.log(err[0], force_stdout=True, code='31') if not child and len(err) > 1: child = err[1] shutit.fail("Encountered some errors, quitting", child=child) shutit.record_config() do_remove(shutit) do_build(shutit) do_test(shutit) do_finalize(shutit) finalize_target(shutit) shutit.log(shutit_util.build_report(shutit, '#Module: N/A (END)'), prefix=False, force_stdout=True, code='32') if cfg['build']['build_log']: cfg['build']['report_final_messages'] += "Build log file: " + cfg['host']['logfile'] # Show final report messages (ie messages to show after standard report). if cfg['build']['report_final_messages'] != '': shutit.log(cfg['build']['report_final_messages'], prefix=False, force_stdout=True, code='31') if cfg['build']['interactive'] >= 3: shutit.log('\n' + 'The build is complete. You should now have a target ' + 'called ' + cfg['target']['name'] + ' and a new image if you chose to commit it.\n\n' + 'Look and play with the following files from the newly-created ' + 'module directory to dig deeper:\n\n configs/build.cnf\n ' + '*.py\n\nYou can rebuild at any time by running the supplied ' + './build.sh and run with the supplied ./run.sh. These may need ' + 'tweaking for your particular environment, eg sudo\n\n' + 'You can inspect the details of the build in the target image\'s ' + cfg['build']['build_db_dir'] + ' directory.', force_stdout=True, code='32') # Mark the build as completed cfg['build']['completed'] = True def do_phone_home(msg=None,question='Error seen - would you like to inform the maintainers?'): """Report message home. msg - message to send home question - question to ask - assumes Y/y for send message, else no """ cfg = shutit.cfg if msg is None: msg = {} if shutit_global.cfg['build']['interactive'] == 0: return msg.update({'shutitrunstatus':'fail','pwd':os.getcwd(),'user':os.environ.get('LOGNAME', '')}) if question != '' and shutit_util.util_raw_input(prompt=question + ' (Y/n)\n') not in ('y','Y',''): return try: urllib.urlopen("http://shutit.tk?" + urllib.urlencode(msg)) except Exception as e: shutit_global.shutit.log('failed to send message: ' + str(e.message)) signal.signal(signal.SIGINT, shutit_util.ctrl_c_signal_handler) if __name__ == '__main__': main() PKUG3 package_map.py"""Stores known package maps for different distributions. """ #The MIT License (MIT) # #Copyright (C) 2014 OpenBet Limited # #Permission is hereby granted, free of charge, to any person obtaining a copy of #this software and associated documentation files (the "Software"), to deal in #the Software without restriction, including without limitation the rights to #use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies #of the Software, and to permit persons to whom the Software is furnished to do #so, subject to the following conditions: # #The above copyright notice and this permission notice shall be included in all #copies or substantial portions of the Software. # #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR #IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, #ITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL #THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER #LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE #SOFTWARE. # Structured by package, then another dict with # install_type -> mapped package inside that. # The keys are then the canonical package names. PACKAGE_MAP = { 'apache2': {'apt':'apache2', 'yum':'httpd'}, 'adduser': {'apt':'adduser', 'yum':''}, 'php5': {'apt':'php5', 'yum':'php'}, 'ruby-dev': {'apt':'ruby-dev', 'yum':'ruby-devel', 'brew':'ruby-build'}, 'git': {'emerge':'dev-vcs/git'}, 'vagrant': {'brew':'Caskroom/cask/vagrant'}, 'virtualbox': {'brew':'Caskroom/cask/virtualbox'}, 'build-essential': {'brew':'gcc', 'yum':'gcc make'}, 'sudo': {'brew':''}, } def map_packages(package_str, install_type): res = '' for package in package_str.split(): res = res + ' ' + map_package(package,install_type) return res def map_package(package, install_type): """If package mapping exists, then return it, else return package. """ if package in PACKAGE_MAP.keys(): for itype in PACKAGE_MAP[package].keys(): if itype == install_type: return PACKAGE_MAP[package][install_type] # Otherwise, simply return package return package def find_package(sought_package): """Is this name mentioned anywhere? Then return it as a suggestion? """ # First check top-level keys if sought_package in PACKAGE_MAP.keys(): return PACKAGE_MAP[sought_package] for package in PACKAGE_MAP.keys(): for install_type in PACKAGE_MAP[package].keys(): if sought_package == PACKAGE_MAP[package][install_type]: return {package:PACKAGE_MAP[package]} return None PK'F~"~" shutit_srv.py""" """ #The MIT License (MIT) # #Copyright (C) 2014 OpenBet Limited # #Permission is hereby granted, free of charge, to any person obtaining a copy of #this software and associated documentation files (the "Software"), to deal in #the Software without restriction, including without limitation the rights to #use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies #of the Software, and to permit persons to whom the Software is furnished to do #so, subject to the following conditions: # #The above copyright notice and this permission notice shall be included in all #copies or substantial portions of the Software. # #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR #IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, #ITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL #THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER #LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE #SOFTWARE. import os import json import copy import threading import StringIO import subprocess import bottle from bottle import route, request, response, static_file import shutit_main import shutit_global from shutit_module import ShutItException import shutit_util ORIG_MOD_CFG = None shutit = None STATUS = None def build_shutit(): """Performs ShutIt remove/build/test/finalize cycle. """ global STATUS try: shutit_main.do_remove(shutit) shutit_main.do_build(shutit) shutit_main.do_test(shutit) shutit_main.do_finalize(shutit) shutit_main.finalize_target(shutit) except ShutItException as error: STATUS['errs'] = [str(error)] STATUS['build_done'] = True def update_modules(to_build, cfg): """ Updates modules to be built with the passed-in config. Updating each individual module section will propogate the changes to STATUS as well (as the references are the same). Note that we have to apply overrides in a specific order - 1) reset the modules being built to the defaults 2) set any modules selected for build as build = True 3) reset configs using config_collection_for_built 4) apply config overrides """ global STATUS selected = set(to_build) for module_id in shutit.cfg: if module_id in ORIG_MOD_CFG and 'shutit.core.module.build' in ORIG_MOD_CFG[module_id]: shutit.cfg[module_id]['shutit.core.module.build'] = ORIG_MOD_CFG[module_id]['shutit.core.module.build'] if module_id in selected: shutit.cfg[module_id]['shutit.core.module.build'] = True if cfg is not None: sec, key, val = cfg ORIG_MOD_CFG[sec][key] = val for module_id in ORIG_MOD_CFG: for cfgkey in ORIG_MOD_CFG[module_id]: if cfgkey == 'shutit.core.module.build': continue shutit.cfg[module_id][cfgkey] = ORIG_MOD_CFG[module_id][cfgkey] errs = [] errs.extend(shutit_main.check_deps(shutit)) # There is a complexity here in that module configs may depend on # configs from other modules (!). We assume this won't happen as we # would have to override each module at the correct time. shutit_main.config_collection_for_built(shutit) errs.extend(shutit_main.check_conflicts(shutit)) # Cache first errs.extend(shutit_main.check_ready(shutit, throw_error=False)) errs.extend(shutit_main.check_ready(shutit)) STATUS['errs'] = [err[0] for err in errs] STATUS['modules'] = [ { "module_id": module_id, "description": shutit.shutit_map[module_id].description, "run_order": float(shutit.shutit_map[module_id].run_order), "build": shutit.cfg[module_id]['shutit.core.module.build'], "selected": module_id in selected } for module_id in shutit_main.allowed_module_ids(shutit) ] @route('/info', method='POST') def info(): """Handles requests to build, reset, defaulting to returning current STATUS if none of these are requested. """ can_check = not (STATUS['build_started'] or STATUS['resetting']) can_build = not (STATUS['build_started'] or STATUS['resetting']) can_reset = not ((STATUS['build_started'] and not STATUS['build_done']) or STATUS['resetting']) if can_check and 'to_build' in request.json and 'cfg' in request.json: update_modules(request.json['to_build'], request.json['cfg']) if can_build and 'build' in request.json and len(STATUS['errs']) == 0: STATUS["build_started"] = True thread = threading.Thread(target=build_shutit) thread.daemon = True thread.start() if can_reset and 'reset' in request.json: shutit_reset() return json.dumps(STATUS) @route('/log', method='POST') def log(): """Returns log information to the client. """ cmd_offset, log_offset = request.json if STATUS['resetting']: command_list = [] log = '' else: command_list = shutit.shutit_command_history[cmd_offset:] log = shutit.cfg['build']['build_log_file'].getvalue()[log_offset:] # Working around issues with encoding in a cowardly way. return json.dumps({ "cmds": command_list, "logs": log }, encoding='iso8859-1') @route('/') def index(): """Serves the main file. """ return static_file('index.html', root='./web') @route('/static/.js') def static_srv(path): """Serves a static file. """ return static_file(path + '.js', root='./web') @route('/export/.tar.gz') def export_srv(cid): # TODO: this blocks all other requests global shutit docker = shutit.cfg['host']['docker_executable'].split(' ') export_cmd = docker + ['export', cid] # TODO: check for error on stderr for both the below commands tar_export = subprocess.Popen(export_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) gzip_export = subprocess.Popen(['gzip', '--stdout'], stdin=tar_export.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) response.content_type = 'application/x-gzip' response.set_header('Content-disposition', 'attachment') if 'wsgi.file_wrapper' in request.environ: return request.environ['wsgi.file_wrapper'](gzip_export.stdout) else: return iter(lambda: gzip_export.stdout.read(1024*1024), '') def shutit_reset(): """Handles a ShutIt reset, clearing the STATUS global. """ global ORIG_MOD_CFG global shutit global STATUS ORIG_MOD_CFG = {} if shutit is not None: for child in shutit.pexpect_children.values(): # Try to clean up the old children... child.send('\n') child.sendeof() child.readlines() image_tag = shutit.cfg['target']['docker_image'] else: image_tag = '' shutit = None STATUS = { 'build_done': False, 'build_started': False, 'resetting': True, 'modules': [], 'errs': [], 'cid': '', 'cfg': {}, 'image_tag': image_tag } def reset_thread(): global ORIG_MOD_CFG global shutit # Start with a fresh shutit object shutit = shutit_global.shutit = shutit_global.init() # This has already happened but we have to do it again on top of our new # shutit object shutit_util.parse_args(shutit) shutit.cfg['build']['interactive'] = 0 # The rest of the loading from shutit_main shutit_util.load_configs(shutit) shutit_util.load_mod_from_file(shutit, os.path.join(shutit.shutit_main_dir, 'shutit_setup.py')) shutit_util.load_shutit_modules(shutit) shutit_main.init_shutit_map(shutit) shutit_main.config_collection(shutit) # Here we can set the starting image. if STATUS['image_tag'] != '': shutit.cfg['target']['docker_image'] = STATUS['image_tag'] else: STATUS['image_tag'] = shutit.cfg['target']['docker_image'] shutit_main.conn_target(shutit) shutit_main.config_collection_for_built(shutit) # Some hacks for server mode shutit.cfg['build']['build_log_file'] = StringIO.StringIO() shutit.cfg['build']['interactive'] = 0 STATUS['cid'] = shutit.cfg['target']['container_id'] for module_id in shutit.shutit_map: ORIG_MOD_CFG[module_id] = STATUS['cfg'][module_id] = shutit.cfg[module_id] # Add in core sections for module_id in ['repository', 'target']: ORIG_MOD_CFG[module_id] = STATUS['cfg'][module_id] = shutit.cfg[module_id] # Make sure that ORIG_MOD_CFG can be updated seperately to # STATUS and shutit.cfg (which remain linked), as it will hold # our overrides ORIG_MOD_CFG = copy.deepcopy(ORIG_MOD_CFG) update_modules([], None) STATUS['resetting'] = False thread = threading.Thread(target=reset_thread) thread.daemon = True thread.start() def start(): """ Start the ShutIt server. Environment variables: SHUTIT_HOST - hostname for the server (default: localhost) SHUTIT_PORT - port for the server (default: 8080) """ shutit_reset() # Start the server host = os.environ.get('SHUTIT_HOST', '0.0.0.0') port = int(os.environ.get('SHUTIT_PORT', 8080)) bottle.debug(True) bottle.run(host=host, port=port) if __name__ == '__main__': print "PLEASE START VIA SHUTIT_MAIN INSTEAD" PKƓrGq:  shutit_global.py"""Contains all the core ShutIt methods and functionality. """ #The MIT License (MIT) # #Copyright (C) 2014 OpenBet Limited # #Permission is hereby granted, free of charge, to any person obtaining a copy of #this software and associated documentation files (the "Software"), to deal in #the Software without restriction, including without limitation the rights to #use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies #of the Software, and to permit persons to whom the Software is furnished to do #so, subject to the following conditions: # #The above copyright notice and this permission notice shall be included in all #copies or substantial portions of the Software. # #THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR #IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, #ITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL #THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER #LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, #OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE #SOFTWARE. import sys import os import shutil import socket import time import shutit_util import string import re import textwrap import base64 import getpass import package_map import datetime import pexpect from shutit_module import ShutItFailException class ShutIt(object): """ShutIt build class. Represents an instance of a ShutIt build with associated config. """ def __init__(self, **kwargs): """Constructor. Sets up: - pexpect_children - pexpect objects representing shell interactions - shutit_modules - representation of loaded shutit modules - shutit_main_dir - directory in which shutit is located - cfg - dictionary of configuration of build - cwd - working directory of build - shutit_map - maps module_ids to module objects """ # These used to be in shutit_global, so we pass them in as args so # the original reference can be put in shutit_global self.pexpect_children = kwargs['pexpect_children'] self.shutit_modules = kwargs['shutit_modules'] self.shutit_main_dir = kwargs['shutit_main_dir'] self.cfg = kwargs['cfg'] self.cwd = kwargs['cwd'] self.shutit_command_history = kwargs['shutit_command_history'] self.shutit_map = kwargs['shutit_map'] # These are new members we dont have to provide compaitibility for self.conn_modules = set() # Hidden attributes self._default_child = [None] self._default_expect = [None] self._default_check_exit = [None] def module_method_start(self): """Gets called automatically by the metaclass decorator in shutit_module when a module method is called. This allows setting defaults for the 'scope' of a method. """ if self._default_child[-1] is not None: self._default_child.append(self._default_child[-1]) if self._default_expect[-1] is not None: self._default_expect.append(self._default_expect[-1]) if self._default_check_exit[-1] is not None: self._default_check_exit.append(self._default_check_exit[-1]) def module_method_end(self): """Gets called automatically by the metaclass decorator in shutit_module when a module method is finished. This allows setting defaults for the 'scope' of a method. """ if len(self._default_child) != 1: self._default_child.pop() if len(self._default_expect) != 1: self._default_expect.pop() self._default_check_exit.pop() def get_default_child(self): """Returns the currently-set default pexpect child. @return: default pexpect child object """ if self._default_child == None: print 'Default child not set yet, exiting' sys.exit(1) if self._default_child[-1] is None: print '''Couldn't get default child''' sys.exit(1) return self._default_child[-1] def get_default_expect(self): """Returns the currently-set default pexpect string (usually a prompt). @return: default pexpect string """ if self._default_expect[-1] is None: self.fail("Couldn't get default expect, quitting") return self._default_expect[-1] def get_default_check_exit(self): """Returns default value of check_exit. See send method. @rtype: boolean @return: Default check_exit value """ if self._default_check_exit[-1] is None: self.fail("Couldn't get default check exit") return self._default_check_exit[-1] def set_default_child(self, child): """Sets the default pexpect child. @param child: pexpect child to set as default """ self._default_child[-1] = child def set_default_expect(self, expect=None, check_exit=True): """Sets the default pexpect string (usually a prompt). Defaults to the configured root prompt if no argument is passed. @param expect: String to expect in the output @type expect: string @param check_exit: Whether to check the exit value of the command @type check_exit: boolean """ cfg = self.cfg if expect == None: expect = cfg['expect_prompts']['root'] self._default_expect[-1] = expect self._default_check_exit[-1] = check_exit def fail(self, msg, child=None, throw_exception=False): """Handles a failure, pausing if a pexpect child object is passed in. @param child: pexpect child to work on @param throw_exception: Whether to throw an exception. @type throw_exception: boolean """ # Note: we must not default to a child here if child is not None: self.pause_point('Pause point on fail: ' + msg, child=child, colour='31') print >> sys.stderr, 'Error caught: ' + msg print >> sys.stderr if throw_exception: if shutit_util.determine_interactive(self): self.pause_point('FAIL: ' + msg) else: raise ShutItFailException(msg) else: # This is an "OK" failure, ie we don't need to throw an exception. # However, it's still a failure, so return 1 if shutit_util.determine_interactive(self): self.pause_point('FAIL: ' + msg) else: print msg print 'Error seen, exiting with status 1' sys.exit(1) def log(self, msg, code=None, pause=0, prefix=True, force_stdout=False, add_final_message=False): """Logging function. @param code: Colour code for logging. Ignored if we are in serve mode @param pause: Length of time to pause after logging @param prefix: Whether to output logging prefix (LOG: