PK}sIإ_hostlist/__init__.py#!/usr/bin/env python3 "Sync hostlist and builds config files for services." from hostlist import buildfiles __version__ = '1.0.1' if __name__ == "__main__": buildfiles.main() PK~sI[^""hostlist/addhost.py#!/usr/bin/env python3 import os.path import logging import subprocess import argparse from distutils.util import strtobool import sys from . import hostlist def yes_no_query(question, empty_means=None): if empty_means is None: choices = '[y/n]' elif empty_means is True: choices = '[Y/n]' elif empty_means is False: choices = '[y/N]' sys.stdout.write('%s %s' % (question, choices)) while True: inp = input() if inp.strip() == '': if empty_means is not None: return empty_means else: try: return strtobool(inp) except ValueError: print('Please respond with \'y\' or \'n\'.') def get_next_ip(ipstart, ipend, existing_ips): newip = ipstart while newip <= ipend: if newip not in existing_ips: return newip newip += 1 return None def parse_args(): parser = argparse.ArgumentParser(description="Add a new host to hostlist.") parser.add_argument('hostname', help='New hostname, e.g. fullname.itp.kit.edu, ttpmyfriend, itpalbatros98, newmachine.particle') parser.add_argument('hostlist', help='Filename to add to, e.g. desktops.itp.' ' The prefix "hostlists/" is optional.') parser.add_argument('--ip', help='Give IP to use.') parser.add_argument('--mac', help='Give mac to use.') parser.add_argument('--verbose', '-v', action='count', help='more output', default=0) args = parser.parse_args() logging.basicConfig(format='%(levelname)s:%(message)s') logging.getLogger().setLevel(logging.WARNING) if args.verbose == 1: logging.getLogger().setLevel(logging.INFO) elif args.verbose >= 2: logging.getLogger().setLevel(logging.DEBUG) return args def clean_args(args): args = clean_hostlist(args) args = clean_hostname(args) logging.debug("Using hostname %s and hostlist %s." % (args.hostname, args.hostlist)) return args def clean_hostlist(args): """makes sure the hostlist file exists also set institute based on hostlist file """ if not os.path.isfile(args.hostlist): newhostlist = './hostlists/' + args.hostlist if not os.path.isfile(newhostlist): logging.error("Neiter %s nor %s is a file." % (args.hostlist, newhostlist)) sys.exit(10) else: logging.info("Using %s as hostlist." % newhostlist) args.hostlist = newhostlist args.institute = os.path.splitext(os.path.basename(args.hostlist))[0].split('-')[1] return args def clean_hostname(args): h = args.hostname domain = '.kit.edu' if h.endswith(domain): args.fqdn = h else: args.fqdn = h + '.' + args.institute + domain instlist = ['itp', 'ttp', 'particle'] for i in instlist: if h.startswith(i) and i != args.institute: logging.warning("Hostname prefix and institute of hostlist don't agree.") return args def get_mac(oldmacs): print("Enter penny password if prompted.") macout = subprocess.check_output('./tools/get_last_mac.sh') macs = macout.split() macs = [m.decode() for m in macs] for m in macs: if m in oldmacs: logging.info("Found MAC %s that already exists as %s. Ignoring it." % (m, oldmacs[m])) macs.remove(m) if len(macs) == 0: logging.error("No MAC found") sys.exit(3) if len(macs) == 1: mac = macs[0] logging.info("Found MAC " + mac) else: mac = None while mac is None: print("Found MACs:") for ind, m in enumerate(macs): print("%s: %s" % (ind, m)) inp = input("Enter id to continue, empty means top entry: ") if inp == "": mac = macs[0] else: try: ind = int(inp) if ind < 0: raise IndexError mac = macs[ind] except ValueError: print("Please enter an integer.") except IndexError: print("Please enter a value between 0 and %s" % str(len(macs) - 1)) return mac def get_ip(oldips, hostlist, filename): "Return a free IP matching the range in filename, that is not yet part for hostlist." header = hostlist.fileheaders[filename] ipstart, ipend = header['iprange'] newip = get_next_ip(ipstart, ipend, oldips) return newip def main(): args = parse_args() args = clean_args(args) myhostlist = hostlist.YMLHostlist() oldhostnames = {alias: h for h in myhostlist for alias in h.aliases} if args.hostname in oldhostnames: logging.error("Hostname %s already exists: %s" % (args.hostname, oldhostnames[args.hostname])) sys.exit(2) oldips = {h.ip: h for h in myhostlist} if args.ip is not None: ip = args.ip if ip in oldips: logging.error("The given IP already exists: %s" % oldips[ip]) sys.exit(6) else: filename = os.path.basename(args.hostlist) ip = str(get_ip(oldips, myhostlist, filename)) if ip is None: logging.error("Could not find a free IP in correct range.") sys.exit(5) oldmacs = {h.mac: h for h in myhostlist if hasattr(h, 'mac')} if args.mac is not None: mac = args.mac else: mac = get_mac(oldmacs) if "notebook" in args.hostlist: sys.stdout.write("Enter user associated with notebook (leave empty to ignore): ") user = input() sys.stdout.write("Enter end_date for notebook in format YYYY-MM-DD (leave empty to ignore): ") end_date = input() hostline = """ - hostname: %s mac: %s ip: %s""" % (args.hostname, mac, ip) if user: hostline += "\n user: %s" % user if end_date: hostline += "\n end_date: %s" % end_date print("Will add%s" % hostline) usercontinue = yes_no_query("Continue?", empty_means=True) if not usercontinue: sys.exit(4) with open(args.hostlist, 'a') as f: f.write(hostline + '\n') if __name__ == '__main__': main() # TODO: implement addind a host to queue # if [ "$institute" == "itp" ] && [ "$group" == "desktops" ]; then # echo -n "Register with condor master? [yes/no] " # read commit # if [ "$commit" != "yes" ] && [ "$commit" != "no" ] ; then # die "Unrecognized option" # fi # if [ "$commit" == "yes" ] ; then # echo "Registering host with SGE master ..." # ssh root@itpcondor "/itp/admin/sbin/addqhost $host" # fi # fi # if [ "$institute" == "itp" ] && [ "$group" == "clusternode" ]; then # echo -n "Register with condor master? [yes/no] " # read commit # if [ "$commit" != "yes" ] && [ "$commit" != "no" ] ; then # die "Unrecognized option" # fi # if [ "$commit" == "yes" ] ; then # echo "Registering host with SGE master ..." # ssh root@itpcondor "/itp/admin/sbin/addqhost --no-submit $host" # fi # fi PKc}sIK)hostlist/buildfiles.py#!/usr/bin/env python3 # pylint: disable=broad-except import argparse import logging import subprocess import types from distutils.util import strtobool import sys from hostlist import hostlist from hostlist import cnamelist from hostlist import output_services from hostlist.config import CONFIGINSTANCE as Config try: from hostlist.dnsvs import sync from hostlist.dnsvs import dnsvs_webapi as dnsvs HAS_DNSVS = True except ImportError: HAS_DNSVS = False def parse_args(services): "setup parser for script arguments" parser = argparse.ArgumentParser( description='create all configuration files based on host input', ) parser.add_argument('--verbose', '-v', action='count', help='more output', default=0) parser.add_argument('--quiet', '-q', action='store_true', help='quiet run, only output errors') parser.add_argument('--dryrun', '-d', action='store_true', help='only create build files, but don\'t deploy') parser.add_argument('--stdout', action='store_true', help='Output to stdout instead of writing to a file.' ' Only implemented for ssh_known_hosts so far.') parser.add_argument('--dnsvs', action='store_true', help='sync with dnsvs') for service in services: parser.add_argument('--' + service, action='store_true', help='run ' + service) args = parser.parse_args() return args def combine_diffs(*diffs): """combines several diffs into one""" total = types.SimpleNamespace() total.add, total.remove = [], [] for diff in diffs: total.add.extend(diff.add) total.remove.extend(diff.remove) total.empty = (not total.add) and (not total.remove) return total def sync_dnsvs(file_hostlist, file_cnames, dryrun): "sync hostlist with dnsvs" if not HAS_DNSVS: logging.error("Import of DNSVS failed. Are all requirements installed?") return try: con = dnsvs.dnsvs_interface() logging.info("loading hostlist from dnsvs") dnsvs_hostlist = hostlist.DNSVSHostlist(con) logging.info("loading cnames from dnsvs") dnsvs_cnames = cnamelist.DNSVSCNamelist(con) except Exception as exc: logging.error(exc) logging.error("Failed to connect to DNSVS." " Please make sure you have a valid ssl key," " cf. Readme.md.") logging.error("Not syncing with DNSVS.") else: dnsvs_hostlist.check_consistency(dnsvs_cnames) dnsvs_diff = file_hostlist.diff(dnsvs_hostlist) dnsvs_cnames_diff = file_cnames.diff(dnsvs_cnames) total_diff = combine_diffs(dnsvs_diff, dnsvs_cnames_diff) if total_diff.empty: logging.info("DNSVS and local files agree, nothing to do") else: sync.print_diff(total_diff) if not dryrun and not total_diff.empty: print("Do you want to apply this patch to dnsvs? (y/n)") choice = input().lower() if choice != '' and strtobool(choice): sync.apply_diff(total_diff) def run_deploy(): "Deploy DNS/DHCP settings to servers" deployhosts = Config['deployhosts'] for host in deployhosts: print("Do you want to deploy to %s? (y/n)" % host) choice = input().lower() if choice != '' and strtobool(choice): print("Please enter deploy password:") ssh_res = subprocess.check_call([ 'ssh', "root@%s" % host, '-o', 'ControlPath=~/.ssh/controlmasters-%r@%h:%p', '-o', 'ControlMaster=auto', 'echo', 'success' ]) if ssh_res != 0: logging.error("Failed to establigh ssh connection to %s." " Skipping deploy.", host) continue print("Running in checkmode with diff first.") try: stdout = subprocess.check_output([ "ansible-playbook", '--check', '--diff', "copy_dns_dhcp_to_server.yml", "-l", host]) except subprocess.CalledProcessError: logging.error("Ansible check failed, skipping deploy.") continue print(stdout.decode()) print("Do you really want to deploy to %s? (y/n)" % host) choice = input().lower() if choice != '' and strtobool(choice): subprocess.call(["ansible-playbook", "copy_dns_dhcp_to_server.yml", "-l", host]) def run_servies(args, servicedict, file_hostlist, file_cnames): "Run all services according to servicedict on hosts in file_hostlist." for service, start in servicedict.items(): if start: outputcls = getattr(output_services, service.title() + "Output", None) if outputcls: logging.info("generating output for " + service) outputcls.gen_content(file_hostlist, file_cnames, args.stdout) else: logging.error("missing make function for " + service) def main(): "main routine" logging.basicConfig(format='%(levelname)s:%(message)s') services = ['dhcp', 'hosts', 'munin', 'ssh_known_hosts', 'ansible', 'ethers'] args = parse_args(services) if args.stdout: args.quiet = True args.dryrun = True logging.getLogger().setLevel(logging.INFO) if args.verbose >= 1: logging.getLogger().setLevel(logging.DEBUG) if args.quiet: logging.getLogger().setLevel(logging.CRITICAL) # get a dict of the arguments argdict = vars(args) servicedict = {s: argdict[s] for s in services} if args.stdout and not sum(servicedict.values()) == 1: logging.error("For stdout output exactly one service has to be enabled.") sys.exit(1) logging.info("loading hostlist from yml files") file_hostlist = hostlist.YMLHostlist() logging.info("loading cnames from file") file_cnames = cnamelist.FileCNamelist() file_hostlist.check_consistency(file_cnames) rundeploy = False rundnsvs = False # run set of default operations when none specified if not any(servicedict.values()): servicedict['dhcp'] = True servicedict['hosts'] = True rundeploy = True # don't sync with dnsvs on dryrun rundnsvs = not args.dryrun if args.dnsvs or rundnsvs: sync_dnsvs(file_hostlist, file_cnames, args.dryrun) run_servies(args, servicedict, file_hostlist, file_cnames) if not args.quiet: subprocess.call(["git", "--no-pager", "diff", "-U0", "build"]) if rundeploy and not args.dryrun: run_deploy() if not args.quiet: print('-' * 40) print("please remember to commit and push when you are done") print("git commit -av && git push") PKY{sI hostlist/cnamelist.py#!/usr/bin/env python3 import logging import types from .config import CONFIGINSTANCE as Config class CNamelist(list): "Representation of the list of CNames" def __str__(self): return '\n'.join([str(h) for h in self]) def diff(self, othercnames): diff = types.SimpleNamespace() diff.add, diff.remove = [], [] fqdns = {h.fqdn: h for h in self} otherfqdns = {h.fqdn: h for h in othercnames} for cname in self: if cname.fqdn not in otherfqdns or otherfqdns[cname.fqdn].dest != cname.dest: diff.add.append(cname) for cname in othercnames: if cname.fqdn not in fqdns or fqdns[cname.fqdn].dest != cname.dest: diff.remove.append(cname) diff.empty = (not diff.add) and (not diff.remove) return diff class FileCNamelist(CNamelist): "File based CNamelist" def __init__(self): source = 'cnames' # TODO: move to config fname = Config["hostlistdir"] + source try: infile = open(fname) except: logging.error('file missing %s' % fname) return content = infile.readlines() for line in content: try: uncommented = line.split('#', 1)[0] parseline = uncommented.strip() if not parseline: # line is empty (up to a comment), don't parse continue else: self.append(CNameConfline(parseline)) except Exception as e: logging.error("Failed to parse host (%s) in %s." % (line.strip(), infile.name)) logging.error(e) raise class DNSVSCNamelist(CNamelist): "DNSVS based CNamelist" def __init__(self, con): "expects a dnsvs interface passed as con" cnames = con.get_cnames() for fqdn, dest in cnames.items(): self.append(CName(fqdn, dest)) class CName: def __init__(self, fqdn, dest): if not fqdn or not dest: raise Exception("wrong initialization of CName, " "need both fqdn and dest") self.fqdn = fqdn self.dest = dest def __repr__(self): return 'CNAME: %s -> %s' % (self.fqdn, self.dest) class CNameConfline(CName): def __init__(self, line): assert line.startswith('cname=') rhs = line.split('=')[1] hostnames = rhs.split(',') if len(hostnames) != 2: raise Exception('Cnames config line has the wrong format.') self.fqdn = hostnames[0].strip() self.dest = hostnames[1].strip() PKY{sI: 4mhostlist/config.py#!/usr/bin/env python3 import yaml import logging class Config(dict): "provides access to config settings" CONFIGNAME = "config.yml" def __init__(self): "read config from file" try: with open(self.CONFIGNAME, 'r') as configfile: self.update(yaml.safe_load(configfile)) logging.info("loaded " + self.CONFIGNAME) except: logging.error("failed to load " + self.CONFIGNAME) CONFIGINSTANCE = Config() PK~sIk*hostlist/host.py#!/usr/bin/env python3 import ipaddress import logging import subprocess import datetime import re from .config import CONFIGINSTANCE as Config class Host: """ Representation of one host with several properties """ def __init__(self, hostname, ip, is_nonunique=False): self._set_defaults() self.ip = ipaddress.ip_address(ip) self.hostname = hostname self.vars['unique'] = not is_nonunique self._set_fqdn() self._set_publicip() def _set_defaults(self): self.vars = { 'needs_mac': True, 'needs_ip': True, 'unique': True, } self.ip = None self.mac = None self.hostname = "" self.publicip = True self.header = None # stores header of input file def _set_fqdn(self): if self.hostname.endswith(Config["domain"]): dot_parts = self.hostname.split('.') self.prefix = dot_parts[0] self.domain = '.'.join(dot_parts[1:]) self.fqdn = self.hostname else: self.prefix = self.hostname self.domain = self.get_domain(self.vars['institute']) self.fqdn = self.hostname + '.' + self.domain def _set_publicip(self): if not self.ip or self.ip in ipaddress.ip_network(Config["iprange"]["internal"]): self.publicip = False else: assert self.ip in ipaddress.ip_network(Config["iprange"]["external"]) self.publicip = True def get_domain(self, institute): domain = "%s.%s" % (institute, Config["domain"]) return domain def __repr__(self): return self.output(delim=' ') def __str__(self): return self.output(delim='\t') def output(self, delim='\n', printmac=False): infos = [ ("Hostname: ", self.fqdn), ("IP: ", str(self.ip) + " (nonunique)" if not self.vars['unique'] else self.ip), ] if printmac: infos.append(("MAC: ", self.mac)) out = [] for a, b in infos: if b: out += [a + str(b)] else: out += [a + '(empty)'] return delim.join(out) @property def aliases(self): "Generate hostname aliases for DNS" if self.prefix.startswith(self.vars['institute']): return [self.fqdn, self.prefix, self.prefix[len(self.vars['institute']):]] else: return [self.fqdn, self.prefix] class YMLHost(Host): "Host generated from yml file entry" _num = '(2[0-5]|1[0-9]|[0-9])?[0-9]' IPREGEXP = re.compile(r'^(' + _num + '\.){3}(' + _num + ')$') MACREGEXP = re.compile(r'^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$') def __init__(self, inputdata, hosttype, institute, header=None): """ parses a config file line of the form #host=host1.abc.kit.edu hwadress=00:12:34:ab:cd:ef ipadress=127.0.0.1 """ self._set_defaults() self.vars['hosttype'] = hosttype self.vars['institute'] = institute if header: for var, value in header.items(): self.vars[var] = value for var, value in inputdata.items(): self.vars[var] = value self.hostname = self.vars['hostname'] self._check_vars() self._check_user() self._check_end_date() self._set_fqdn() self._set_publicip() if header: if 'iprange' in header: self._check_iprange(header['iprange']) self.header = header logging.debug("Added " + str(self)) def _check_end_date(self): "Check that end_date is not over yet" if 'end_date' in self.vars: end_date = self.vars['end_date'] if not isinstance(end_date, datetime.date): logging.error("Parsing of end_date %s led to non-date datatype %s for host %s." % (end_date, end_date.__class__, self.hostname)) return if end_date < datetime.date.today(): logging.error("Host end_date in the past for host %s." % self.hostname) def _check_user(self): "Check that user (still) exists if set" if 'user' in self.vars: try: subprocess.check_output(['id', self.vars['user']]) except subprocess.CalledProcessError: logging.error("User %s does not exist and is listed for host %s." % (self.vars['user'], self.hostname)) def _check_vars(self): "Check validity of vars set for host." try: assert self.IPREGEXP.match(self.vars['ip']) self.ip = ipaddress.ip_address(self.vars['ip']) assert isinstance(self.ip, ipaddress.IPv4Address) except: raise Exception("Host %s does not have a valid IP address (%s)." % (self.hostname, self.vars['ip'])) if 'mac' in self.vars: try: assert self.MACREGEXP.match(self.vars['mac']) self.mac = MAC(self.vars['mac']) except: raise Exception("Host %s does not have a valid MAC address (%s)." % (self.hostname, self.mac)) if not self.hostname: raise Exception("No valid hostname given for %s." % str(self)) if not self.vars['institute']: raise Exception("No institute given for %s." % str(self)) def _check_iprange(self, iprange): "Check whether the given IP is in the range defined at the file header." if iprange is None: return assert len(iprange) == 2 if self.ip < iprange[0] or self.ip > iprange[1]: raise Exception("%s has IP %s outside of range %s-%s." % (self.fqdn, self.ip, iprange[0], iprange[1])) class MAC(str): """Representation of a MAC address Based on string, but always lowercase and replacing '-' with ':'. """ def __new__(cls, value): return str.__new__(cls, value.lower().replace('-', ':')) PK7sIz\q!q!hostlist/hostlist.py#!/usr/bin/env python3 import logging import types from collections import defaultdict import os import sys import ipaddress import itertools import glob import yaml try: from yaml import CSafeLoader as SafeLoader except ImportError: from yaml import SafeLoader from hostlist import host # use termcolor when available, otherwise ignore try: from termcolor import colored except ImportError: def colored(text, col): return text from .config import CONFIGINSTANCE as Config class Hostlist(list): def __init__(self): super().__init__() def __str__(self): return '\n'.join([str(h) for h in self]) def check_consistency(self, cnames): checks = [ self.check_nonunique(), self.check_cnames(cnames), self.check_duplicates(), self.check_missing_mac_ip(), ] if isinstance(self, YMLHostlist): checks.append(self.check_iprange_overlap()) logging.info("consistency check finished") if not all(checks): sys.exit(1) def check_nonunique(self): """ensure nonunique flag agrees with nonunique_ips config""" success = True nonunique_ips = defaultdict(list) for h in self: ip_fit = str(h.ip) in Config["nonunique_ips"] if ip_fit and h.vars['unique']: nonunique_ips[str(h.ip)].append(h) if not ip_fit and not h.vars['unique']: logging.error("Host %s has nonunique ip flag, " "but its ip is not listed in the config." % h) success = False for ip in nonunique_ips: if len(nonunique_ips[ip]) > 1: logging.error("More than one host uses a given nonunique ip" " without being flagged:\n" + ('\n'.join((str(x) for x in nonunique_ips[ip])))) success = False return success def check_cnames(self, cnames): """ensure there are no duplicates between hostlist and cnames""" success = True for cname in cnames: has_dest = False for h in self: if h.fqdn == cname.fqdn: logging.error("%s conflicts with %s." % (cname, h)) success = False if cname.dest == h.fqdn: has_dest = True if not has_dest: logging.error("%s points to a non-existing host." % cname) success = False return success def check_duplicates(self): """check consistency of hostlist detect duplicates (ip, mac, hostname)""" success = True inverselist = {} tocheck_props = ['ip', 'mac', 'hostname'] for prop in tocheck_props: inverselist[prop] = {} for h in self: myhostprop = getattr(h, prop) if myhostprop is None: continue if prop == 'ip' and str(myhostprop) in Config["nonunique_ips"]: # allow nonunique ips if listed in config continue if myhostprop in inverselist[prop]: logging.error("Found duplicate %s for hosts \n%s\n%s" % (prop, inverselist[prop][myhostprop], h)) success = False inverselist[prop][myhostprop] = h return success def check_missing_mac_ip(self): """check if hosts are missing an ip or mac""" success = True for h in self: if 'needs_ip' in h.vars and h.vars['needs_ip'] and h.ip is None: logging.error("Missing IP in %s " % h) success = False if isinstance(self, YMLHostlist): for h in self: if h.vars['needs_mac'] and h.mac is None: logging.error("Missing MAC in %s " % h) success = False return success def check_iprange_overlap(self): "check whether any of the ipranges given in headers overlap" overlaps = [] for ita, itb in itertools.combinations(self.fileheaders.items(), 2): filea, headera = ita fileb, headerb = itb try: a = headera['iprange'] b = headerb['iprange'] except KeyError: # one of the files does not have iprange defined, ignore it continue if ('iprange_allow_overlap' in headera and headera['iprange_allow_overlap']) or \ ('iprange_allow_overlap' in headerb and headerb['iprange_allow_overlap']): # FIXME: check overlap for internal IPs continue # check if there is overlap between a and b overlap_low = max(a[0], b[0]) overlap_high = min(a[1], b[1]) if overlap_low <= overlap_high: overlaps.append((overlap_low, overlap_high, filea, fileb)) if overlaps: for overlap in overlaps: logging.error("Found overlap from %s to %s in files %s and %s." % overlap) return not bool(overlaps) def diff(self, otherhostlist): diff = types.SimpleNamespace() diff.add, diff.remove = [], [] hostnames = {h.fqdn: h.ip for h in self if h.publicip} inversehostlist = {h.fqdn: h for h in self} otherhostnames = {h.fqdn: h.ip for h in otherhostlist if h.publicip} inverseotherhostlist = {h.fqdn: h for h in otherhostlist} for fqdn, ip in hostnames.items(): if fqdn not in otherhostnames or otherhostnames[fqdn] != ip: diff.add.append(inversehostlist[fqdn]) for fqdn, ip in otherhostnames.items(): if fqdn not in hostnames or otherhostnames[fqdn] != ip: diff.remove.append(inverseotherhostlist[fqdn]) diff.empty = (not diff.add) and (not diff.remove) return diff class DNSVSHostlist(Hostlist): "Hostlist filed from DNSVS" def __init__(self, con): super().__init__() hosts = con.get_hosts() for hostname, data in hosts.items(): ip, is_nonunique = data self.append(host.Host(hostname, ip, is_nonunique)) class YMLHostlist(Hostlist): "Hostlist filed from yml file" def __init__(self): super().__init__() self.fileheaders = {} input_ymls = sorted(glob.glob(Config["hostlistdir"] + '/*.yml')) for inputfile in input_ymls: self._add_ymlhostfile(inputfile) def _add_ymlhostfile(self, fname): "parse all hosts in fname and add them to this hostlist" shortname = os.path.splitext(os.path.basename(fname))[0] if shortname.count('-') > 1: logging.error('Filename %s contains to many dashes. Skipped.') return if '-' in shortname: # get abc, def from hostlists/abc-def.yml hosttype, institute = shortname.split('-') else: hosttype = shortname institute = None try: infile = open(fname, 'r') except: logging.error('file %s not readable' % fname) return try: yamlsections = yaml.load_all(infile, Loader=SafeLoader) except yaml.YAMLError as e: logging.error('file %s not correct yml' % fname) logging.error(str(e)) return for yamlout in yamlsections: if 'header' not in yamlout: logging.error('missing header field in %s' % fname) if 'hosts' not in yamlout: logging.error('missing hosts field in %s' % fname) header = yamlout['header'] if 'iprange' in header: ipstart, ipend = header['iprange'] header['iprange'] = ipaddress.ip_address(ipstart), ipaddress.ip_address(ipend) self.fileheaders[os.path.basename(fname)] = header for hostdata in yamlout["hosts"]: self.append(host.YMLHost(hostdata, hosttype, institute, header)) # do replacements for docker for h in self: if 'docker' in h.vars and 'ports' in h.vars['docker']: # prefix docker ports with container IP h.vars['docker']['ports'] = [ str(h.ip) + ':' + port for port in h.vars['docker']['ports'] ] PKY{sIhostlist/output_services.py#!/usr/bin/env python3 from collections import defaultdict import json import ipaddress import os import logging from .config import CONFIGINSTANCE as Config class OutputBase: "Baseclass for output services" @classmethod def gen_content(cls, hostlist, cnames, stdout): "Return output for requested service" pass @staticmethod def write(content, buildname, stdout=False): "Write content to a file or stdout" if stdout: print(content) else: builddir = Config["builddir"] if not os.path.isdir(builddir): os.mkdir(builddir) fname = builddir + buildname with open(fname, 'w') as f: f.write(content) class Ssh_Known_HostsOutput(OutputBase): "Generate hostlist for ssh-keyscan" @classmethod def gen_content(cls, hostlist, cnames, stdout): # only scan keys on hosts that are in ansible scan_hosts = [h for h in hostlist if h.vars.get('gen_ssh_known_hosts', False)] aliases = [alias for host in scan_hosts for alias in host.aliases if host.ip] aliases += [str(host.ip) for host in scan_hosts if host.ip] aliases += [cname.fqdn for cname in cnames] fcont = '\n'.join(aliases) cls.write(fcont, Config["ssh"]["hostlist"], stdout) logging.debug("wrote ssh hostlist to file") class HostsOutput(OutputBase): "Config output for /etc/hosts format" @classmethod def gen_content(cls, hostlist, cnames, stdout): hoststrings = (str(h.ip) + " " + " ".join(h.aliases) for h in hostlist if h.ip) content = '\n'.join(hoststrings) cls.write(content, Config["hosts"]["build"], stdout) class MuninOutput(OutputBase): "Config output for Munin" @classmethod def gen_content(cls, hostlist, cnames, stdout): hostnames = (h for h in hostlist if h.vars.get('gen_munin', False) and h.publicip) fcont = '' for host in hostnames: fcont += cls._get_hostblock(host) cls.write(fcont, Config["munin"]["build"], stdout) @staticmethod def _get_hostblock(host): cont = '[{institute}{hosttype};{h}]\naddress {h}\n'.format( h=host.fqdn, institute=host.vars['institute'], hosttype=host.vars['hosttype'], ) if 'munin' in host.vars: for line in host.vars['munin']: cont += line + '\n' return cont class DhcpOutput(OutputBase): "DHCP config output" @classmethod def gen_content(cls, hostlist, cnames, stdout): dhcpout = "" dhcp_internal = "" for host in hostlist: entry = cls._gen_hostline(host) if not entry: continue if host.ip in ipaddress.ip_network(Config['iprange']['internal']): dhcp_internal += entry + '\n' else: dhcpout += entry + '\n' cls.write(dhcpout, Config["dhcp"]["build"], stdout) cls.write(dhcp_internal, Config["dhcp"]["build_internal"], stdout) @staticmethod def _gen_hostline(host): if host.mac and host.ip: # curly brackets doubled for python format funciton return """host {fqdn} {{ hardware ethernet {mac}; fixed-address {ip}; option host-name "{hostname}"; option domain-name "{domain}"; }}""".format(fqdn=host.fqdn, mac=host.mac, ip=host.ip, hostname=host.hostname, domain=host.domain) class AnsibleOutput(OutputBase): "Ansible inventory output" @classmethod def gen_content(cls, hostlist, cnames, stdout): """generate json inventory for ansible form: { "_meta": { "hostvars": { "myhost.abc.kit.edu": { "hosttype": "desktop", "institute": "abc", "custom_variable": "foo", } ... } }, "groupname" : [ "myhost2.abc.kit.edu", ] ... """ assert stdout, "Ansible Output only works for stdout" resultdict = defaultdict(lambda: {'hosts': []}) hostvars = {} docker_services = {} for host in hostlist: # online add hosts that have ansible=yes if 'ansible' in host.vars and not host.vars['ansible']: continue ans = cls._gen_host_content(host) hostvars[ans['fqdn']] = ans['vars'] for groupname in ans['groups']: resultdict[groupname]['hosts'] += [ans['fqdn']] if ans['vars']['hosttype'] == 'docker': resultdict['dockerhost-' + host.hostname]['hosts'] += [ans['vars']['docker']['host']] docker_services[host.hostname] = ans['vars']['docker'] docker_services[host.hostname]['fqdn'] = host.fqdn docker_services[host.hostname]['ip'] = str(host.ip) resultdict['vserverhost']['vars'] = {'docker_services': docker_services} resultdict['_meta'] = {'hostvars': hostvars} jsonout = json.dumps(resultdict, sort_keys=True, indent=1) cls.write(jsonout, None, stdout=True) @staticmethod def _gen_host_content(host): "Generate output for one host" hostgroups = [host.vars['institute'], host.vars['hosttype'], host.vars['institute'] + host.vars['hosttype']] result = { 'fqdn': host.fqdn, 'groups': hostgroups, 'vars': {}, } if host.ip: result['vars']['ip'] = str(host.ip) ansiblevars = ['subnet', 'institute', 'hosttype', 'docker'] for avar in ansiblevars: if avar in host.vars: result['vars'][avar] = host.vars[avar] return result class EthersOutput(OutputBase): "/etc/ethers format output" @classmethod def gen_content(cls, hostlist, cnames, stdout): entries = ( "%s %s" % (host.mac, host.fqdn) for host in hostlist if host.mac ) out = '\n'.join(entries) cls.write(out, Config["ethers"], stdout) PKY{sIhostlist/dnsvs/__init__.pyPK~sIgJ"J"hostlist/dnsvs/dnsvs_webapi.py#!/usr/bin/python3 import requests import json import os.path import logging from hostlist import host from hostlist import cnamelist class dnsvs_interface: certfilename = '~/.ssl/net-webapi.key' certfilename = os.path.expanduser(certfilename) root_url = 'https://www-net.scc.kit.edu/api/2.0/dns' geturl = root_url + '/record/list' createurl = root_url + '/record/create' deleteurl = root_url + '/record/delete' # all our entries ar IPv4 inttype_a = "host:0100,:,402,A" inttype_nonunique = "domain:1000,:,400,A" inttype_cname = "alias:0000,host:0100,011,CNAME" headers_dict = {"Content-Type": "application/json"} def _execute(self, url, method, data=None): """Actually perform an operation on the DNS server.""" try: if method == "get": response = requests.get(url=url, headers=self.headers_dict, cert=self.certfilename) elif method == "post": response = requests.post(url=url, data=data, headers=self.headers_dict, cert=self.certfilename) if response.ok: return response.json() else: raise requests.exceptions.RequestException(response.status_code, response.text) except Exception as e: logging.error(e) raise return json.dumps({}) def get_hosts(self): """Reads A records from the server.""" result = self._execute(self.geturl, method="get") # continue with normal request (process result) hosts = {} for entry in result: fqdn = entry['fqdn'].rstrip(".") if entry['type'] == 'A': is_nonunique = entry['inttype'] == self.inttype_nonunique hosts[fqdn] = (entry['data'], is_nonunique) return hosts def get_cnames(self): """Reads CNAME records from the server.""" result = self._execute(self.geturl, method="get") # continue with normal request (process result) cname = {} for entry in result: fqdn = entry['fqdn'].rstrip(".") if entry['type'] == 'CNAME': cname[fqdn] = entry['data'].rstrip(".") return cname def add(self, entry): """generic interface to add_*""" if isinstance(entry, host.Host): self.add_host(entry) elif isinstance(entry, cnamelist.CName): self.add_cname(entry) def remove(self, entry): """generic interface to remove_*""" if isinstance(entry, host.Host): self.remove_host(entry) elif isinstance(entry, cnamelist.CName): self.remove_cname(entry) def add_host(self, host): """Adds an A record to the server.""" # TODO: handle these errors in the response # check whether there is already a CNAME with that fqdn # url = self.root_url+"/record/list?type=CNAME&fqdn="+host.fqdn+"." # dependencies = self._execute(url=url, method="get") # if dependencies!=[]: # raise Exception('Attempting to overwrite an existing CNAME record in DNSVS with an A record!') # url = self.root_url+"/record/list?type=A&fqdn="+host.fqdn+"." # dependencies = self._execute(url=url, method="get") # if dependencies!=[]: # if dependencies[0]['data']==str(host.ip): # logging.warning('Attempting to add already an existing A record.') # return # elif dependencies[0]['data']!=str(host.ip): # raise Exception('Attempting to overwrite an existing A record with a different one.') inttype = self.inttype_nonunique if not host.vars['unique'] else self.inttype_a data = [ {"param_list": [ {"name": "fqdn", "new_value": host.fqdn + "."}, {"name": "data", "new_value": str(host.ip)}, {"name": "inttype", "new_value": inttype}, ]}, ] json_string = json.dumps(data) self._execute(url=self.createurl, method="post", data=json_string) def remove_host(self, host): """Remove an A record from the server.""" # TODO: handle these errors in the response # before removing, check whether a cname points to that record # https://www-net.scc.kit.edu/api/2.0/dns/record/list?target_fqdn_regexp=ttpseth.ttp.kit.edu. # url = self.root_url+"/record/list?type=CNAME&target_fqdn="+host.fqdn+"." # dependencies = self._execute(url=url, method="get") # if dependencies!=[]: # raise Exception('Attempting to remove an A record of a host to which a cname ist pointing.') # url = self.root_url+"/record/list?type=A&fqdn="+host.fqdn+"." # dnsvs_records = self._execute(url=url, method="get") # if dnsvs_records!=[]: # if dnsvs_records[0]['data']!=str(host.ip): # raise Exception('Attempting to remove an existing A record, for which the IP adress does not match.') # else: # logging.warning('Attempting to remove a nonexistent A record.') # return # remove # TODO: can we use host.fqdn/host.ip here? # trust our data more then theirs inttype = self.inttype_a if host.vars['unique'] else self.inttype_nonunique data = [ {"param_list": [ {"name": "fqdn", "old_value": host.fqdn + "."}, {"name": "data", "old_value": str(host.ip)}, {"name": "inttype", "old_value": inttype}, ]}, ] json_string = json.dumps(data) self._execute(url=self.deleteurl, method="post", data=json_string) def add_cname(self, cname): """Adds a CNAME record given by (alias, hostname) to the server.""" fqdn, dest = cname.fqdn, cname.dest # TODO: handle these errors in the response # check whether the cname record is already there # url = self.root_url+"/record/list?type=CNAME&target_fqdn_regexp="+dest+"."+"&fqdn="+fqdn+"." # dependencies = self._execute(url=url, method="get") # if dependencies!=[]: # logging.warning('Attempting to add an already existing CNAME record to DNSVS!') # return # else: # # check whether there is a different CNAME record for the same fqdn # url = self.root_url+"/record/list?type=CNAME&fqdn="+fqdn+"." # dependencies = self._execute(url=url, method="get") # if dependencies!=[]: # raise Exception('Attempting to overwrite an existing CNAME record in DNSVS with a different one!') # # check whether there is an A record with the same fqdn # url = self.root_url+"/record/list?type=A&fqdn="+fqdn+"." # dependencies = self._execute(url=url, method="get") # if dependencies!=[]: # raise Exception('Attempting to overwrite an A record in DNSVS with a CNAME record!') # url = self.root_url+"/record/list?type=A&fqdn="+dest+"." # inarecords = self._execute(url=url, method="get") # url = self.root_url+"/record/list?type=CNAME&fqdn="+fqdn+"." # incnames = self._execute(url=url, method="get") # if inarecords==[] and incnames==[]: # raise Exception('Attempting to add a CNAME record do DNSVS pointing to a nonexistent fqdn!') # write data = [ {"param_list": [ {"name": "fqdn", "new_value": fqdn + "."}, {"name": "data", "new_value": dest + "."}, {"name": "inttype", "new_value": self.inttype_cname}, ]}, ] json_string = json.dumps(data) self._execute(url=self.createurl, method="post", data=json_string) def remove_cname(self, cname): """Remove a CNAME record from the server.""" fqdn, dest = cname.fqdn, cname.dest # TODO: handle these errors in the response # check whether the cname record is there in the first place # url = self.root_url+"/record/list?type=CNAME&target_fqdn_regexp="+dest+"."+"&fqdn="+fqdn+"." # dependencies = self._execute(url=url, method="get") # if dependencies == []: # logging.warning('Attempting to remove a nonexistent CNAME record from DNSVS!') # return # remove data = [ {"param_list": [ {"name": "fqdn", "old_value": fqdn + "."}, {"name": "data", "old_value": dest + "."}, {"name": "inttype", "old_value": self.inttype_cname}, ]}, ] json_string = json.dumps(data) self._execute(url=self.deleteurl, method="post", data=json_string) PK~sIervvhostlist/dnsvs/sync.py#!/usr/bin/env python3 import logging from hostlist.dnsvs import dnsvs_webapi as dnsvs # use termcolor when available, otherwise ignore try: from termcolor import colored except ImportError: def colored(text, col): return text def apply_diff(diff): con = dnsvs.dnsvs_interface() for entry in diff.remove: logging.info('removing\t' + str(entry)) con.remove(entry) for entry in diff.add: logging.info('adding\t' + str(entry)) con.add(entry) def print_diff(diff): if diff.add: print(colored("Only in local files: ", 'green')) for h in sorted(diff.add, key=lambda h: h.fqdn): print(colored('+' + str(h), 'green')) if diff.remove: print(colored("Only in DNSVS: ", 'red')) for h in sorted(diff.remove, key=lambda h: h.fqdn): print(colored('-' + str(h), 'red')) PK!HL0|=Y)hostlist-1.0.1.dist-info/entry_points.txtN+I/N.,()JLI/.QUQ9%zP!<̜̜bd%Q*.PK!H;@QPhostlist-1.0.1.dist-info/WHEEL1 0 RZq+D-Dv;_[*7Fp ܦpv/fݞoL(*IPK!Hu2Zz!hostlist-1.0.1.dist-info/METADATA]n@ E .[+ &ԑ(c9hrtl qJQ_T.ew7QDηR^ZL{B{e anXh,lR{%Ak#[m($u<3 D1x"?A_}fNUз6# ::G^7ȳM 3U[uk:ɧ3>jEtMxTs]ݮFu7 6ķWd?PK!HIghostlist-1.0.1.dist-info/RECORDuǒH< aXa$`.V@CD!Fݹd2h{T=%,Qn니2k.ItӋ^(z0QXZˋ?1r6Fpԏˊ3k`UDkwJɢ]~`灨Q,:LcI|[Pi^YLn zAݸyROerZ펱4ɽFMVfLb-AgLX@-qUڮ,^3ݬs-c$K0+yy_QXe,9w29Il'~-}ۗ3'IҶɲ ʢw̠( '9b n{c{Fݸx]>"JGoq 52k (L~QQkS%k:Yz*;ҟW'R#Pjg)_JU*#^[];s>yt7SyPcG]622KyZ;/qZ}Xsbi9U5N_RܟaGOa?PK}sIإ_hostlist/__init__.pyPK~sI[^""hostlist/addhost.pyPKc}sIK)<hostlist/buildfiles.pyPKY{sI t:hostlist/cnamelist.pyPKY{sI: 4mCEhostlist/config.pyPK~sIk*\Ghostlist/host.pyPK7sIz\q!q!E_hostlist/hostlist.pyPKY{sIhostlist/output_services.pyPKY{sIhostlist/dnsvs/__init__.pyPK~sIgJ"J"hostlist/dnsvs/dnsvs_webapi.pyPK~sIervv큟hostlist/dnsvs/sync.pyPK!HL0|=Y)Ihostlist-1.0.1.dist-info/entry_points.txtPK!H;@QPhostlist-1.0.1.dist-info/WHEELPK!Hu2Zz!Zhostlist-1.0.1.dist-info/METADATAPK!HIghostlist-1.0.1.dist-info/RECORDPK*