PK 4J hostlist/__init__.py#!/usr/bin/env python3
"Sync hostlist and builds config files for services."
from . import buildfiles
__version__ = '1.2.3'
if __name__ == "__main__":
buildfiles.main()
PK +JIrE3 3 hostlist/addhost.py#!/usr/bin/env python3
import os.path
import logging
import subprocess
import argparse
from distutils.util import strtobool
import sys
from . import hostlist
def yes_no_query(question, empty_means=None):
if empty_means is None:
choices = '[y/n]'
elif empty_means is True:
choices = '[Y/n]'
elif empty_means is False:
choices = '[y/N]'
sys.stdout.write('%s %s' % (question, choices))
while True:
inp = input()
if inp.strip() == '' and empty_means is not None:
return empty_means
else:
try:
return strtobool(inp)
except ValueError:
print('Please respond with \'y\' or \'n\'.')
def get_next_ip(ipstart, ipend, existing_ips):
newip = ipstart
while newip <= ipend:
if newip not in existing_ips:
return newip
newip += 1
return None
def parse_args():
parser = argparse.ArgumentParser(description="Add a new host to hostlist.")
parser.add_argument('hostname', help='New hostname, e.g. fullname.itp.kit.edu, ttpmyfriend, itpalbatros98, newmachine.particle')
parser.add_argument('hostlist', help='Filename to add to, e.g. desktops.itp.'
' The prefix "hostlists/" is optional.')
parser.add_argument('--ip', help='Give IP to use.')
parser.add_argument('--mac', help='Give mac to use.')
parser.add_argument('--verbose',
'-v',
action='count',
help='more output',
default=0)
args = parser.parse_args()
logging.basicConfig(format='%(levelname)s:%(message)s')
logging.getLogger().setLevel(logging.WARNING)
if args.verbose == 1:
logging.getLogger().setLevel(logging.INFO)
elif args.verbose >= 2:
logging.getLogger().setLevel(logging.DEBUG)
return args
def clean_args(args):
args = clean_hostlist(args)
args = clean_hostname(args)
logging.debug("Using hostname %s and hostlist %s." % (args.hostname, args.hostlist))
return args
def clean_hostlist(args):
"""makes sure the hostlist file exists
also set institute based on hostlist file
"""
if not os.path.isfile(args.hostlist):
newhostlist = './hostlists/' + args.hostlist
if not os.path.isfile(newhostlist):
logging.error("Neiter %s nor %s is a file." % (args.hostlist, newhostlist))
sys.exit(10)
else:
logging.info("Using %s as hostlist." % newhostlist)
args.hostlist = newhostlist
args.institute = os.path.splitext(os.path.basename(args.hostlist))[0].split('-')[1]
return args
def clean_hostname(args):
h = args.hostname
domain = '.kit.edu'
if h.endswith(domain):
args.fqdn = h
else:
args.fqdn = h + '.' + args.institute + domain
instlist = ['itp', 'ttp', 'particle']
for i in instlist:
if h.startswith(i) and i != args.institute:
logging.warning("Hostname prefix and institute of hostlist don't agree.")
return args
def get_mac(oldmacs):
print("Enter penny password if prompted.")
macout = subprocess.check_output('./tools/get_last_mac.sh')
macs = macout.split()
macs = [m.decode() for m in macs]
for m in macs:
if m in oldmacs:
logging.info("Found MAC %s that already exists as %s. Ignoring it." % (m, oldmacs[m]))
macs.remove(m)
if len(macs) == 0:
logging.error("No MAC found")
sys.exit(3)
if len(macs) == 1:
mac = macs[0]
logging.info("Found MAC " + mac)
else:
mac = None
while mac is None:
print("Found MACs:")
for ind, m in enumerate(macs):
print("%s: %s" % (ind, m))
inp = input("Enter id to continue, empty means top entry: ")
if inp == "":
mac = macs[0]
else:
try:
ind = int(inp)
if ind < 0:
raise IndexError
mac = macs[ind]
except ValueError:
print("Please enter an integer.")
except IndexError:
print("Please enter a value between 0 and %s" % str(len(macs) - 1))
return mac
def get_ip(oldips, hostlist, filename):
"Return a free IP matching the range in filename, that is not yet part for hostlist."
header = hostlist.fileheaders[filename]
ipstart, ipend = header['iprange']
newip = get_next_ip(ipstart, ipend, oldips)
return newip
def main():
args = parse_args()
args = clean_args(args)
myhostlist = hostlist.YMLHostlist()
oldhostnames = {alias: h for h in myhostlist for alias in h.aliases}
if args.hostname in oldhostnames:
logging.error("Hostname %s already exists: %s" % (args.hostname, oldhostnames[args.hostname]))
sys.exit(2)
oldips = {h.ip: h for h in myhostlist}
if args.ip is not None:
ip = args.ip
if ip in oldips:
logging.error("The given IP already exists: %s" % oldips[ip])
sys.exit(6)
else:
filename = os.path.basename(args.hostlist)
ip = str(get_ip(oldips, myhostlist, filename))
if ip is None:
logging.error("Could not find a free IP in correct range.")
sys.exit(5)
oldmacs = {h.mac: h for h in myhostlist if hasattr(h, 'mac')}
if args.mac is not None:
mac = args.mac
else:
mac = get_mac(oldmacs)
user, end_date = "", ""
if "notebook" in args.hostlist:
sys.stdout.write("Enter user associated with notebook (leave empty to ignore): ")
user = input()
sys.stdout.write("Enter end_date for notebook in format YYYY-MM-DD (leave empty to ignore): ")
end_date = input()
hostline = """
- hostname: %s
mac: %s
ip: %s""" % (args.hostname, mac, ip)
if user:
hostline += "\n user: %s" % user
if end_date:
hostline += "\n end_date: %s" % end_date
print("Will add%s" % hostline)
usercontinue = yes_no_query("Continue?", empty_means=True)
if not usercontinue:
sys.exit(4)
with open(args.hostlist, 'a') as f:
f.write(hostline + '\n')
if __name__ == '__main__':
main()
# TODO: implement addind a host to queue
# if [ "$institute" == "itp" ] && [ "$group" == "desktops" ]; then
# echo -n "Register with condor master? [yes/no] "
# read commit
# if [ "$commit" != "yes" ] && [ "$commit" != "no" ] ; then
# die "Unrecognized option"
# fi
# if [ "$commit" == "yes" ] ; then
# echo "Registering host with SGE master ..."
# ssh root@itpcondor "/itp/admin/sbin/addqhost $host"
# fi
# fi
# if [ "$institute" == "itp" ] && [ "$group" == "clusternode" ]; then
# echo -n "Register with condor master? [yes/no] "
# read commit
# if [ "$commit" != "yes" ] && [ "$commit" != "no" ] ; then
# die "Unrecognized option"
# fi
# if [ "$commit" == "yes" ] ; then
# echo "Registering host with SGE master ..."
# ssh root@itpcondor "/itp/admin/sbin/addqhost --no-submit $host"
# fi
# fi
PK *4J<} } hostlist/buildfiles.py#!/usr/bin/env python3
# pylint: disable=broad-except
import argparse
import logging
import types
from distutils.util import strtobool
import sys
import json
from . import hostlist
from . import cnamelist
from . import output_services
from .config import CONFIGINSTANCE as Config
try:
from .dnsvs import sync
from .dnsvs import dnsvs_webapi as dnsvs
HAS_DNSVS = True
except ImportError:
HAS_DNSVS = False
def parse_args(services):
"setup parser for script arguments"
parser = argparse.ArgumentParser(
description='create all configuration files based on host input',
)
parser.add_argument('--verbose',
'-v',
action='count',
help='more output',
default=0)
parser.add_argument('--quiet',
'-q',
action='store_true',
help='quiet run, only output errors')
parser.add_argument('--dryrun',
'-d',
action='store_true',
help='only parse, don\'t sync')
parser.add_argument('filter',
nargs='*',
help='''Print hosts matching a given filter. This can be hostnames or groupnames.''')
for service in services:
parser.add_argument('--' + service,
action='store_true',
help='run ' + service)
args = parser.parse_args()
return args
def combine_diffs(*diffs):
"""combines several diffs into one"""
total = types.SimpleNamespace()
total.add, total.remove = [], []
for diff in diffs:
total.add.extend(diff.add)
total.remove.extend(diff.remove)
total.empty = (not total.add) and (not total.remove)
return total
def sync_dnsvs(file_hostlist, file_cnames, dryrun):
"sync hostlist with dnsvs"
if not HAS_DNSVS:
logging.error("Import of DNSVS failed. Are all requirements installed?")
return
try:
con = dnsvs.dnsvs_interface()
logging.info("loading hostlist from dnsvs")
dnsvs_hostlist = hostlist.DNSVSHostlist(con)
logging.info("loading cnames from dnsvs")
dnsvs_cnames = cnamelist.DNSVSCNamelist(con)
except Exception as exc:
logging.error(exc)
logging.error("Failed to connect to DNSVS."
" Please make sure you have a valid ssl key,"
" cf. Readme.md.")
logging.error("Not syncing with DNSVS.")
else:
dnsvs_diff = file_hostlist.diff(dnsvs_hostlist)
dnsvs_cnames_diff = file_cnames.diff(dnsvs_cnames)
total_diff = combine_diffs(dnsvs_diff, dnsvs_cnames_diff)
if total_diff.empty:
logging.info("DNSVS and local files agree, nothing to do")
else:
sync.print_diff(total_diff)
if not dryrun and not total_diff.empty:
print("Do you want to apply this patch to dnsvs? (y/n)")
choice = input().lower()
if choice != '' and strtobool(choice):
sync.apply_diff(total_diff)
def run_service(service, file_hostlist, file_cnames):
"Run all services according to servicedict on hosts in file_hostlist."
outputcls = getattr(output_services, service.title() + "Output", None)
if outputcls:
logging.info("generating output for " + service)
out = outputcls.gen_content(file_hostlist, file_cnames)
if isinstance(out, str):
print(out)
else:
print(json.dumps(out, indent=2))
else:
logging.critical("missing make function for " + service)
def main():
"main routine"
logging.basicConfig(format='%(levelname)s:%(message)s')
services = ['dhcp', 'dhcpinternal', 'hosts', 'munin',
'ssh_known_hosts', 'ansible', 'ethers', 'web']
args = parse_args(services)
# get a dict of the arguments
argdict = vars(args)
activeservices = {s for s in services if argdict[s]}
if activeservices:
if len(activeservices) > 1:
logging.error("Can only output one service at a time.")
sys.exit(2)
args.quiet = True
args.dryrun = True
logging.getLogger().setLevel(logging.INFO)
if args.verbose >= 1:
logging.getLogger().setLevel(logging.DEBUG)
if args.quiet:
logging.getLogger().setLevel(logging.ERROR)
if not Config.load():
logging.error("Need %s file to run." % Config.CONFIGNAME)
sys.exit(1)
logging.info("loading hostlist from yml files")
file_hostlist = hostlist.YMLHostlist()
logging.info("loading cnames from file")
file_cnames = cnamelist.FileCNamelist()
file_hostlist.check_consistency(file_cnames)
if args.filter:
file_hostlist.print(args.filter)
sys.exit(0)
if activeservices:
run_service(activeservices.pop(), file_hostlist, file_cnames)
if not args.dryrun:
sync_dnsvs(file_hostlist, file_cnames, args.dryrun)
if not args.quiet:
print('-' * 40)
print("please remember to commit and push when you are done")
print("git commit -av && git push")
PK {/J
hostlist/cnamelist.py#!/usr/bin/env python3
import logging
import types
from .config import CONFIGINSTANCE as Config
class CNamelist(list):
"Representation of the list of CNames"
def __str__(self):
return '\n'.join([str(h) for h in self])
def diff(self, othercnames):
diff = types.SimpleNamespace()
diff.add, diff.remove = [], []
fqdns = {h.fqdn: h for h in self}
otherfqdns = {h.fqdn: h for h in othercnames}
for cname in self:
if cname.fqdn not in otherfqdns or otherfqdns[cname.fqdn].dest != cname.dest:
diff.add.append(cname)
for cname in othercnames:
if cname.fqdn not in fqdns or fqdns[cname.fqdn].dest != cname.dest:
diff.remove.append(cname)
diff.empty = (not diff.add) and (not diff.remove)
return diff
class FileCNamelist(CNamelist):
"File based CNamelist"
def __init__(self):
source = 'cnames' # TODO: move to config
fname = Config["hostlistdir"] + source
try:
infile = open(fname)
except:
logging.error('file missing %s' % fname)
return
content = infile.readlines()
for line in content:
try:
uncommented = line.split('#', 1)[0]
parseline = uncommented.strip()
if not parseline:
# line is empty (up to a comment), don't parse
continue
else:
self.append(CNameConfline(parseline))
except Exception as e:
logging.error("Failed to parse host (%s) in %s." %
(line.strip(), infile.name))
logging.error(e)
raise
class DNSVSCNamelist(CNamelist):
"DNSVS based CNamelist"
def __init__(self, con):
"expects a dnsvs interface passed as con"
cnames = con.get_cnames()
for fqdn, dest in cnames.items():
self.append(CName(fqdn, dest))
class CName:
def __init__(self, fqdn, dest):
if not fqdn or not dest:
raise Exception("wrong initialization of CName, "
"need both fqdn and dest")
self.fqdn = fqdn
self.dest = dest
def __repr__(self):
return 'CNAME: %s -> %s' % (self.fqdn, self.dest)
class CNameConfline(CName):
def __init__(self, line):
assert line.startswith('cname=')
rhs = line.split('=')[1]
hostnames = rhs.split(',')
if len(hostnames) != 2:
raise Exception('Cnames config line has the wrong format.')
self.fqdn = hostnames[0].strip()
self.dest = hostnames[1].strip()
PK uI`8 hostlist/config.py#!/usr/bin/env python3
import yaml
import logging
class Config(dict):
"provides access to config settings"
CONFIGNAME = "config.yml"
def __init__(self):
self._loaded = False
def __getitem__(self, *args):
if not self._loaded:
self.load()
return dict.__getitem__(self, *args)
def load(self):
"read config from file"
try:
with open(self.CONFIGNAME, 'r') as configfile:
self.update(yaml.safe_load(configfile))
logging.info("loaded " + self.CONFIGNAME)
self._loaded = True
except:
logging.error("failed to load " + self.CONFIGNAME)
self._loaded = False
return self._loaded
CONFIGINSTANCE = Config()
PK ́4J~$ $ hostlist/daemon.py#!/usr/bin/env python3
# import types
# from distutils.util import strtobool
# import sys
import git
import logging
import datetime
import cherrypy
from . import hostlist
from . import cnamelist
from . import output_services
# from hostlist.config import CONFIGINSTANCE as Config
class Inventory():
def __init__(self):
try:
self.repo = git.Repo('.')
except git.InvalidGitRepositoryError:
self.repo = git.Repo('../')
self.last_update = None
self.fetch_hostlist()
def fetch_hostlist(self, timeout=600):
if self.last_update and datetime.datetime.now() - self.last_update < datetime.timedelta(seconds=timeout):
return
self.last_update = datetime.datetime.now()
try:
pullresult = self.repo.remote().pull()[-1]
if not pullresult.flags & pullresult.HEAD_UPTODATE:
logging.error("Hosts repo not up to date after pull.")
except:
logging.error("Failed to pull hosts repo.")
self.hostlist = hostlist.YMLHostlist()
self.cnames = cnamelist.FileCNamelist()
print("Refreshed cache.")
@cherrypy.expose
@cherrypy.tools.json_out()
def ansible(self):
self.fetch_hostlist()
return output_services.AnsibleOutput.gen_content(self.hostlist, self.cnames)
@cherrypy.expose
def munin(self):
self.fetch_hostlist()
return output_services.MuninOutput.gen_content(self.hostlist, self.cnames)
@cherrypy.expose
def dhcp(self):
self.fetch_hostlist()
return output_services.DhcpOutput.gen_content(self.hostlist, self.cnames)
@cherrypy.expose
def hosts(self):
self.fetch_hostlist()
return output_services.HostsOutput.gen_content(self.hostlist, self.cnames)
@cherrypy.expose
def ethers(self):
self.fetch_hostlist()
return output_services.EthersOutput.gen_content(self.hostlist, self.cnames)
@cherrypy.expose
def list(self):
self.fetch_hostlist()
return output_services.WebOutput.gen_content(self.hostlist, self.cnames)
@cherrypy.expose
def status(self):
result = 'Have a hostlist with %s hosts and %s cnames.' % (len(self.hostlist), len(self.cnames))
result += '\nLast updated: %s' % self.last_update
return result
@cherrypy.expose
@cherrypy.config(**{'tools.caching.delay': 10})
def refreshcache(self):
cherrypy.lib.caching.cherrypy._cache.clear()
self.fetch_hostlist(timeout=10)
@cherrypy.expose
def index(self):
return 'See github.com/particleKIT/hostlist how to use this API.'
def _auth_config(app):
if app.config.get('/', {}).get('tools.auth_digest.on', False):
users = app.config['authentication']
app.config['/'].update({'tools.auth_digest.get_ha1': cherrypy.lib.auth_digest.get_ha1_dict_plain(users)})
elif app.config.get('/', {}).get('tools.auth_basic.on', False):
users = app.config['authentication']
def check_pass(realm, user, password):
if user not in users:
return False
else:
return users[user] == password
app.config['/'].update({'tools.auth_basic.checkpassword': check_pass})
def main():
cherrypy.config.update('daemon.conf')
app = cherrypy.tree.mount(Inventory(), '/', 'daemon.conf')
_auth_config(app)
cherrypy.engine.signals.subscribe()
cherrypy.engine.start()
cherrypy.engine.block()
if __name__ == '__main__':
main()
PK 4J
hostlist/host.py#!/usr/bin/env python3
import ipaddress
import logging
import subprocess
import datetime
import re
from .config import CONFIGINSTANCE as Config
class Host:
"""
Representation of one host with several properties
"""
def __init__(self, hostname, ip, is_nonunique=False):
self._set_defaults()
self.ip = ipaddress.ip_address(ip)
self.hostname = hostname
self.vars['unique'] = not is_nonunique
self._set_fqdn()
self._set_publicip()
def _set_defaults(self):
self.vars = {
'needs_mac': True,
'needs_ip': True,
'unique': True,
}
self.ip = None
self.mac = None
self.hostname = ""
self.publicip = True
self.header = None # stores header of input file
def _set_fqdn(self):
if self.hostname.endswith(Config["domain"]):
dot_parts = self.hostname.split('.')
self.prefix = dot_parts[0]
self.domain = '.'.join(dot_parts[1:])
self.fqdn = self.hostname
else:
self.prefix = self.hostname
self.domain = self.get_domain(self.vars['institute'])
self.fqdn = self.hostname + '.' + self.domain
def _set_publicip(self):
if not self.ip or self.ip in ipaddress.ip_network(Config["iprange"]["internal"]):
self.publicip = False
else:
assert self.ip in ipaddress.ip_network(Config["iprange"]["external"])
self.publicip = True
def get_domain(self, institute):
domain = "%s.%s" % (institute, Config["domain"])
return domain
def __repr__(self):
return self.output(delim=' ')
def __str__(self):
return self.output(delim='\t')
def output(self,
delim='\n',
printmac=False,
printgroups=False,
printallvars=False,
printvars=[]):
infos = [
("Hostname: ", self.fqdn),
("IP: ", str(self.ip) + " (nonunique)" if not self.vars['unique'] else self.ip),
]
if printmac:
infos.append(("MAC: ", self.mac))
if printallvars:
printvars = self.vars.keys()
for var in sorted(printvars):
infos.append((var + ': ', self.vars.get(var)))
if printgroups:
infos.append(('Groups: ', ', '.join(self.groups)))
out = []
for a, b in infos:
if b:
out += [a + str(b)]
else:
out += [a + '(empty)']
return delim.join(out)
@property
def aliases(self):
"Generate hostname aliases for DNS"
if self.prefix.startswith(self.vars['institute']):
return [self.fqdn, self.prefix, self.prefix[len(self.vars['institute']):]]
else:
return [self.fqdn, self.prefix]
class YMLHost(Host):
"Host generated from yml file entry"
_num = '(2[0-5]|1[0-9]|[0-9])?[0-9]'
IPREGEXP = re.compile(r'^(' + _num + '\.){3}(' + _num + ')$')
MACREGEXP = re.compile(r'^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$')
def __init__(self, inputdata, hosttype, institute, header=None):
"""
parses a config file line of the form
#host=host1.abc.kit.edu hwadress=00:12:34:ab:cd:ef ipadress=127.0.0.1
"""
self._set_defaults()
self.vars['hosttype'] = hosttype
self.vars['institute'] = institute
self.groups = set()
if header:
for var, value in header.items():
self.vars[var] = value
self.groups.update(header.get('groups', {}))
for var, value in inputdata.items():
self.vars[var] = value
self.groups.update(inputdata.get('groups', {}))
self.groups.difference_update(inputdata.get('notgroups', {}))
if 'hostname' not in self.vars:
raise Exception("Entry without hostname.")
self.hostname = self.vars['hostname']
if not self.vars['institute']:
raise Exception("No institute given for %s." % self.hostname)
self.groups.update({
self.vars['hosttype'],
self.vars['institute'],
self.vars['institute'] + self.vars['hosttype']
})
self._check_macip()
self._set_fqdn()
self._set_publicip()
if header and 'iprange' in header:
self._check_iprange(header['iprange'])
self.header = header
logging.debug("Added " + str(self))
def _check_macip(self):
"Check validity of vars set for host."
try:
assert self.IPREGEXP.match(self.vars['ip'])
self.ip = ipaddress.ip_address(self.vars['ip'])
assert isinstance(self.ip, ipaddress.IPv4Address)
except:
raise Exception("Host %s does not have a valid IP address (%s)." % (self.hostname, self.vars['ip']))
if 'mac' in self.vars:
try:
assert self.MACREGEXP.match(self.vars['mac'])
self.mac = MAC(self.vars['mac'])
except:
raise Exception("Host %s does not have a valid MAC address (%s)." % (self.hostname, self.mac))
def _check_iprange(self, iprange):
"Check whether the given IP is in the range defined at the file header."
if iprange is None:
return
assert len(iprange) == 2
if self.ip < iprange[0] or self.ip > iprange[1]:
raise Exception("%s has IP %s outside of range %s-%s." %
(self.fqdn, self.ip, iprange[0], iprange[1]))
def run_checks(self):
checks = [
self._check_user(),
self._check_end_date(),
]
return all(checks)
def _check_end_date(self):
"Check that end_date is not over yet"
if 'end_date' in self.vars:
end_date = self.vars['end_date']
if not isinstance(end_date, datetime.date):
logging.error("Parsing of end_date %s led to non-date datatype %s for host %s." % (end_date, end_date.__class__, self.hostname))
return False
if end_date < datetime.date.today():
logging.error("Host end_date in the past for host %s." % self.hostname)
return False
return True
def _check_user(self):
"Check that user (still) exists if set"
if 'user' in self.vars:
try:
subprocess.check_output(['id', self.vars['user']])
except subprocess.CalledProcessError:
logging.error("User %s does not exist and is listed for host %s." % (self.vars['user'], self.hostname))
return False
return True
def filter(self, filter):
assert filter.__class__ == list
return self.hostname in filter or any([g in filter for g in self.groups])
class MAC(str):
"""Representation of a MAC address
Based on string, but always lowercase and replacing '-' with ':'.
"""
def __new__(cls, value):
return str.__new__(cls, value.lower().replace('-', ':'))
PK 4J:W# W# hostlist/hostlist.py#!/usr/bin/env python3
import logging
import types
from collections import defaultdict
import os
import sys
import ipaddress
import itertools
import glob
import yaml
try:
from yaml import CSafeLoader as SafeLoader
except ImportError:
from yaml import SafeLoader
from . import host
from .config import CONFIGINSTANCE as Config
class Hostlist(list):
def __init__(self):
super().__init__()
def __str__(self):
return '\n'.join([str(h) for h in self])
def diff(self, otherhostlist):
diff = types.SimpleNamespace()
diff.add, diff.remove = [], []
hostnames = {h.fqdn: h.ip for h in self if h.publicip}
inversehostlist = {h.fqdn: h for h in self}
otherhostnames = {h.fqdn: h.ip for h in otherhostlist if h.publicip}
inverseotherhostlist = {h.fqdn: h for h in otherhostlist}
for fqdn, ip in hostnames.items():
if otherhostnames.get(fqdn) != ip:
diff.add.append(inversehostlist[fqdn])
for fqdn, ip in otherhostnames.items():
if hostnames.get(fqdn) != ip:
diff.remove.append(inverseotherhostlist[fqdn])
diff.empty = (not diff.add) and (not diff.remove)
return diff
class DNSVSHostlist(Hostlist):
"Hostlist filed from DNSVS"
def __init__(self, con):
super().__init__()
hosts = con.get_hosts()
for hostname, data in hosts.items():
ip, is_nonunique = data
self.append(host.Host(hostname, ip, is_nonunique))
class YMLHostlist(Hostlist):
"Hostlist filed from yml file"
def __init__(self):
super().__init__()
self.fileheaders = {}
self.groups = defaultdict(list)
input_ymls = sorted(glob.glob(Config["hostlistdir"] + '/*.yml'))
logging.debug("Using %s" % ', '.join(input_ymls))
for inputfile in input_ymls:
self._add_ymlhostfile(inputfile)
def _add_ymlhostfile(self, fname):
"parse all hosts in fname and add them to this hostlist"
shortname = os.path.splitext(os.path.basename(fname))[0]
if shortname.count('-') > 1:
logging.error('Filename %s contains to many dashes. Skipped.')
return
if '-' in shortname:
# get abc, def from hostlists/abc-def.yml
hosttype, institute = shortname.split('-')
else:
hosttype = shortname
institute = None
try:
infile = open(fname, 'r')
except:
logging.error('file %s not readable' % fname)
return
try:
yamlsections = yaml.load_all(infile, Loader=SafeLoader)
except yaml.YAMLError as e:
logging.error('file %s not correct yml' % fname)
logging.error(str(e))
return
for yamlout in yamlsections:
self._parse_section(yamlout, fname, hosttype, institute)
self._fix_docker_ports()
def _parse_section(self, yamlout, fname, hosttype, institute):
for field in ('header', 'hosts'):
if field not in yamlout:
logging.error('missing field %s in %s' % (field, fname))
header = yamlout['header']
if 'iprange' in header:
ipstart, ipend = header['iprange']
header['iprange'] = ipaddress.ip_address(ipstart), ipaddress.ip_address(ipend)
self.fileheaders[os.path.basename(fname)] = header
for hostdata in yamlout["hosts"]:
newhost = host.YMLHost(hostdata, hosttype, institute, header)
self.append(newhost)
for group in newhost.groups:
self.groups[group].append(newhost)
def _fix_docker_ports(self):
for h in self:
if 'docker' in h.vars and 'ports' in h.vars['docker']:
# prefix docker ports with container IP
h.vars['docker']['ports'] = [
str(h.ip) + ':' + port for port in h.vars['docker']['ports']
]
def print(self, filter):
filtered = [h for h in self if h.filter(filter)]
for h in filtered:
if logging.getLogger().level == logging.DEBUG:
print(h.output(printgroups=True, printallvars=True))
elif logging.getLogger().level == logging.INFO:
print(h.output(delim='\t', printgroups=True))
else:
print(h.hostname)
def check_consistency(self, cnames):
checks = [
self.check_nonunique(),
self.check_cnames(cnames),
self.check_duplicates(),
self.check_missing_mac_ip(),
self.check_iprange_overlap(),
all(h.run_checks() for h in self),
]
logging.info("consistency check finished")
if not all(checks):
sys.exit(1)
def check_nonunique(self):
"""ensure nonunique flag agrees with nonunique_ips config"""
success = True
nonunique_ips = defaultdict(list)
for h in self:
ip_fit = str(h.ip) in Config["nonunique_ips"]
if ip_fit and h.vars['unique']:
nonunique_ips[str(h.ip)].append(h)
if not ip_fit and not h.vars['unique']:
logging.error("Host %s has nonunique ip flag, "
"but its ip is not listed in the config." % h)
success = False
for ip in nonunique_ips:
if len(nonunique_ips[ip]) > 1:
logging.error("More than one host uses a given nonunique ip"
" without being flagged:\n" +
('\n'.join((str(x) for x in nonunique_ips[ip]))))
success = False
return success
def check_cnames(self, cnames):
"""ensure there are no duplicates between hostlist and cnames"""
success = True
for cname in cnames:
has_dest = False
for h in self:
if h.fqdn == cname.fqdn:
logging.error("%s conflicts with %s." % (cname, h))
success = False
if cname.dest == h.fqdn:
has_dest = True
if not has_dest:
logging.error("%s points to a non-existing host." % cname)
success = False
return success
def check_duplicates(self):
"""check consistency of hostlist
detect duplicates (ip, mac, hostname)"""
success = True
inverselist = {}
tocheck_props = ['ip', 'mac', 'hostname']
for prop in tocheck_props:
inverselist[prop] = {}
for h in self:
myhostprop = getattr(h, prop)
if myhostprop is None:
continue
if prop == 'ip' and str(myhostprop) in Config["nonunique_ips"]:
# allow nonunique ips if listed in config
continue
if myhostprop in inverselist[prop]:
logging.error("Found duplicate %s for hosts \n%s\n%s"
% (prop, inverselist[prop][myhostprop], h))
success = False
inverselist[prop][myhostprop] = h
return success
def check_missing_mac_ip(self):
"""check if hosts are missing an ip or mac"""
success = True
for h in self:
if 'needs_ip' in h.vars and h.vars['needs_ip'] and h.ip is None:
logging.error("Missing IP in %s " % h)
success = False
if isinstance(self, YMLHostlist):
for h in self:
if h.vars['needs_mac'] and h.mac is None:
logging.error("Missing MAC in %s " % h)
success = False
return success
def check_iprange_overlap(self):
"check whether any of the ipranges given in headers overlap"
overlaps = []
for ita, itb in itertools.combinations(self.fileheaders.items(), 2):
filea, headera = ita
fileb, headerb = itb
try:
a = headera['iprange']
b = headerb['iprange']
except KeyError:
# one of the files does not have iprange defined, ignore it
continue
if ('iprange_allow_overlap' in headera and headera['iprange_allow_overlap']) or \
('iprange_allow_overlap' in headerb and headerb['iprange_allow_overlap']):
# FIXME: check overlap for internal IPs
continue
# check if there is overlap between a and b
overlap_low = max(a[0], b[0])
overlap_high = min(a[1], b[1])
if overlap_low <= overlap_high:
overlaps.append((overlap_low, overlap_high, filea, fileb))
if overlaps:
for overlap in overlaps:
logging.error("Found overlap from %s to %s in files %s and %s." % overlap)
return not bool(overlaps)
PK ӥ4JU hostlist/output_services.py#!/usr/bin/env python3
from collections import defaultdict
import os
from .config import CONFIGINSTANCE as Config
class Ssh_Known_HostsOutput:
"Generate hostlist for ssh-keyscan"
@classmethod
def gen_content(cls, hostlist, cnames):
# only scan keys on hosts that are in ansible
scan_hosts = [h for h in hostlist if h.vars.get('gen_ssh_known_hosts', False)]
aliases = [alias
for host in scan_hosts
for alias in host.aliases
if host.ip]
aliases += [str(host.ip)
for host in scan_hosts
if host.ip]
aliases += [cname.fqdn for cname in cnames]
fcont = '\n'.join(aliases)
return fcont
class HostsOutput:
"Config output for /etc/hosts format"
@classmethod
def gen_content(cls, hostlist, cnames):
hoststrings = (str(h.ip) + " " + " ".join(h.aliases) for h in hostlist if h.ip)
content = '\n'.join(hoststrings)
return content
class MuninOutput:
"Config output for Munin"
@classmethod
def gen_content(cls, hostlist, cnames):
hostnames = (h for h in hostlist if h.vars.get('gen_munin', False))
fcont = ''
for host in hostnames:
fcont += cls._get_hostblock(host)
return fcont
@staticmethod
def _get_hostblock(host):
cont = '[{institute}{hosttype};{h}]\naddress {h}\n'.format(
h=host.fqdn,
institute=host.vars['institute'],
hosttype=host.vars['hosttype'],
)
if 'munin' in host.vars:
for line in host.vars['munin']:
cont += line + '\n'
return cont
class DhcpOutput:
"DHCP config output"
@classmethod
def gen_content(cls, hostlist, cnames):
out = ""
for host in hostlist:
entry = cls._gen_hostline(host)
if not entry:
continue
out += entry + '\n'
return out
@staticmethod
def _gen_hostline(host):
if host.mac and host.ip:
# curly brackets doubled for python format function
return """host {fqdn} {{
hardware ethernet {mac};
fixed-address {ip};
option host-name "{hostname}";
option domain-name "{domain}";
}}""".format(fqdn=host.fqdn, mac=host.mac, ip=host.ip, hostname=host.hostname, domain=host.domain)
class AnsibleOutput:
"Ansible inventory output"
@classmethod
def gen_content(cls, hostlist, cnames):
"""generate json inventory for ansible
form:
{
"_meta": {
"hostvars": {
"myhost.abc.kit.edu": {
"hosttype": "desktop",
"institute": "abc",
"custom_variable": "foo",
}
...
}
},
"groupname" : [
"myhost2.abc.kit.edu",
]
...
"""
resultdict = defaultdict(lambda: {'hosts': []})
hostvars = {}
docker_services = {}
for host in hostlist:
# online add hosts that have ansible=yes
if 'ansible' in host.vars and not host.vars['ansible']:
continue
ans = cls._gen_host_content(host)
hostvars[ans['fqdn']] = ans['vars']
for groupname in ans['groups']:
resultdict[groupname]['hosts'] += [ans['fqdn']]
if ans['vars']['hosttype'] == 'docker':
resultdict['dockerhost-' + host.hostname]['hosts'] += [ans['vars']['docker']['host']]
docker_services[host.hostname] = ans['vars']['docker']
docker_services[host.hostname]['fqdn'] = host.fqdn
docker_services[host.hostname]['ip'] = str(host.ip)
resultdict['_meta'] = {'hostvars': hostvars}
if docker_services:
resultdict['vserverhost']['vars'] = {'docker_services': docker_services}
return resultdict
@staticmethod
def _gen_host_content(host):
"Generate output for one host"
result = {
'fqdn': host.fqdn,
'groups': host.groups,
'vars': {},
}
if host.ip:
result['vars']['ip'] = str(host.ip)
ansiblevars = Config.get('ansiblevars', []) + ['hosttype', 'institute', 'docker']
for avar in ansiblevars:
if avar in host.vars:
result['vars'][avar] = host.vars[avar]
return result
class EthersOutput:
"/etc/ethers format output"
@classmethod
def gen_content(cls, hostlist, cnames):
entries = (
"%s %s" % (h.mac, alias)
for h in hostlist
for alias in h.aliases
if h.mac
)
out = '\n'.join(entries)
return out
class WebOutput:
"HTML Table of hosts"
@classmethod
def gen_content(cls, hostlist, cnames):
if os.path.exists('header.html'):
with open('header.html') as file:
header = file.read()
else:
header = ''
fields = Config.get('weboutput_columns', ['institute', 'hosttype', 'hostname'])
thead = '| ' + ' | '.join(fields) + ' |
|---|
\n'
footer = '
'
hostlist = '\n'.join(
'| ' + ' | '.join(str(h.vars.get(field, '')) for field in fields) + ' |
'
for h in hostlist
)
return header + thead + hostlist + footer
PK {/J hostlist/dnsvs/__init__.pyPK {/Jf>" >" hostlist/dnsvs/dnsvs_webapi.py#!/usr/bin/python3
import requests
import json
import os.path
import logging
from .. import host
from .. import cnamelist
class dnsvs_interface:
certfilename = '~/.ssl/net-webapi.key'
certfilename = os.path.expanduser(certfilename)
root_url = 'https://www-net.scc.kit.edu/api/2.0/dns'
geturl = root_url + '/record/list'
createurl = root_url + '/record/create'
deleteurl = root_url + '/record/delete'
# all our entries ar IPv4
inttype_a = "host:0100,:,402,A"
inttype_nonunique = "domain:1000,:,400,A"
inttype_cname = "alias:0000,host:0100,011,CNAME"
headers_dict = {"Content-Type": "application/json"}
def _execute(self, url, method, data=None):
"""Actually perform an operation on the DNS server."""
try:
if method == "get":
response = requests.get(url=url, headers=self.headers_dict, cert=self.certfilename)
elif method == "post":
response = requests.post(url=url, data=data, headers=self.headers_dict, cert=self.certfilename)
if response.ok:
return response.json()
else:
raise requests.exceptions.RequestException(response.status_code, response.text)
except Exception as e:
logging.error(e)
raise
return json.dumps({})
def get_hosts(self):
"""Reads A records from the server."""
result = self._execute(self.geturl, method="get")
# continue with normal request (process result)
hosts = {}
for entry in result:
fqdn = entry['fqdn'].rstrip(".")
if entry['type'] == 'A':
is_nonunique = entry['inttype'] == self.inttype_nonunique
hosts[fqdn] = (entry['data'], is_nonunique)
return hosts
def get_cnames(self):
"""Reads CNAME records from the server."""
result = self._execute(self.geturl, method="get")
# continue with normal request (process result)
cname = {}
for entry in result:
fqdn = entry['fqdn'].rstrip(".")
if entry['type'] == 'CNAME':
cname[fqdn] = entry['data'].rstrip(".")
return cname
def add(self, entry):
"""generic interface to add_*"""
if isinstance(entry, host.Host):
self.add_host(entry)
elif isinstance(entry, cnamelist.CName):
self.add_cname(entry)
def remove(self, entry):
"""generic interface to remove_*"""
if isinstance(entry, host.Host):
self.remove_host(entry)
elif isinstance(entry, cnamelist.CName):
self.remove_cname(entry)
def add_host(self, host):
"""Adds an A record to the server."""
# TODO: handle these errors in the response
# check whether there is already a CNAME with that fqdn
# url = self.root_url+"/record/list?type=CNAME&fqdn="+host.fqdn+"."
# dependencies = self._execute(url=url, method="get")
# if dependencies!=[]:
# raise Exception('Attempting to overwrite an existing CNAME record in DNSVS with an A record!')
# url = self.root_url+"/record/list?type=A&fqdn="+host.fqdn+"."
# dependencies = self._execute(url=url, method="get")
# if dependencies!=[]:
# if dependencies[0]['data']==str(host.ip):
# logging.warning('Attempting to add already an existing A record.')
# return
# elif dependencies[0]['data']!=str(host.ip):
# raise Exception('Attempting to overwrite an existing A record with a different one.')
inttype = self.inttype_nonunique if not host.vars['unique'] else self.inttype_a
data = [
{"param_list": [
{"name": "fqdn", "new_value": host.fqdn + "."},
{"name": "data", "new_value": str(host.ip)},
{"name": "inttype", "new_value": inttype},
]},
]
json_string = json.dumps(data)
self._execute(url=self.createurl, method="post", data=json_string)
def remove_host(self, host):
"""Remove an A record from the server."""
# TODO: handle these errors in the response
# before removing, check whether a cname points to that record
# https://www-net.scc.kit.edu/api/2.0/dns/record/list?target_fqdn_regexp=ttpseth.ttp.kit.edu.
# url = self.root_url+"/record/list?type=CNAME&target_fqdn="+host.fqdn+"."
# dependencies = self._execute(url=url, method="get")
# if dependencies!=[]:
# raise Exception('Attempting to remove an A record of a host to which a cname ist pointing.')
# url = self.root_url+"/record/list?type=A&fqdn="+host.fqdn+"."
# dnsvs_records = self._execute(url=url, method="get")
# if dnsvs_records!=[]:
# if dnsvs_records[0]['data']!=str(host.ip):
# raise Exception('Attempting to remove an existing A record, for which the IP adress does not match.')
# else:
# logging.warning('Attempting to remove a nonexistent A record.')
# return
# remove
# TODO: can we use host.fqdn/host.ip here?
# trust our data more then theirs
inttype = self.inttype_a if host.vars['unique'] else self.inttype_nonunique
data = [
{"param_list": [
{"name": "fqdn", "old_value": host.fqdn + "."},
{"name": "data", "old_value": str(host.ip)},
{"name": "inttype", "old_value": inttype},
]},
]
json_string = json.dumps(data)
self._execute(url=self.deleteurl, method="post", data=json_string)
def add_cname(self, cname):
"""Adds a CNAME record given by (alias, hostname) to the server."""
fqdn, dest = cname.fqdn, cname.dest
# TODO: handle these errors in the response
# check whether the cname record is already there
# url = self.root_url+"/record/list?type=CNAME&target_fqdn_regexp="+dest+"."+"&fqdn="+fqdn+"."
# dependencies = self._execute(url=url, method="get")
# if dependencies!=[]:
# logging.warning('Attempting to add an already existing CNAME record to DNSVS!')
# return
# else:
# # check whether there is a different CNAME record for the same fqdn
# url = self.root_url+"/record/list?type=CNAME&fqdn="+fqdn+"."
# dependencies = self._execute(url=url, method="get")
# if dependencies!=[]:
# raise Exception('Attempting to overwrite an existing CNAME record in DNSVS with a different one!')
# # check whether there is an A record with the same fqdn
# url = self.root_url+"/record/list?type=A&fqdn="+fqdn+"."
# dependencies = self._execute(url=url, method="get")
# if dependencies!=[]:
# raise Exception('Attempting to overwrite an A record in DNSVS with a CNAME record!')
# url = self.root_url+"/record/list?type=A&fqdn="+dest+"."
# inarecords = self._execute(url=url, method="get")
# url = self.root_url+"/record/list?type=CNAME&fqdn="+fqdn+"."
# incnames = self._execute(url=url, method="get")
# if inarecords==[] and incnames==[]:
# raise Exception('Attempting to add a CNAME record do DNSVS pointing to a nonexistent fqdn!')
# write
data = [
{"param_list": [
{"name": "fqdn", "new_value": fqdn + "."},
{"name": "data", "new_value": dest + "."},
{"name": "inttype", "new_value": self.inttype_cname},
]},
]
json_string = json.dumps(data)
self._execute(url=self.createurl, method="post", data=json_string)
def remove_cname(self, cname):
"""Remove a CNAME record from the server."""
fqdn, dest = cname.fqdn, cname.dest
# TODO: handle these errors in the response
# check whether the cname record is there in the first place
# url = self.root_url+"/record/list?type=CNAME&target_fqdn_regexp="+dest+"."+"&fqdn="+fqdn+"."
# dependencies = self._execute(url=url, method="get")
# if dependencies == []:
# logging.warning('Attempting to remove a nonexistent CNAME record from DNSVS!')
# return
# remove
data = [
{"param_list": [
{"name": "fqdn", "old_value": fqdn + "."},
{"name": "data", "old_value": dest + "."},
{"name": "inttype", "old_value": self.inttype_cname},
]},
]
json_string = json.dumps(data)
self._execute(url=self.deleteurl, method="post", data=json_string)
PK 4J hostlist/dnsvs/sync.py#!/usr/bin/env python3
import logging
from . import dnsvs_webapi as dnsvs
from ..cnamelist import CName
from ..host import Host
# use termcolor when available, otherwise ignore
try:
from termcolor import colored
except ImportError:
def colored(text, _):
return text
def apply_diff(diff):
con = dnsvs.dnsvs_interface()
removelist = \
list(filter(lambda h: isinstance(h, CName), diff.remove)) + \
list(filter(lambda h: isinstance(h, Host), diff.remove))
addlist = \
list(filter(lambda h: isinstance(h, Host), diff.add)) + \
list(filter(lambda h: isinstance(h, CName), diff.add))
for entry in removelist:
logging.info('removing\t' + str(entry))
con.remove(entry)
for entry in addlist:
logging.info('adding\t' + str(entry))
con.add(entry)
def print_diff(diff):
if diff.add:
print(colored("Only in local files: ", 'green'))
for h in sorted(diff.add, key=lambda h: h.fqdn):
print(colored('+' + str(h), 'green'))
if diff.remove:
print(colored("Only in DNSVS: ", 'red'))
for h in sorted(diff.remove, key=lambda h: h.fqdn):
print(colored('-' + str(h), 'red'))
PK !HV\|ML ) hostlist-1.2.3.dist-info/entry_points.txtN+I/N.,()JLI/.QU Q9%zP!<̜̜bd%Q*nJbjn~ RD PK !H;@Q P hostlist-1.2.3.dist-info/WHEEL1
0 RZq+D-Dv;_[*7Fp ܦpv/fݞ oL(*IPK !H p ! hostlist-1.2.3.dist-info/METADATAuW]w6}ǯ)JTCݳ> o6}!" %3 $%7ՃM̝; QU*Z~]|#nTײv!6&Dqx%U~X˻T4U[[ ˠ:'ev?.Wu)J:)7q5 ;m_n~yF\v~-?뼴ԭ2ZzPx|'4{o ߿
lFyU\R DIݕ:7ȷjyoSN҇WX4*j/_}wxחgjβPulp>Q
G]Ѷdu+/X Ut>m{hnŮ[ozxglt&~:=k!ČkAp-] :|C7<_kb=?ne*zA 4Tll6qĢ⌻|Ό_)!b9xڮPᬬӻ\|<,B.$Joռ
ʹs!LdpơIn=>}9H
!*> z] E7!Ǜ *|-ĩ\.ktdpu@
q Ń_2}'=6!m.-YJ@<>bh ::'+%|J YIVUSdJ xN@X}fUG BDdr_ڋ9)O_ѷeB=& _FlD= (w־UrZsA2}
%'jʞ
kp6;J)ULVrVxKw;p) C"@ΙGj!MȀZ `$yewzO3~_A!_Ř p 7$!^+CXZWU5 Y"B LIP"} qBQ\ r_9ێ1U4wO&%
2ZLn;xYkT^1&ӵR4=0@mˁ@d8 ,6p-D]]R^ݦ
!9m 3hY
[z7*,<
8TzxS`eEQ|JQ0~KhdoA
]֠3C5ʉ #NrpZ?KI#X&|y6\q$ F]0'*0ਣ?T~<#Zϓ)(h }w A<'<+J)r
I6utX>M
oqI*̣0݀cʲȗ6U'+a:fF
ˣިE99lcM[G;ߗ<4/~2
L{ U mOB~;4v/0oѵISLdq)8h}-hPH4nڊ IgJ 4|C"L݄>2=_t E;k^SW)w
=XGְk}JSTh 8'GR.nG$GJ4 U0*no~+_ 5ݪm&-M݀a,ceGPK !H hostlist-1.2.3.dist-info/RECORD}ɒ:} a1xP짿Tm+FuVT4TQ$Q4k .bN~ L IdsS{.7DDJKP\j9Ѹ[
@ 0$O}nHx *dv2hҠ 0*ߩʜA=onXAm;{kNi"14:_+n 1]5'TpzC 쌛(k~Dm "xFFY; Gv2 X]wxK _c`rڽ:(o,,k>赏__ m.$ncv VF&WJazQ32=0Zk=2å^2^}2fVmrw/J\woIQ?PK 4J hostlist/__init__.pyPK +JIrE3 3 hostlist/addhost.pyPK *4J<} } F hostlist/buildfiles.pyPK {/J
1 hostlist/cnamelist.pyPK uI`8 < hostlist/config.pyPK ́4J~$ $ ? hostlist/daemon.pyPK 4J
HN hostlist/host.pyPK 4J:W# W# wj hostlist/hostlist.pyPK ӥ4JU hostlist/output_services.pyPK {/J 1 hostlist/dnsvs/__init__.pyPK {/Jf>" >" i hostlist/dnsvs/dnsvs_webapi.pyPK 4J hostlist/dnsvs/sync.pyPK !HV\|ML ) hostlist-1.2.3.dist-info/entry_points.txtPK !H;@Q P p hostlist-1.2.3.dist-info/WHEELPK !H p ! hostlist-1.2.3.dist-info/METADATAPK !H hostlist-1.2.3.dist-info/RECORDPK j