PKق-K(Froro/__init__.py"""roro - commandline tool for accessing all the sevices in RorodataPlatform. """ from .projects import get_current_project __version__ = '0.1.0' PKق-K; =3"3" roro/cli.pyimport os import stat import time import itertools import click import datetime from netrc import netrc from tabulate import tabulate from . import projects from . import helpers as h from .helpers import get_host_name from .projects import Project, login as roro_login from .path import Path from firefly.client import FireflyError from requests import ConnectionError class PathType(click.ParamType): name = 'path' def convert(self, value, param, ctx): return Path(value) class CatchAllExceptions(click.Group): def __call__(self, *args, **kwargs): try: return self.main(*args, **kwargs) except Exception as exc: click.echo('ERROR %s' % exc) @click.group(cls=CatchAllExceptions) def cli(): pass @cli.command() @click.option('--email', prompt='Email address') @click.option('--password', prompt=True, hide_input=True) def login(email, password): """Login to rorodata platform. """ netrc_file = create_netrc_if_not_exists() try: token = roro_login(email, password) rc = netrc() _fix_netrc(rc) with open(netrc_file, 'w') as f: host_name = get_host_name(projects.SERVER_URL) rc.hosts[host_name] = (email, None, token) f.write(str(rc)) except ConnectionError: click.echo('unable to connect to the server, try again later') except FireflyError as e: click.echo(e) # this is here because of this issue # https://github.com/python/cpython/pull/2491 # TODO: should be removed once it is fixed def _fix_netrc(rc): for host in rc.hosts: login, _, password = rc.hosts[host] # should be safe as we use alphanumneric chars in token login = login.strip("'") password = password.strip("'") rc.hosts[host] = (login, _, password) def create_netrc_if_not_exists(): prefix = '.' if os.name != 'nt' else '_' netrc_file = os.path.join(os.path.expanduser('~'), prefix+'netrc') # theese flags works both on windows and linux according to this stackoverflow # https://stackoverflow.com/questions/27500067/chmod-issue-to-change-file-permission-using-python#27500153 if not os.path.exists(netrc_file): with open(netrc_file, 'w'): pass os.chmod(netrc_file, stat.S_IREAD|stat.S_IWRITE) return netrc_file @cli.command(name="projects") def _projects(): """Lists all the projects. """ projects = Project.find_all() for p in projects: print(p.name) @cli.command() @click.argument('project') def create(project): """Creates a new Project. """ p = Project(project) p.create() print("Created project:", project) @cli.command() def deploy(): """Pushes the local changes to the cloud and restarts all the services. """ # TODO: validate credentials project = projects.current_project() response = project.deploy() click.echo(response) @cli.command() @click.argument('src', type=PathType()) @click.argument('dest', type=PathType()) def cp(src, dest): """Copy files to and from volumes to you local disk. Example: $ roro cp volume:/dataset.txt ./dataset.txt downloads the file dataset.txt from the server $ roro cp ./dataset.txt volume:/dataset.txt uploads dataset.txt to the server """ if src.is_volume() is dest.is_volume(): raise Exception('One of the arguments has to be a volume, other a local path') project = projects.current_project() project.copy(src, dest) @cli.command() @click.option('-a', '--all', default=False, is_flag=True) def ps(all): """Shows all the processes running in this project. """ project = projects.current_project() jobs = project.ps(all=all) rows = [] for job in jobs: start = h.parse_time(job['start_time']) end = h.parse_time(job['end_time']) total_time = (end - start) total_time = datetime.timedelta(total_time.days, total_time.seconds) command = " ".join(job["details"]["command"]) rows.append([job['jobid'], job['status'], h.datestr(start), str(total_time), job['instance_type'], h.truncate(command, 50)]) print(tabulate(rows, headers=['JOBID', 'STATUS', 'WHEN', 'TIME', 'INSTANCE TYPE', 'CMD'], disable_numparse=True)) @cli.command(name='ps:restart') @click.argument('name') def ps_restart(name): """Restarts the service specified by name. """ pass @cli.command() def config(): """Lists all config vars of this project. """ project = projects.current_project() config = project.get_config() print("=== {} Config Vars".format(project.name)) for k, v in config.items(): print("{}: {}".format(k, v)) @cli.command(name='config:set') @click.argument('vars', nargs=-1) def env_set(vars): """Sets one or more the config vars. """ project = projects.current_project() d = {} for var in vars: if "=" in var: k, v = var.split("=", 1) d[k] = v else: d[var] = "" project.set_config(d) print("Updated config vars") @cli.command(name='config:unset') @click.argument('names', nargs=-1) def env_unset(names): """Unsets one or more config vars. """ project = projects.current_project() project.unset_config(names) print("Updated config vars") @cli.command(context_settings={"allow_interspersed_args": False}) @click.argument('command', nargs=-1) def run(command): """Runs the given script in foreground. """ project = projects.current_project() job = project.run(command) print("Started new job", job["jobid"]) @cli.command(name='run:notebook', context_settings={"allow_interspersed_args": False}) def run_notebook(): """Runs a notebook. """ project = projects.current_project() job = project.run_notebook() _logs(project, job["jobid"], follow=True, end_marker="-" * 40) @cli.command() @click.argument('jobid') def stop(jobid): """Stops a service with reference to jobid """ project = projects.current_project() project.stop(jobid) @cli.command() @click.argument('jobid') @click.option('-s', '--show-timestamp', default=False, is_flag=True) @click.option('-f', '--follow', default=False, is_flag=True) def logs(jobid, show_timestamp, follow): """Shows all the logs of the project. """ project = projects.current_project() _logs(project, jobid, follow, show_timestamp) def _logs(project, job_id, follow=False, show_timestamp=False, end_marker=None): """Shows the logs of job_id. """ def get_logs(job_id, follow=False): if follow: seen = 0 while True: logs = project.logs(job_id) for log in logs[seen:]: yield log seen = len(logs) job = project.ps(job_id) if job['status'] in ['success', 'cancelled', 'failed']: break time.sleep(0.5) else: logs = project.logs(job_id) for log in logs: yield log logs = get_logs(job_id, follow) if end_marker: logs = itertools.takewhile(lambda log: not log['message'].startswith(end_marker), logs) _display_logs(logs, show_timestamp=show_timestamp) def _display_logs(logs, show_timestamp=False): def parse_time(timestamp): t = datetime.datetime.fromtimestamp(timestamp//1000) return t.isoformat() if show_timestamp: log_pattern = "[{timestamp}] {message}" else: log_pattern = "{message}" for line in logs: line['timestamp'] = parse_time(line['timestamp']) click.echo(log_pattern.format(**line)) @cli.command() @click.argument('project') def project_logs(project): """Shows all the logs of the process with name the project. """ pass @cli.command() def volumes(): """Lists all the volumes. """ project = projects.current_project() volumes = project.list_volumes() if not volumes: click.echo('No volumes are attached to {}'.format(project.name)) for volume in project.list_volumes(): click.echo(volume) @cli.command(name='volumes:add') @click.argument('volume_name') def create_volume(volume_name): """Creates a new volume. """ project = projects.current_project() volume = project.add_volume(volume_name) click.echo('Volume {} added to the project {}'.format(volume, project.name)) @cli.command(name='volumes:remove') @click.argument('volume_name') def remove_volume(volume_name): """Removes a new volume. """ pass def main_dev(): projects.SERVER_URL = "http://api.local.rorodata.com:8080/" cli() PK-Knm>>roro/helpers.pyimport datetime import urllib def parse_time(timestr): if not timestr: return datetime.datetime.utcnow() try: return datetime.datetime.strptime(timestr, "%Y-%m-%d %H:%M:%S.%f") except ValueError: return datetime.datetime.strptime(timestr, "%Y-%m-%d %H:%M:%S") def datestr(then, now=None): """Converts time to a human readable string. Wrapper over web.datestr. >>> from datetime import datetime >>> datestr(datetime(2010, 1, 2), datetime(2010, 1, 1)) '1 day ago' """ s = _web_datestr(then, now) if 'milliseconds' in s or 'microseconds' in s: s = 'Just now' return s def _web_datestr(then, now=None): """ datestr utility from web.py (public domain). source: https://github.com/webpy/webpy Converts a (UTC) datetime object to a nice string representation. >>> from datetime import datetime, timedelta >>> d = datetime(1970, 5, 1) >>> datestr(d, now=d) '0 microseconds ago' >>> for t, v in iteritems({ ... timedelta(microseconds=1): '1 microsecond ago', ... timedelta(microseconds=2): '2 microseconds ago', ... -timedelta(microseconds=1): '1 microsecond from now', ... -timedelta(microseconds=2): '2 microseconds from now', ... timedelta(microseconds=2000): '2 milliseconds ago', ... timedelta(seconds=2): '2 seconds ago', ... timedelta(seconds=2*60): '2 minutes ago', ... timedelta(seconds=2*60*60): '2 hours ago', ... timedelta(days=2): '2 days ago', ... }): ... assert datestr(d, now=d+t) == v >>> datestr(datetime(1970, 1, 1), now=d) 'January 1' >>> datestr(datetime(1969, 1, 1), now=d) 'January 1, 1969' >>> datestr(datetime(1970, 6, 1), now=d) 'June 1, 1970' >>> datestr(None) '' """ def agohence(n, what, divisor=None): if divisor: n = n // divisor out = str(abs(n)) + ' ' + what # '2 day' if abs(n) != 1: out += 's' # '2 days' out += ' ' # '2 days ' if n < 0: out += 'from now' else: out += 'ago' return out # '2 days ago' oneday = 24 * 60 * 60 if not then: return "" if not now: now = datetime.datetime.utcnow() if type(now).__name__ == "DateTime": now = datetime.datetime.fromtimestamp(now) if type(then).__name__ == "DateTime": then = datetime.datetime.fromtimestamp(then) elif type(then).__name__ == "date": then = datetime.datetime(then.year, then.month, then.day) delta = now - then deltaseconds = int(delta.days * oneday + delta.seconds + delta.microseconds * 1e-06) deltadays = abs(deltaseconds) // oneday if deltaseconds < 0: deltadays *= -1 # fix for oddity of floor if deltadays: if abs(deltadays) < 4: return agohence(deltadays, 'day') # Trick to display 'June 3' instead of 'June 03' # Even though the %e format in strftime does that, it doesn't work on Windows. out = then.strftime('%B %d').replace(" 0", " ") if then.year != now.year or deltadays < 0: out += ', %s' % then.year return out if int(deltaseconds): if abs(deltaseconds) > (60 * 60): return agohence(deltaseconds, 'hour', 60 * 60) elif abs(deltaseconds) > 60: return agohence(deltaseconds, 'minute', 60) else: return agohence(deltaseconds, 'second') deltamicroseconds = delta.microseconds if delta.days: deltamicroseconds = int(delta.microseconds - 1e6) # datetime oddity if abs(deltamicroseconds) > 1000: return agohence(deltamicroseconds, 'millisecond', 1000) return agohence(deltamicroseconds, 'microsecond') def truncate(text, width): if len(text) > width: text = text[:width-3] + "..." return text def get_host_name(url): host = urllib.parse.urlparse(url).netloc host_name = host.split(':')[0] return host_name PK-KX5]roro/models.py def get_model_repository(project, name): """Returns the ModelRepository with given name from the specified project. :param project: the name of the project :param name: name of the repository """ return ModelRepository(project, name) class ModelRepository: def __init__(self, project, name): self.project = project self.name = name def get_model_image(self, tag="latest", version=None): pass def new_model_image(self, model, metadata={}): """Creates a new ModelImage. """ class ModelImage: def __init__(self, repo, metadata, model=None, comment=None): self._repo = repo self._model = model self._metadata = metadata self.comment = comment @property def id(self): return self._metadata.get("Model-Id") @property def version(self): return self._metadata["Model-Version"] @property def name(self): return self._metadata["Model-Name"] def __getitem__(self, name): return self._metadata[name] def __setitem__(self, name, value): self._metadata[name] = value def get(self, name, default=None): return self._metadata.get(name, default) def get_model(self): pass def save(self, comment): """Saves a new version of the model image. """ if self.id is not None: raise Exception("ModelImage can't be modified once created.") PK-KCv88 roro/path.pyimport shutil import pathlib class Path: def __init__(self, path): self.volume = None if ':' in path: self.volume, self.path = path.split(':') else: self.path = path self._path = pathlib.Path(self.path) def is_volume(self): return self.volume is not None def open(self, *args, **kwargs): return self._path.open(*args, **kwargs) def safe_write(self, fileobj): if not self._path.parent.exists(): raise FileNotFoundError('Directory {} does not exist'.format(self._path.parent.absolute)) p = self._path.with_suffix('.tmp') with p.open('wb') as f: shutil.copyfileobj(fileobj, f) p.rename(self._path) @property def size(self): return self._path.stat().st_size PK-KUăroro/projects.pyimport os import shutil import yaml import firefly from . import models from click import ClickException SERVER_URL = "https://api.rorodata.com/" def login(email, password): client = firefly.Client(SERVER_URL) return client.login(email=email, password=password) class Project: def __init__(self, name, runtime=None): self.name = name self.runtime = runtime self.client = firefly.Client(SERVER_URL) def create(self): return self.client.create(name=self.name) def run(self, command): job = self.client.run(project=self.name, command=command) return job def stop(self, jobid): self.client.stop(project=self.name, jobid=jobid) def run_notebook(self): job = self.client.run_notebook(project=self.name) return job def ps(self, all=False, jobid=None): return self.client.ps(project=self.name, jobid=jobid, all=all) def logs(self, jobid): return self.client.logs(project=self.name, jobid=jobid) #return self.client.logs(project=self.name) def deploy(self): archive = self.archive() size = os.path.getsize(archive) with open(archive, 'rb') as f: format = 'tar' response = self.client.deploy( project=self.name, archived_project=f, size=size, format=format ) return response def archive(self, format='tar'): root_dir = os.path.realpath(os.path.curdir) dir_name = os.path.basename(root_dir) return shutil.make_archive(dir_name, format) def get_config(self): return self.client.get_config(project=self.name) def set_config(self, config_vars): return self.client.set_config(project=self.name, config_vars=config_vars) def unset_config(self, names): return self.client.unset_config(project=self.name, names=names) def list_volumes(self): volumes = self.client.volumes(project=self.name) return [volume['volume'] for volume in volumes] def add_volume(self, volume_name): volume = self.client.add_volume(project=self.name, name=volume_name) return volume['volume'] def get_model_repository(self, name): """Returns the ModelRepository from this project with given name. """ return models.get_model_repository(project=self.name, name=name) def copy(self, src, dest): if src.is_volume(): self._get_file(src, dest) else: self._put_file(src, dest) def _get_file(self, src, dest): fileobj = self.client.get_file( project=self.name, volume=src.volume, path=src.path ) dest.safe_write(fileobj) def _put_file(self, src, dest): with src.open('rb') as fileobj: self.client.put_file( project=self.name, fileobj=fileobj, volume=dest.volume, path=dest.path, size=src.size ) @staticmethod def find_all(): client = firefly.Client(SERVER_URL) projects = client.projects() return [Project(p['name'], p.get('runtime')) for p in projects] def current_project(): if os.path.exists("roro.yml"): d = yaml.safe_load(open("roro.yml")) return Project(d['project'], d.get('runtime', 'python36')) else: raise ClickException("Unable to find roro.yml") get_current_project = current_project def list_projects(): return Project.find_all() PK!H4D%roro-0.1.0.dist-info/entry_points.txtN+I/N.,()*/MI-SU1s2r3b`Id PK!H١Wdroro-0.1.0.dist-info/WHEEL HM K-*ϳR03rOK-J,/RH,Q0343 /, (-JLR()*M ILR(4KM̫#DPK!HQQ roro-0.1.0.dist-info/METADATAePKO1WLEH D&JemcmgM;D1ow|ŘLDa1 NK;p AB;k<`tUaJ7j~xذ1o\[M,vJ lTF*>G5I,yOlx)ǷR,zjC |v m_a6N}ZgWj<>!JΈ0ֺʁ9SdPK!H4Broro-0.1.0.dist-info/RECORDuGr`@2 " U$:fON'g/{c?aXua2W30251WYNx)w!?Nݎ]k.{0cMrFd'q̳,Ri[)gqzq4o#*l)dH/Kg^ij 2fi7dkܰ͋Q Yk)6z2C4/J8vRlV8Y4/TX"4ƾΒ5F)EMi;J8}';1ufNJkБDrHi.߳np ď)]tGy!L1\dAqWo{ hߠ񐀗b)#A브_aD:tuc~m VlgmR[VA/2{⋊E=jX3WpNWPKق-K(Froro/__init__.pyPKق-K; =3"3" roro/cli.pyPK-Knm>>#roro/helpers.pyPK-KX5]3roro/models.pyPK-KCv88 {9roro/path.pyPK-KUă<roro/projects.pyPK!H4D%%Kroro-0.1.0.dist-info/entry_points.txtPK!H١WdKroro-0.1.0.dist-info/WHEELPK!HQQ +Lroro-0.1.0.dist-info/METADATAPK!H4BrMroro-0.1.0.dist-info/RECORDPK O