PK;K^Z@roro/__init__.py"""roro - commandline tool for accessing all the sevices in RorodataPlatform. """ from .projects import get_current_project __version__ = '0.1.5' PK;K[  roro/auth.pyimport os import sys import pathlib import stat from netrc import netrc as _netrc import firefly from . import config from .helpers import get_host_name, PY2 def login(email, password): token = just_login(email, password) save_token(email, token) def just_login(email, password): client = firefly.Client(config.SERVER_URL) token = client.login(email=email, password=password) return token def whoami(): client = firefly.Client(config.SERVER_URL) return client.whoami() def get_saved_login(): create_netrc_if_not_exists() rc = netrc() hostname = get_host_name(config.SERVER_URL) if hostname in rc.hosts: login, _, password = rc.hosts[hostname] return { "email": login, "password": password } def save_token(email, token): netrc_file = create_netrc_if_not_exists() rc = netrc() with open(netrc_file, 'w') as f: host_name = get_host_name(config.SERVER_URL) if PY2: email = email.encode('utf-8') token = token.encode('utf-8') rc.hosts[host_name] = (email, None, token) f.write(str(rc)) def create_netrc_if_not_exists(): prefix = '.' if os.name != 'nt' else '_' netrc_file = os.path.join(os.path.expanduser('~'), prefix+'netrc') # theese flags works both on windows and linux according to this stackoverflow # https://stackoverflow.com/questions/27500067/chmod-issue-to-change-file-permission-using-python#27500153 if not os.path.exists(netrc_file): with open(netrc_file, 'w'): pass os.chmod(netrc_file, stat.S_IREAD|stat.S_IWRITE) return netrc_file class netrc(_netrc): """Extended the netrc from standard library to fix an issue. See https://github.com/python/cpython/pull/2491 """ def __init__(self, file=None): # The stdlib netrc doesn't find the right netrc file by default # work-around to fix that if file is None: file = self.find_default_file() super().__init__(file=file) def find_default_file(self): filename = "_netrc" if sys.platform == 'win32' else ".netrc" p = pathlib.Path.home().joinpath(filename) return str(p) def __repr__(self): """Dump the class data in the format of a .netrc file.""" rep = "" for host in self.hosts.keys(): attrs = self.hosts[host] rep = rep + "machine "+ host + "\n\tlogin " + attrs[0] + "\n" if attrs[1]: rep = rep + "account " + attrs[1] rep = rep + "\tpassword " + attrs[2] + "\n" for macro in self.macros.keys(): rep = rep + "macdef " + macro + "\n" for line in self.macros[macro]: rep = rep + line rep = rep + "\n" return rep PKy;K)x'x' roro/cli.pyfrom __future__ import print_function import time import itertools import click import datetime import sys from tabulate import tabulate from . import config from . import projects from . import helpers as h from .projects import Project from . import auth from .path import Path from . import __version__ from firefly.client import FireflyError from requests import ConnectionError class PathType(click.ParamType): name = 'path' def convert(self, value, param, ctx): return Path(value) class CatchAllExceptions(click.Group): def __call__(self, *args, **kwargs): try: return self.main(*args, **kwargs) except FireflyError as e: if e.args and e.args[0] == "Forbidden": click.echo("Unauthorized. Please login and try again.") sys.exit(2) else: click.echo('ERROR %s' % e) sys.exit(3) except Exception as exc: click.echo('ERROR %s' % exc) sys.exit(3) @click.group(cls=CatchAllExceptions) @click.version_option(version=__version__) def cli(): pass @cli.command() @click.option('--email', prompt='Email address') @click.option('--password', prompt=True, hide_input=True) def login(email, password): """Login to rorodata platform. """ try: auth.login(email, password) click.echo("Login successful.") except ConnectionError: click.echo('unable to connect to the server, try again later') except FireflyError as e: click.echo(e) raise @cli.command() def version(): """Prints the version of roro client.""" cli.main(args=['--version']) @cli.command() def whoami(): """prints the details of current user. """ user = auth.whoami() if user: click.echo(user['email']) else: click.echo("You are not logged in yet.") sys.exit(1) @cli.command(name="projects") def _projects(): """Lists all the projects. """ projects = Project.find_all() for p in projects: print(p.name) @cli.command() @click.argument('project') def create(project): """Creates a new Project. """ p = Project(project) p.create() print("Created project:", project) @cli.command() def deploy(): """Pushes the local changes to the cloud and restarts all the services. """ # TODO: validate credentials project = projects.current_project() response = project.deploy() click.echo(response) @cli.command() @click.argument('src', type=PathType()) @click.argument('dest', type=PathType()) def cp(src, dest): """Copy files to and from volumes to you local disk. Example: $ roro cp volume:/dataset.txt ./dataset.txt downloads the file dataset.txt from the server $ roro cp ./dataset.txt volume:/dataset.txt uploads dataset.txt to the server """ if src.is_volume() is dest.is_volume(): raise Exception('One of the arguments has to be a volume, other a local path') project = projects.current_project() project.copy(src, dest) @cli.command() @click.option('-a', '--all', default=False, is_flag=True) def ps(all): """Shows all the processes running in this project. """ project = projects.current_project() jobs = project.ps(all=all) rows = [] for job in jobs: start = h.parse_time(job['start_time']) end = h.parse_time(job['end_time']) total_time = (end - start) total_time = datetime.timedelta(total_time.days, total_time.seconds) command = " ".join(job["details"]["command"]) rows.append([job['jobid'], job['status'], h.datestr(start), str(total_time), job['instance_type'], h.truncate(command, 50)]) print(tabulate(rows, headers=['JOBID', 'STATUS', 'WHEN', 'TIME', 'INSTANCE TYPE', 'CMD'], disable_numparse=True)) @cli.command(name='ps:restart') @click.argument('name') def ps_restart(name): """Restarts the service specified by name. """ pass @cli.command(name="config") def _config(): """Lists all config vars of this project. """ project = projects.current_project() config = project.get_config() print("=== {} Config Vars".format(project.name)) for k, v in config.items(): print("{}: {}".format(k, v)) @cli.command(name='config:set') @click.argument('vars', nargs=-1) def env_set(vars): """Sets one or more the config vars. """ project = projects.current_project() d = {} for var in vars: if "=" in var: k, v = var.split("=", 1) d[k] = v else: d[var] = "" project.set_config(d) print("Updated config vars") @cli.command(name='config:unset') @click.argument('names', nargs=-1) def env_unset(names): """Unsets one or more config vars. """ project = projects.current_project() project.unset_config(names) print("Updated config vars") @cli.command(context_settings={"allow_interspersed_args": False}) @click.argument('command', nargs=-1) def run(command): """Runs the given script in foreground. """ project = projects.current_project() job = project.run(command) print("Started new job", job["jobid"]) @cli.command(name='run:notebook', context_settings={"allow_interspersed_args": False}) def run_notebook(): """Runs a notebook. """ project = projects.current_project() job = project.run_notebook() _logs(project, job["jobid"], follow=True, end_marker="-" * 40) @cli.command() @click.argument('jobid') def stop(jobid): """Stops a service with reference to jobid """ project = projects.current_project() project.stop(jobid) @cli.command() @click.argument('jobid') @click.option('-s', '--show-timestamp', default=False, is_flag=True) @click.option('-f', '--follow', default=False, is_flag=True) def logs(jobid, show_timestamp, follow): """Shows all the logs of the project. """ project = projects.current_project() _logs(project, jobid, follow, show_timestamp) def _logs(project, job_id, follow=False, show_timestamp=False, end_marker=None): """Shows the logs of job_id. """ def get_logs(job_id, follow=False): if follow: seen = 0 while True: logs = project.logs(job_id) for log in logs[seen:]: yield log seen = len(logs) job = project.ps(job_id) if job['status'] in ['success', 'cancelled', 'failed']: break time.sleep(0.5) else: logs = project.logs(job_id) for log in logs: yield log logs = get_logs(job_id, follow) if end_marker: logs = itertools.takewhile(lambda log: not log['message'].startswith(end_marker), logs) _display_logs(logs, show_timestamp=show_timestamp) def _display_logs(logs, show_timestamp=False): def parse_time(timestamp): t = datetime.datetime.fromtimestamp(timestamp//1000) return t.isoformat() if show_timestamp: log_pattern = "[{timestamp}] {message}" else: log_pattern = "{message}" for line in logs: line['timestamp'] = parse_time(line['timestamp']) click.echo(log_pattern.format(**line)) @cli.command() @click.argument('project') def project_logs(project): """Shows all the logs of the process with name the project. """ pass @cli.command() def volumes(): """Lists all the volumes. """ project = projects.current_project() volumes = project.list_volumes() if not volumes: click.echo('No volumes are attached to {}'.format(project.name)) for volume in project.list_volumes(): click.echo(volume) @cli.command(name='volumes:add') @click.argument('volume_name') def create_volume(volume_name): """Creates a new volume. """ project = projects.current_project() volume = project.add_volume(volume_name) click.echo('Volume {} added to the project {}'.format(volume, project.name)) @cli.command(name='volumes:remove') @click.argument('volume_name') def remove_volume(volume_name): """Removes a new volume. """ pass @cli.command(name='volumes:ls') @click.argument('path') def ls_volume(path): """Lists you files in a volume. Example: \b roro volume:ls lists all files in volume "volume_name" \b roro volume:ls lists all filies at directory "dir" in volume "volume" """ path = path+':' if ':' not in path else path path = Path(path) project = projects.current_project() stat = project.ls(path) rows = [[item['mode'], item['size'], item['name']] for item in stat] click.echo(tabulate(rows, tablefmt='plain')) @cli.command() def models(): project = projects.current_project() for repo in project.list_model_repositories(): print(repo.name) @cli.command(name="models:log") @click.argument('name', required=False) @click.option('-a', '--all', default=False, is_flag=True, help="Show all fields") def models_log(name=None, all=False): project = projects.current_project() images = project.get_model_activity(repo=name) for im in images: if all: print(im) else: print(im.get_summary()) @cli.command(name="models:show") @click.argument('modelref') def models_show(modelref): project = projects.current_project() model = modelref tag = None version = None if ":" in modelref: model, version_or_tag = modelref.split(":", 1) if version_or_tag.isnumeric(): version = int(version_or_tag) else: tag = version_or_tag repo = project.get_model_repository(model) image = repo and repo.get_model_image(version=version, tag=tag) if not image: click.echo("Invalid model reference {!r}".format(model)) sys.exit(1) print(image) def main_dev(): config.SERVER_URL = "http://api.local.rorodata.com:8080/" cli() PKx;K_lEroro/client.py"""The rorodata client """ import base64 import firefly from . import auth class Client(firefly.Client): def prepare_headers(self): login = auth.get_saved_login() if not login: return {} both = "{}:{}".format(login['email'], login['password']).encode('utf-8') basic_auth = base64.b64encode(both).decode("ascii") return { 'Authorization': 'Basic {}'.format(basic_auth) } PKx;Kpv**roro/config.py SERVER_URL = "https://api.rorodata.com/" PKx;Ko%roro/helpers.pyimport datetime import sys try: from urllib.parse import urlparse except ImportError: # python 2 from urlparse import urlparse PY2 = (sys.version_info.major == 2) def parse_time(timestr): if not timestr: return datetime.datetime.utcnow() try: return datetime.datetime.strptime(timestr, "%Y-%m-%d %H:%M:%S.%f") except ValueError: return datetime.datetime.strptime(timestr, "%Y-%m-%d %H:%M:%S") def datestr(then, now=None): """Converts time to a human readable string. Wrapper over web.datestr. >>> from datetime import datetime >>> datestr(datetime(2010, 1, 2), datetime(2010, 1, 1)) '1 day ago' """ s = _web_datestr(then, now) if 'milliseconds' in s or 'microseconds' in s: s = 'Just now' return s def _web_datestr(then, now=None): """ datestr utility from web.py (public domain). source: https://github.com/webpy/webpy Converts a (UTC) datetime object to a nice string representation. >>> from datetime import datetime, timedelta >>> d = datetime(1970, 5, 1) >>> datestr(d, now=d) '0 microseconds ago' >>> for t, v in iteritems({ ... timedelta(microseconds=1): '1 microsecond ago', ... timedelta(microseconds=2): '2 microseconds ago', ... -timedelta(microseconds=1): '1 microsecond from now', ... -timedelta(microseconds=2): '2 microseconds from now', ... timedelta(microseconds=2000): '2 milliseconds ago', ... timedelta(seconds=2): '2 seconds ago', ... timedelta(seconds=2*60): '2 minutes ago', ... timedelta(seconds=2*60*60): '2 hours ago', ... timedelta(days=2): '2 days ago', ... }): ... assert datestr(d, now=d+t) == v >>> datestr(datetime(1970, 1, 1), now=d) 'January 1' >>> datestr(datetime(1969, 1, 1), now=d) 'January 1, 1969' >>> datestr(datetime(1970, 6, 1), now=d) 'June 1, 1970' >>> datestr(None) '' """ def agohence(n, what, divisor=None): if divisor: n = n // divisor out = str(abs(n)) + ' ' + what # '2 day' if abs(n) != 1: out += 's' # '2 days' out += ' ' # '2 days ' if n < 0: out += 'from now' else: out += 'ago' return out # '2 days ago' oneday = 24 * 60 * 60 if not then: return "" if not now: now = datetime.datetime.utcnow() if type(now).__name__ == "DateTime": now = datetime.datetime.fromtimestamp(now) if type(then).__name__ == "DateTime": then = datetime.datetime.fromtimestamp(then) elif type(then).__name__ == "date": then = datetime.datetime(then.year, then.month, then.day) delta = now - then deltaseconds = int(delta.days * oneday + delta.seconds + delta.microseconds * 1e-06) deltadays = abs(deltaseconds) // oneday if deltaseconds < 0: deltadays *= -1 # fix for oddity of floor if deltadays: if abs(deltadays) < 4: return agohence(deltadays, 'day') # Trick to display 'June 3' instead of 'June 03' # Even though the %e format in strftime does that, it doesn't work on Windows. out = then.strftime('%B %d').replace(" 0", " ") if then.year != now.year or deltadays < 0: out += ', %s' % then.year return out if int(deltaseconds): if abs(deltaseconds) > (60 * 60): return agohence(deltaseconds, 'hour', 60 * 60) elif abs(deltaseconds) > 60: return agohence(deltaseconds, 'minute', 60) else: return agohence(deltaseconds, 'second') deltamicroseconds = delta.microseconds if delta.days: deltamicroseconds = int(delta.microseconds - 1e6) # datetime oddity if abs(deltamicroseconds) > 1000: return agohence(deltamicroseconds, 'millisecond', 1000) return agohence(deltamicroseconds, 'microsecond') def truncate(text, width): if len(text) > width: text = text[:width-3] + "..." return text def get_host_name(url): host = urlparse(url).netloc host_name = host.split(':')[0] return host_name PKx;KVudroro/models.pyfrom __future__ import print_function import io import joblib import re def get_model_repository(client, project, name): """Returns the ModelRepository with given name from the specified project. :param project: the name of the project :param name: name of the repository """ return ModelRepository(client, project, name) def list_model_repositories(client, project): return ModelRepository.find_all(client, project) class ModelRepository: def __init__(self, client, project, name): self.client = client self.project = project self.name = name def get_model_image(self, tag=None, version=None): metadata = self.client.get_model_version( project=self.project, name=self.name, tag=tag, version=version) return ModelImage(repo=self, metadata=metadata) def new_model_image(self, model, metadata={}): """Creates a new ModelImage. """ return ModelImage(repo=self, metadata=metadata, model=model) def get_activity(self): response = self.client.get_activity(project=self.project, name=self.name) return [ModelImage(repo=self, metadata=x) for x in response] @staticmethod def find_all(client, project): response = client.list_models(project=project) return [ModelRepository(client, project, name) for name in response] def __repr__(self): return "".format(self.project, self.name) class ModelImage: def __init__(self, repo, metadata, model=None, comment=None): self._repo = repo self._model = model self._metadata = metadata self.comment = comment or self.get("Comment") @property def id(self): return self._metadata.get("Model-Id") @property def version(self): return self._metadata["Model-Version"] @property def name(self): return self._metadata["Model-Name"] def __getitem__(self, name): return self._metadata[name] def __setitem__(self, name, value): self._metadata[name] = value def get_summary(self): keys = ["Model-ID", "Model-Name", "Model-Version", "Date"] f = io.StringIO() for k in keys: print("{}: {}".format(k, self.get(k, "")), file=f) print(file=f) comment = self._indent(self.comment) print(" {}".format(comment), file=f) return f.getvalue() def get_details(self): keys = ["Model-ID", "Model-Name", "Model-Version", "Date"] lower_keys = {k.lower(): i for i, k in enumerate(keys)} items = sorted(self._metadata.items(), key=lambda kv: (lower_keys.get(kv[0].lower(), 100), kv[0])) special_keys = ["comment", "tag"] items = [(k, v) for k, v in items if k.lower() not in special_keys] f = io.StringIO() for k, v in items: print("{}: {}".format(k, v), file=f) print(file=f) comment = self._indent(self.comment) print(" {}".format(comment), file=f) return f.getvalue() def _indent(self, text): text = text or "" return re.compile("^", re.M).sub(" ", text) def get(self, name, default=None): return self._metadata.get(name, default) def get_model(self): if self._model is None: self._model = self._load_model() return self._model def save(self, comment): """Saves a new version of the model image. """ if self.id is not None: raise Exception("ModelImage can't be modified once created.") if self._model is None: raise Exception("model object is not specified") f = io.BytesIO() joblib.dump(self._model, f) self['Content-Encoding'] = 'joblib' self._repo.client.save_model( project=self._repo.project, name=self._repo.name, model=f, comment=comment, **self._metadata) # TODO: Update the version and model-id def __repr__(self): return "".format(self._repo.project, self._repo.name, self.version) def __str__(self): return self.get_details() PKx;KjhOpff roro/path.pyimport shutil import pathlib from .helpers import PY2 if PY2: # FileNotFoundError is not defined for Python 2 FileNotFoundError = IOError try: FileNotFoundError except NameError: FileNotFoundError = IOError class Path: def __init__(self, path): self.volume = None if ':' in path: self.volume, self.path = path.split(':') else: self.path = path self._path = pathlib.Path(self.path) def is_volume(self): return self.volume is not None def open(self, *args, **kwargs): if self._path.is_dir(): raise Exception('Cannot copy, {} is a directory'.format(str(self._path))) return self._path.open(*args, **kwargs) def safe_write(self, fileobj, name): file_path = self._get_file_path(name) if file_path.is_dir(): raise Exception('Cannot copy, {} is a directory'.format(str(file_path))) p = file_path.with_suffix('.tmp') with p.open('wb') as f: shutil.copyfileobj(fileobj, f) p.rename(file_path) def _get_file_path(self, name): dest = self._path if dest.name != name: if not name: raise Exception('Name of the file is required when path is poiting to a dir') dest = dest/pathlib.Path(name) if not dest.parent.is_dir(): raise FileNotFoundError('No such file or directory: {}'.format(self.name+':'+str(dest))) return dest @property def size(self): return self._path.stat().st_size @property def name(self): return self._path.name PKy;KВUUUroro/projects.pyimport os import shutil import yaml from . import models, config from .client import Client from .helpers import PY2 from click import ClickException if PY2: from backports import tempfile else: import tempfile class Project: def __init__(self, name, runtime=None): self.name = name self.runtime = runtime self.client = Client(config.SERVER_URL) def create(self): return self.client.create(name=self.name) def run(self, command): job = self.client.run(project=self.name, command=command) return job def stop(self, jobid): self.client.stop(project=self.name, jobid=jobid) def run_notebook(self): job = self.client.run_notebook(project=self.name) return job def ps(self, jobid=None, all=False): return self.client.ps(project=self.name, jobid=jobid, all=all) def ls(self, path): return self.client.ls_volume( project=self.name, volume=path.volume, path=path.path ) def logs(self, jobid): return self.client.logs(project=self.name, jobid=jobid) #return self.client.logs(project=self.name) def deploy(self): print("Deploying project {}. This may take a few moments ...".format(self.name)) with tempfile.TemporaryDirectory() as tmpdir: archive = self.archive(tmpdir) size = os.path.getsize(archive) with open(archive, 'rb') as f: format = 'tar' response = self.client.deploy( project=self.name, archived_project=f, size=size, format=format ) return response def archive(self, rootdir, format='tar'): base_name = os.path.join(rootdir, "roro-project-" + self.name) return shutil.make_archive(base_name, format) def get_config(self): return self.client.get_config(project=self.name) def set_config(self, config_vars): return self.client.set_config(project=self.name, config_vars=config_vars) def unset_config(self, names): return self.client.unset_config(project=self.name, names=names) def list_volumes(self): volumes = self.client.volumes(project=self.name) return [volume['volume'] for volume in volumes] def add_volume(self, volume_name): volume = self.client.add_volume(project=self.name, name=volume_name) return volume['volume'] def get_model_repository(self, name): """Returns the ModelRepository from this project with given name. """ return models.get_model_repository(client=self.client, project=self.name, name=name) def list_model_repositories(self): """Returns a list of all the ModelRepository objects present in this project. """ return models.list_model_repositories(client=self.client, project=self.name) def get_model_activity(self, repo=None): response = self.client.get_activity(project=self.name, name=repo) return [models.ModelImage(repo=self, metadata=x) for x in response] def copy(self, src, dest): if src.is_volume(): self._get_file(src, dest) else: self._put_file(src, dest) def _get_file(self, src, dest): fileobj = self.client.get_file( project=self.name, volume=src.volume, path=src.path ) dest.safe_write(fileobj, src.name) def _put_file(self, src, dest): with src.open('rb') as fileobj: self.client.put_file( project=self.name, fileobj=fileobj, volume=dest.volume, path=dest.path, name=src.name, size=src.size ) @staticmethod def find_all(): client = Client(config.SERVER_URL) projects = client.projects() return [Project(p['name'], p.get('runtime')) for p in projects] def __repr__(self): return "".format(self.name) def current_project(): if os.path.exists("roro.yml"): d = yaml.safe_load(open("roro.yml")) return Project(d['project'], d.get('runtime')) else: raise ClickException("Unable to find roro.yml") get_current_project = current_project def list_projects(): return Project.find_all() PK!H4D%roro-0.1.5.dist-info/entry_points.txtN+I/N.,()*/MI-SU1s2r3b`Id PK!H١Wdroro-0.1.5.dist-info/WHEEL HM K-*ϳR03rOK-J,/RH,Q0343 /, (-JLR()*M ILR(4KM̫#DPK!HkqZ;=@roro-0.1.5.dist-info/METADATAuQN0+FSR *ZQmX 9$1lRIR JmY{up0kaG< T+mӓ@f護RhoXY ]|p/ᑌč*dwFaPFW. : ]dRlEݴ#Kr<%lY~V}T\_8Mтms_[| 9չl!^ۛgCS󴦫5NM\iMܧIHѐ/;2 ~} \h:zR*cU g /)+6Yl'Bu}PK!HHaZroro-0.1.5.dist-info/RECORDmrP} L(   ytLʮuE ț. )64؉=) _tLYpNqװ# }:G17w̌=l).WCfYG9%O$cuB@"{jnZx:sتZoΖa$A43d1bVI`oPR*}^aħ[(ÅvLک;WM8 %۰GszU,`Wr9 q ;(mLI4&WH˖7}(өATLLmںB9+%h fsHryl ؁Ob`F$S?[Ũf$Hek/ǃ # EifԊ)VdB3s),zֳa(A8 "λϛxyTͮ|i~g7=eki~%Vm8wV7hS1U~4mᾒ$j`("~Tu4uCUR[kREi 逸KPK;K^Z@roro/__init__.pyPK;K[  roro/auth.pyPKy;K)x'x'  roro/cli.pyPKx;K_lE3roro/client.pyPKx;Kpv**5roro/config.pyPKx;Ko%5roro/helpers.pyPKx;KVudFroro/models.pyPKx;KjhOpff Wroro/path.pyPKy;KВUUUV^roro/projects.pyPK!H4D%ororo-0.1.5.dist-info/entry_points.txtPK!H١WdPproro-0.1.5.dist-info/WHEELPK!HkqZ;=@proro-0.1.5.dist-info/METADATAPK!HHaZWrroro-0.1.5.dist-info/RECORDPK It