PKrLR=nyptune/__init__.py"""Nyptune hides a copy of your environment in your Jypyter notebooks so that other people can easily reproduce your work""" import time, os, platform __version__ = '0.0.1.'+str(int(time.time())) from .magic import * # from IPython import get_ipython # ipy = get_ipython() def load_ipython_extension(ipy): print("loading nyptune") ipy.register_magics(CheckpointMagics) pid = os.fork() if pid == 0: if '64' in platform.machine(): machine = 'amd64' else: machine = '386' name = platform.system().lower() ipfs = str(Path(os.path.realpath(__file__)).parent / 'go-ipfs' / 'ipfs') os.execl(ipfs, "ipfs", "daemon")PK/vL44nyptune/jupyter.pyfrom subprocess import * from pathlib import Path import os def presave(path, model, contents_manager): magix = model['content']['metadata''']['magix'] = {} conda = run(["conda", "list", "--explicit"], stdout = PIPE, encoding = 'utf-8', shell=False) magix['conda'] = conda.stdout.split('\n') pip = run(["pip", "list", "--format", "freeze"], stdout = PIPE, encoding = 'utf-8', shell = False) magix['pip'] = pip.stdout.split('\n') path = Path(path) cache = path.parent / '.cache' os.makedirs(cache, exist_ok = True) procs = [] magix['cache']={} ipfs = run(["ipfs", "add", "--nocopy", "-r", str(cache)], stdout = PIPE, encoding = 'utf-8', shell = False) for line in ipfs.stdout.strip().split('\n'): action, sig, name = line.split() magix['cache'][name] = sigPK{L%K""nyptune/magic.pyimport pprint from subprocess import * from pathlib import Path from abc import ABC, abstractmethod from collections import namedtuple import pickle, os import hashlib from IPython.core.magic import * from .util import * class CheckpointHandler(ABC): @abstractmethod def checkpoint(self, ipython, name, value): pass @abstractmethod def restore(self, ipython, name, metadata): pass @classmethod def path_of(cls, name): m = hashlib.sha256() m.update(name.encode("utf-8")) os.makedirs(".cache", exist_ok = True) return Path(".cache") / m.hexdigest() class PathHandler(CheckpointHandler): def checkpoint(self, ipython, name, value): print(value) if Path(str(value)).is_file(): link_f(Path(str(value)), self.path_of(name)) return str(value) else: return None def restore(self, ipython, name, metadata): if Path(metadata).is_file(): print("not overwriting existing file: "+metadata) else: os.link(self.path_of(name), metadata) class PickleHandler(CheckpointHandler): def checkpoint(self, ipython, name, value): with open(self.path_of(name), "wb") as file: pickle.dump(value, file) return True def restore(self, ipython, name, metadata): with open(self.path_of(name), "rb") as file: ipython.user_ns[name] = pickle.load(file) @magics_class class CheckpointMagics(Magics): def __init__(self, handlers = [PathHandler(), PickleHandler()], **kwargs): super(CheckpointMagics, self).__init__(**kwargs) self.enabled = False self.handlers = handlers def save(self, name): value = self.shell.user_ns[name] for handler in self.handlers: metadata = handler.checkpoint(self.shell, name, value) if metadata: all = (type(handler), metadata) with open(handler.path_of(name).with_suffix(".metadata"), 'wb') as file: pickle.dump(all, file) return @line_magic def uncache(self, line, cell = None): name = line.strip() unlink_f(handler.path_of(name)) unlink_f(handler.path_of(name).with_suffix(".metadata")) @line_cell_magic def recache(self, line, cell = None): self.cache(line, cell, overwrite = True) @line_cell_magic def cache(self, line, cell = None, overwrite = False): name = line.strip() if cell: if self.enabled: metadata_path = CheckpointHandler.path_of(name).with_suffix(".metadata") if metadata_path.is_file() and not overwrite: print("loading from cache") with open(metadata_path, 'rb') as file: metadata = pickle.load(file) cls = metadata[0] for handler in self.handlers: if type(handler) == cls: handler.restore(self.shell, name, metadata[1]) return else: self.shell.run_cell(cell) self.save(name) else: self.shell.run_cell(cell) else: self.save(name) @line_magic def caching(self, line): line = line.strip() if line == "on": self.enabled = True elif line == "off": self.enabled = False PKsL nyptune/script.pyfrom pathlib import Path from subprocess import * from tempfile import * import re, platform, os, json, sys, argparse, signal, urllib, tarfile, tempfile def main(): parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(help='a subcommand') init_parser = subparsers.add_parser('init', help='initialize ipfs and add the save hook to the jupyter config file') init_parser.set_defaults(func=init) launch_parser = subparsers.add_parser('launch', help='create an environment for and launch a jupyter notebook') launch_parser.set_defaults(func=launch) start_parser = subparsers.add_parser('start', help='starts the upload daemon') start_parser.set_defaults(func=start) stop_parser = subparsers.add_parser('stop', help='stops the upload daemon') stop_parser.set_defaults(func=stop) launch_parser.add_argument('notebook', metavar='NOTEBOOK', type=str, nargs=1, help='an integer for the accumulator') if len(sys.argv) == 1: parser.print_help() else: parsed = parser.parse_args() parsed.func(parsed) def start(parsed_args): pid = os.fork() if pid == 0: os.execl(ipfs(), "ipfs", "daemon") else: with open(Path(gettempdir()) / "nyptune.pid", "w") as file: file.write(str(pid)+"\n") def stop(parsed_args): pid_path = Path(gettempdir()) / "nyptune.pid" if pid_path.is_file(): with open(pid_path) as file: pid = file.read() os.kill(int(pid), signal.SIGTERM) pid_path.unlink() else: print("Nyptune daemon not running: no pid file found") def ipfs(): return str(Path(os.path.realpath(__file__)).parent / 'go-ipfs' / 'ipfs') def launch(parsed_args): notebook = parsed_args.notebook[0] cache = Path(notebook).parent / '.cache' with open(notebook) as file: model = json.load(file) conda = '\n'.join([line for line in model['metadata']['magix']['conda'] if line != '@EXPLICIT']) with NamedTemporaryFile() as conda_env_yaml: conda_env_yaml.write(conda.encode('utf-8')) conda_env_yaml.flush() print(conda_env_yaml.name) result = run(['conda', 'create', '-y', '--name', sys.argv[1], '--file', conda_env_yaml.name], encoding = 'utf-8', shell = False) if result.returncode != 0: result = run(['conda', 'env', 'update', '-y', '--name', sys.argv[1], '--file', conda_env_yaml.name], encoding = 'utf-8', shell = False) sig = model['metadata']['magix']['cache']['.cache'] run([ipfs(), 'get', sig, '-o', cache], shell = False) with NamedTemporaryFile() as requirements: pip = '\n'.join(model['metadata']['magix']['pip']) requirements.write(pip.encode('utf-8')) requirements.flush() with NamedTemporaryFile() as script: s = [ '#!/bin/bash', 'source activate '+sys.argv[1], 'pip install -y -r '+requirements.name, 'jupyter notebook' ] script.write('\n'.join(s).encode('utf-8')) script.flush() os.chmod(script.name, 0o755) print('running '+script.name) print('\n'.join(s)) os.execl(script.name, 'jupyter') # #!/bin/bash # cd `dirname $0` # # version=0.4.14 # # for platform in darwin linux # do # for arch in 386 amd64 # do # wget https://dist.ipfs.io/go-ipfs/v${version}/go-ipfs_v${version}_${platform}-${arch}.tar.gz # tar -zxvf go-ipfs_v${version}_${platform}-${arch}.tar.gz # mkdir -p nyptune/ipfs/${platform}-${arch} # cp go-ipfs/ipfs nyptune/ipfs/${platform}-${arch}/ # rm -rf go-ipfs # rm go-ipfs_v${version}_${platform}-${arch}.tar.gz # done # done # def init(parsed_args): config = Path.home() / '.jupyter' / 'jupyter_notebook_config.py' if not config.is_file(): print('generating an empty jupyter config file') run(['jupyter', 'notebook', '--generate-config'], encoding = 'utf-8', shell = False) with open(config, 'r') as file: contents = file.read() if 'nyptune' in contents: print('jupyter config file already mentions nyptune') else: with open(config, 'a') as file: print('appending nyptune pre-save-hook to jupyter config') file.write('\nfrom nyptune.jupyter import presave\nc.ContentsManager.pre_save_hook = presave\n') if '64' in platform.machine(): arch = 'amd64' else: arch = '386' plat = platform.system().lower() version='0.4.14' local = Path(tempfile.gettempdir()) / "go-ipfs.tar.gz" urllib.request.urlretrieve(f"https://dist.ipfs.io/go-ipfs/v{version}/go-ipfs_v{version}_{plat}-{arch}.tar.gz", local) with tarfile.open(local, 'r|gz') as tar: tar.extractall(Path(os.path.realpath(__file__)).parent) print('initializing ipfs') run([ipfs(), 'init'], encoding = 'utf-8', shell = False) run([ipfs(), 'config', '--json', 'Experimental.FilestoreEnabled', 'true'], encoding = 'utf-8', shell = False)PK}uL%`nyptune/util.pyimport pathlib, os from pathlib import Path def unlink_f(path): if Path(path).is_file(): os.unlink(path) def link_f(src, target): target = Path(target) if target.is_file(): os.unlink(target) os.link(src, target)PK!HT:'/3nyptune-0.0.1.1526419491.dist-info/entry_points.txtN+I/N.,()ʫ,()Kz <..PK!Hp!Qa(nyptune-0.0.1.1526419491.dist-info/WHEEL HM K-*ϳR03rOK-J,/RH,zd&Y)r$[)T&UD"PK!H2+nyptune-0.0.1.1526419491.dist-info/METADATA=N0 y @uH nzK&g[xz:}~{P,{ U:KIUۺ}X?nf۪Ƣ[`@ zs;/쵄"БyLĠy A D Rp8vT<tӜ|{v~ߚ'lO>n:p԰+=ޮ4M"v0>ܢPK!H/z)nyptune-0.0.1.1526419491.dist-info/RECORDI@}(E-QQ6"S"${e/Bif+/(}ˏxrI %tΜ}3N~A,QU$k_$ҥ"T َ4_M}LrSBǺC)~)ř̝:7Jsd2ZW EFk"Qv`3L6Iǐp(A~tޘwqv8Flio(?NU'o2qٔfh#nYFvlMcnQP<+MPKrLR=nyptune/__init__.pyPK/vL44nyptune/jupyter.pyPK{L%K""Qnyptune/magic.pyPKsL nyptune/script.pyPK}uL%`s*nyptune/util.pyPK!HT:'/3+nyptune-0.0.1.1526419491.dist-info/entry_points.txtPK!Hp!Qa( ,nyptune-0.0.1.1526419491.dist-info/WHEELPK!H2+,nyptune-0.0.1.1526419491.dist-info/METADATAPK!H/z)-nyptune-0.0.1.1526419491.dist-info/RECORDPK /