PK!8&&reprobench/__init__.pyname = "reprobench" VERSION = "0.1.0" PK!{f&&reprobench/console/main.py#!/usr/bin/env python import argparse import os import sys import click import strictyaml from loguru import logger from reprobench.core.bootstrap import cli as bootstrap_cli from reprobench.core.server import cli as server_cli from reprobench.core.schema import schema from reprobench.utils import import_class, read_config from reprobench.runners import cli as runner_cli @click.group() @click.version_option() @click.option("--verbose", "-v", "verbosity", count=True, default=0, help="Verbosity") def cli(verbosity): sys.path.append(os.getcwd()) logger.remove() if verbosity == 0: logger.add(sys.stderr, level="ERROR") elif verbosity == 1: logger.add(sys.stderr, level="WARNING") elif verbosity == 2: logger.add(sys.stderr, level="INFO") elif verbosity == 3: logger.add(sys.stderr, level="DEBUG") elif verbosity >= 4: logger.add(sys.stderr, level="TRACE") cli.add_command(bootstrap_cli) cli.add_command(server_cli) cli.add_command(runner_cli) if __name__ == "__main__": cli() PK!reprobench/core/base.pyimport zmq.green as zmq from reprobench.utils import recv_event class Runner: def __init__(self, config): pass def run(self): pass class Observer: SUBSCRIBED_EVENTS = [] @classmethod def observe(cls, context, backend_address, frontend): socket = context.socket(zmq.SUB) socket.connect(backend_address) for event in cls.SUBSCRIBED_EVENTS: socket.setsockopt(zmq.SUBSCRIBE, event) while True: event_type, payload, address = recv_event(socket) cls.handle_event(event_type, payload, frontend=frontend, address=address) @classmethod def handle_event(cls, event_type, payload, **kwargs): pass class Step: @classmethod def register(cls, config={}): pass @classmethod def execute(cls, context, config={}): pass class Tool: name = "Base Tool" REQUIRED_PATHS = [] @classmethod def setup(cls): pass @classmethod def version(cls): return "1.0.0" @classmethod def pre_run(cls, context): pass @classmethod def cmdline(cls, context): pass @classmethod def post_run(cls, context): pass @classmethod def teardown(cls): pass PK!s  reprobench/core/bootstrap.pyimport itertools from pathlib import Path import click from loguru import logger from tqdm import tqdm from reprobench.core.db import ( MODELS, Limit, Parameter, ParameterGroup, Run, Task, TaskGroup, Tool, ToolParameterGroup, db, ) from reprobench.task_sources.doi import DOISource from reprobench.task_sources.local import LocalSource from reprobench.task_sources.url import UrlSource from reprobench.utils import ( get_db_path, import_class, init_db, is_range_str, read_config, str_to_range, ) def _bootstrap_db(config): logger.info("Bootstrapping db...") db.connect() db.create_tables(MODELS) Limit.insert_many( [{"type": key, "value": value} for (key, value) in config["limits"].items()] ).execute() def _bootstrap_parameters(config): for (group, parameters) in config["parameters"].items(): ranged_enum_parameters = { key: value for key, value in parameters.items() if isinstance(parameters[key], list) } ranged_numbers_parameters = { key: str_to_range(value) for key, value in parameters.items() if isinstance(value, str) and is_range_str(value) } ranged_parameters = {**ranged_enum_parameters, **ranged_numbers_parameters} if len(ranged_parameters) == 0: parameter_group = ParameterGroup.create(name=group) for (key, value) in parameters.items(): Parameter.create(group=parameter_group, key=key, value=value) return constant_parameters = { key: value for key, value in parameters.items() if key not in ranged_parameters } tuples = [ [(key, value) for value in values] for key, values in ranged_parameters.items() ] for combination in itertools.product(*tuples): combination_str = ",".join(f"{key}={value}" for key, value in combination) parameter_group = ParameterGroup.create(name=f"{group}[{combination_str}]") parameters = {**dict(combination), **constant_parameters} for (key, value) in parameters.items(): Parameter.create(group=parameter_group, key=key, value=value) def _bootstrap_tools(config): logger.info("Bootstrapping and running setups on tools...") Tool.insert_many( [ { "name": name, "module": tool["module"], "version": import_class(tool["module"]).version(), } for (name, tool) in config["tools"].items() ] ).execute() for tool in config["tools"].values(): import_class(tool["module"]).setup() for prefix in tool["parameters"]: for parameter_group in ParameterGroup.select().where( ParameterGroup.name.startswith(prefix) ): if parameter_group.name != prefix and parameter_group.name[-1] != "]": continue ToolParameterGroup.create( tool=tool["module"], parameter_group=parameter_group ) def _bootstrap_tasks(config): logger.info("Bootstrapping tasks...") for (group, task) in config["tasks"].items(): task_group = TaskGroup.create(name=group) source = None if task["type"] == "local": source = LocalSource(**task) elif task["type"] == "url": source = UrlSource(**task) elif task["type"] == "doi": source = DOISource(**task) else: raise NotImplementedError( f"No implementation for task source {task['type']}" ) files = source.setup() for file in files: Task.create(group=task_group, path=str(file)) def _register_steps(config): logger.info("Registering steps...") for step in itertools.chain.from_iterable(config["steps"].values()): import_class(step["module"]).register(step.get("config", {})) def _bootstrap_runs(config, output_dir): tools_parameter_groups = ToolParameterGroup.select().iterator() tasks = Task.select().iterator() for (tool_parameter_group, task) in tqdm( itertools.product(tools_parameter_groups, tasks), desc="Bootstrapping runs" ): directory = ( Path(output_dir) / tool_parameter_group.tool_id / tool_parameter_group.parameter_group_id / task.group_id / Path(task.path).name ) directory.mkdir(parents=True, exist_ok=True) Run.create( tool=tool_parameter_group.tool_id, task=task, parameter_group=tool_parameter_group.parameter_group_id, directory=directory, status=Run.SUBMITTED, ) def bootstrap(config, output_dir): Path(output_dir).mkdir(parents=True, exist_ok=True) db_path = get_db_path(output_dir) init_db(db_path) _bootstrap_db(config) _bootstrap_parameters(config) _bootstrap_tools(config) _bootstrap_tasks(config) _register_steps(config) _bootstrap_runs(config, output_dir) @click.command(name="bootstrap") @click.option( "-o", "--output-dir", type=click.Path(file_okay=False, writable=True, resolve_path=True), default="./output", required=True, show_default=True, ) @click.argument("config", type=click.Path()) def cli(config, output_dir): config = read_config(config) bootstrap(config, output_dir) if __name__ == "__main__": cli() PK!screprobench/core/db.pyfrom pathlib import Path from datetime import datetime from peewee import Proxy, Model from playhouse.apsw_ext import ( DateTimeField, CharField, ForeignKeyField, IntegerField, BooleanField, CompositeKey, FloatField, ) db = Proxy() class BaseModel(Model): class Meta: database = db class Limit(BaseModel): type = CharField(max_length=32, primary_key=True) value = CharField() class TaskGroup(BaseModel): name = CharField(primary_key=True) class Task(BaseModel): group = ForeignKeyField(TaskGroup, backref="tasks") path = CharField(primary_key=True) class Tool(BaseModel): module = CharField(primary_key=True) name = CharField() version = CharField(null=True) class ParameterGroup(BaseModel): name = CharField(primary_key=True) class Parameter(BaseModel): group = ForeignKeyField(ParameterGroup, backref="parameters") key = CharField() value = CharField() class Meta: primary_key = CompositeKey("group", "key") class ToolParameterGroup(BaseModel): tool = ForeignKeyField(Tool) parameter_group = ForeignKeyField(ParameterGroup) class Meta: primary_key = CompositeKey("tool", "parameter_group") class Run(BaseModel): FAILED = -2 CANCELED = -1 PENDING = 0 SUBMITTED = 1 RUNNING = 2 DONE = 3 STATUS_CHOICES = ( (FAILED, "Failed"), (CANCELED, "Canceled"), (PENDING, "Pending"), (SUBMITTED, "Submitted"), (RUNNING, "Running"), (DONE, "Done"), ) created_at = DateTimeField(default=datetime.now) tool = ForeignKeyField(Tool, backref="runs") parameter_group = ForeignKeyField(ParameterGroup, backref="runs") task = ForeignKeyField(Task, backref="runs") status = IntegerField(choices=STATUS_CHOICES, default=PENDING) directory = CharField(null=True) class Meta: only_save_dirty = True MODELS = ( Limit, TaskGroup, Task, Tool, ParameterGroup, Parameter, Run, ToolParameterGroup, ) PK!?rrreprobench/core/events.pyRUN_REGISTER = b"run:register" RUN_START = b"run:start" RUN_FINISH = b"run:finish" RUN_COMPLETE = b"run:complete" PK!"]+;77reprobench/core/exceptions.pyclass ExecutableNotFoundError(RuntimeError): pass PK!odreprobench/core/observers.pyimport msgpack import zmq.green as zmq from loguru import logger from reprobench.core.db import Run from reprobench.core.base import Observer from reprobench.core.events import RUN_FINISH, RUN_REGISTER, RUN_START from reprobench.utils import decode_message, encode_message, recv_event class CoreObserver(Observer): SUBSCRIBED_EVENTS = [RUN_REGISTER, RUN_START, RUN_FINISH] @classmethod def handle_event(cls, event_type, payload, **kwargs): frontend = kwargs.pop("frontend") address = kwargs.pop("address") if event_type == RUN_REGISTER: run = Run.get(payload) run_dict = dict( id=run.id, task=run.task_id, tool=run.tool_id, directory=run.directory, parameters=list(run.parameter_group.parameters.dicts()), ) frontend.send_multipart([address, encode_message(run_dict)]) elif event_type == RUN_START: Run.update(status=Run.RUNNING).where(Run.id == payload).execute() elif event_type == RUN_FINISH: Run.update(status=Run.DONE).where(Run.id == payload).execute() PK!Nreprobench/core/schema.pyfrom strictyaml import ( Map, Regex, Seq, Str, Int, Optional, Seq, MapPattern, Enum, Bool, Any, ) limits_schema = Map( { "time": Int(), Optional("memory", default=8192): Int(), Optional("output"): Int(), Optional("cores"): Str(), } ) module_schema = Regex(r"\.?\w+(\.\w+)*") plugin_schema = Map( {"module": module_schema, Optional("config"): MapPattern(Str(), Any())} ) task_sources = Enum(["local", "url"]) schema = Map( { "title": Str(), Optional("description"): Str(), "limits": limits_schema, "steps": Map( {"run": Seq(plugin_schema), Optional("compile"): Seq(plugin_schema)} ), "observers": Seq(plugin_schema), "tasks": MapPattern(Str(), MapPattern(Str(), Any())), "tools": MapPattern( Str(), Map({"module": module_schema, "parameters": Seq(Str())}) ), "parameters": MapPattern(Str(), MapPattern(Str(), Any())), } ) PK!  reprobench/core/server.pyimport atexit import multiprocessing import click import gevent import strictyaml import zmq.green as zmq from loguru import logger from playhouse.apsw_ext import APSWDatabase from reprobench.core.db import Run, db from reprobench.core.events import RUN_COMPLETE from reprobench.core.observers import CoreObserver from reprobench.core.schema import schema from reprobench.utils import clean_up, get_db_path, import_class, read_config BACKEND_ADDRESS = "inproc://backend" class BenchmarkServer: def __init__(self, observers, db_path, **kwargs): super().__init__() db.initialize(APSWDatabase(db_path)) self.frontend_address = kwargs.pop("address", "tcp://*:31334") self.serve_forever = kwargs.pop("forever", False) self.observers = observers + [CoreObserver] self.jobs_waited = Run.select().where(Run.status < Run.DONE).count() def loop(self): while True: address, event_type, payload = self.frontend.recv_multipart() logger.debug((address, event_type, payload)) self.backend.send_multipart([event_type, payload, address]) if event_type == RUN_COMPLETE: self.jobs_waited -= 1 if not self.serve_forever and self.jobs_waited == 0: break def run(self): self.context = zmq.Context() self.frontend = self.context.socket(zmq.ROUTER) self.frontend.bind(self.frontend_address) self.backend = self.context.socket(zmq.PUB) self.backend.bind(BACKEND_ADDRESS) logger.info(f"Listening on {self.frontend_address}") observer_greenlets = [] for observer in self.observers: greenlet = gevent.spawn( observer.observe, self.context, backend_address=BACKEND_ADDRESS, frontend=self.frontend, ) observer_greenlets.append(greenlet) serverlet = gevent.spawn(self.loop) serverlet.join() gevent.killall(observer_greenlets) @click.command(name="server") @click.option("-c", "--config", required=True, type=click.Path()) @click.option("-f", "--forever", help="Serve forever", is_flag=True) @click.option( "-o", "--output-dir", type=click.Path(file_okay=False, writable=True, resolve_path=True), default="./output", required=True, show_default=True, ) @click.argument("server_address") def cli(config, output_dir, server_address): atexit.register(clean_up) config = read_config(config) database = get_db_path(output_dir) observers = [] for observer in config["observers"]: observers.append(import_class(observer["module"])) server = BenchmarkServer(observers, database, address=server_address) server.run() if __name__ == "__main__": cli() PK!-vI I reprobench/core/sysinfo.pyimport platform import psutil from cpuinfo import get_cpu_info from playhouse.apsw_ext import CharField, FloatField, ForeignKeyField, IntegerField from reprobench.core.base import Step, Observer from reprobench.core.db import BaseModel, Run, db from reprobench.utils import send_event, recv_event class Node(BaseModel): hostname = CharField(primary_key=True) platform = CharField(null=True) arch = CharField(null=True) python = CharField(null=True) cpu = CharField(null=True) cpu_count = IntegerField(null=True) cpu_min_freq = FloatField(null=True) cpu_max_freq = FloatField(null=True) mem_total = IntegerField(null=True) mem_available = IntegerField(null=True) swap_total = IntegerField(null=True) swap_available = IntegerField(null=True) class RunNode(BaseModel): run = ForeignKeyField(Run, backref="run_node", primary_key=True) node = ForeignKeyField(Node, backref="runs") MODELS = (Node, RunNode) STORE_SYSINFO = b"sysinfo:store" class SystemInfoObserver(Observer): SUBSCRIBED_EVENTS = [STORE_SYSINFO] @classmethod def handle_event(cls, event_type, payload, **kwargs): if event_type == STORE_SYSINFO: node = payload["node"] run = payload["run_id"] Node.insert(**node).on_conflict("ignore").execute() RunNode.insert(run=run, node=node["hostname"]).on_conflict( "replace" ).execute() class CollectSystemInfo(Step): @classmethod def register(cls, config={}): db.create_tables(MODELS) @classmethod def _get_system_info(cls): cpu_info = get_cpu_info() cpu_freq = psutil.cpu_freq() mem_info = psutil.virtual_memory() swap_info = psutil.swap_memory() info = {} info["platform"] = platform.platform(aliased=True) info["arch"] = cpu_info["arch"] info["python"] = cpu_info["python_version"] info["cpu"] = cpu_info["brand"] info["cpu_count"] = psutil.cpu_count() info["cpu_min_freq"] = cpu_freq.min info["cpu_max_freq"] = cpu_freq.max info["mem_total"] = mem_info.total info["mem_available"] = mem_info.available info["swap_total"] = swap_info.total info["swap_available"] = swap_info.free return info @classmethod def execute(cls, context, config={}): hostname = platform.node() info = cls._get_system_info() run_id = context["run"]["id"] payload = dict(run_id=run_id, node=dict(hostname=hostname, **info)) send_event(context["socket"], STORE_SYSINFO, payload) PK!s݊tt reprobench/executors/__init__.pyfrom .base import RunStatisticObserver # from .runsolver import RunsolverExecutor from .psmon import PsmonExecutor PK!=f))reprobench/executors/base.pyfrom loguru import logger from reprobench.core.base import Step, Observer from reprobench.executors.events import STORE_RUNSTATS from reprobench.utils import recv_event from .db import RunStatistic class RunStatisticObserver(Observer): SUBSCRIBED_EVENTS = [STORE_RUNSTATS] @classmethod def handle_event(cls, event_type, payload, **kwargs): if event_type == STORE_RUNSTATS: RunStatistic.create(**payload) class Executor(Step): @classmethod def register(cls, config={}): RunStatistic.create_table() PK!..reprobench/executors/db.pyfrom datetime import datetime from reprobench.core.db import BaseModel, Run from playhouse.apsw_ext import ( ForeignKeyField, FloatField, CharField, IntegerField, DateTimeField, ) class RunStatistic(BaseModel): TIMEOUT = "TLE" MEMOUT = "MEM" RUNTIME_ERR = "RTE" OUTPUT_LIMIT = "OLE" SUCCESS = "OK" VERDICT_CHOICES = ( (TIMEOUT, "Time Limit Exceeded"), (MEMOUT, "Memory Limit Exceeded"), (RUNTIME_ERR, "Runtime Error"), (OUTPUT_LIMIT, "Output Limit Exceeded"), (SUCCESS, "Run Successfully"), ) created_at = DateTimeField(default=datetime.now) run = ForeignKeyField(Run, backref="statistics", on_delete="cascade") cpu_time = FloatField(help_text="CPU Time (s)", null=True) wall_time = FloatField(help_text="Wall Clock Time (s)", null=True) max_memory = FloatField(help_text="Max Memory Usage (KiB)", null=True) return_code = IntegerField(help_text="Process Return Code", null=True) verdict = CharField(choices=VERDICT_CHOICES, max_length=3, null=True) PK!d,C,,reprobench/executors/events.pySTORE_RUNSTATS = b"executor:store_runstats" PK!~~reprobench/executors/psmon.pyimport subprocess from pathlib import Path from loguru import logger from psmon import ProcessMonitor from psmon.limiters import CpuTimeLimiter, MaxMemoryLimiter, WallTimeLimiter from reprobench.core.events import RUN_FINISH, RUN_START from reprobench.utils import send_event from .base import Executor from .db import RunStatistic from .events import STORE_RUNSTATS class PsmonExecutor(Executor): @classmethod def execute(cls, context, config={}): tool = context["tool"] limits = context["limits"] run_id = context["run"]["id"] tool.pre_run(context) cwd = context["run"]["directory"] out_file = (Path(cwd) / "run.out").open("wb") err_file = (Path(cwd) / "run.err").open("wb") cmd = tool.cmdline(context) logger.debug(f"Running {cwd}") logger.trace(cmd) monitor = ProcessMonitor( cmd, cwd=cwd, stdout=out_file, stderr=err_file, freq=15 ) monitor.subscribe("wall_time", WallTimeLimiter(limits["time"] + 15)) monitor.subscribe("cpu_time", CpuTimeLimiter(limits["time"])) monitor.subscribe("max_memory", MaxMemoryLimiter(limits["memory"])) send_event(context["socket"], RUN_START, run_id) stats = monitor.run() send_event(context["socket"], RUN_FINISH, run_id) logger.debug(f"Finished {cwd}") verdict = None if stats["error"] == TimeoutError: verdict = RunStatistic.TIMEOUT elif stats["error"] == MemoryError: verdict = RunStatistic.MEMOUT elif stats["error"] or stats["return_code"] != 0: verdict = RunStatistic.RUNTIME_ERR else: verdict = RunStatistic.SUCCESS del stats["error"] payload = dict(run_id=run_id, verdict=verdict, **stats) send_event(context["socket"], STORE_RUNSTATS, payload) tool.post_run(context) PK!)a a !reprobench/executors/runsolver.py""" TODO: Update to latest refactor """ import subprocess import functools import operator from pathlib import Path from reprobench.core.base import Step from reprobench.core.db import Run, RunStatistic from reprobench.utils import find_executable, silent_run class RunsolverExecutor(Step): def __init__(self): self.executable = find_executable("runsolver") def run(self, context): tool = context["tool"] limits = context["limits"] tool.pre_run(context) cwd = context["working_directory"] out_file = (Path(cwd) / "run.out").open("wb") err_file = (Path(cwd) / "run.err").open("wb") context["run"].status = Run.RUNNING context["run"].save() process = subprocess.run( [ self.executable, "-w", "run.watcher", "-v", "run.stat", "--cores", limits["cores"], "-C", str(limits["time"]), "--vsize-limit", str(limits["memory"]), # "-O", "0,{}".format(limits["output"]), "--", ] + tool.cmdline(context), cwd=cwd, stdout=out_file, stderr=err_file, ) context["run"].status = Run.DONE context["run"].verdict = Run.SUCCESS context["run"].save() tool.post_run(context) out_file.close() err_file.close() stat_file = Path(cwd) / "run.stat" stat_map = { "WCTIME": RunStatistic.WALL_TIME, "CPUTIME": RunStatistic.CPU_TIME, "MAXVM": RunStatistic.MEM_USAGE, } with stat_file.open() as f: for line in f: if line.startswith("#"): continue key, value = line.split("=") if key in stat_map: RunStatistic.create( run=context["run"], key=stat_map[key], value=value ) elif key == "TIMEOUT" and value == "true": context["run"].verdict = Run.TIMEOUT context["run"].save() elif key == "MEMOUT" and value == "true": context["run"].verdict = Run.MEMOUT context["run"].save() PK!{reprobench/runners/__init__.pyimport click from .local import LocalRunner from .local import cli as local_cli from .slurm import SlurmRunner from .slurm import cli as slurm_cli @click.group("run") def cli(): pass cli.add_command(local_cli) cli.add_command(slurm_cli) PK!AUm m $reprobench/runners/local/__init__.pyimport atexit import itertools import os import shutil import signal import time from datetime import datetime from multiprocessing import Process from multiprocessing.pool import Pool from pathlib import Path import click from loguru import logger from playhouse.apsw_ext import APSWDatabase from tqdm import tqdm from reprobench.core.base import Runner from reprobench.core.bootstrap import bootstrap from reprobench.core.db import Run, db from reprobench.core.server import BenchmarkServer from reprobench.task_sources.local import LocalSource from reprobench.utils import get_db_path, import_class, init_db, read_config from .worker import execute class LocalRunner(Runner): def __init__(self, config, **kwargs): self.config = config self.output_dir = kwargs.pop("output_dir", "./output") self.resume = kwargs.pop("resume", False) self.server_address = kwargs.pop("server", "tcp://127.0.0.1:31313") self.observers = [] self.queue = [] def exit(self): if len(self.queue) > 0 and hasattr(self, "pool"): self.pool.terminate() self.pool.join() if not self.resume and not self.setup_finished: shutil.rmtree(self.output_dir) def populate_unfinished_runs(self): query = Run.select(Run.id).where(Run.status < Run.DONE) self.queue = [(run.id, self.config, self.server_address) for run in query] def run(self): init_db(get_db_path(self.output_dir)) self.populate_unfinished_runs() db.close() if len(self.queue) == 0: logger.success("No tasks remaining to run") exit(0) logger.debug("Executing runs...") with Pool() as pool: it = pool.imap_unordered(execute, self.queue) progress_bar = tqdm(desc="Executing runs", total=len(self.queue)) for _ in it: progress_bar.update() progress_bar.close() logger.debug("Running teardown on all tools...") for tool in self.config["tools"].values(): import_class(tool["module"]).teardown() @click.command("local") @click.option( "-o", "--output-dir", type=click.Path(file_okay=False, writable=True, resolve_path=True), default="./output", show_default=True, ) @click.option("-r", "--resume", is_flag=True) @click.option("-s", "--server", default="tcp://127.0.0.1:31313") @click.argument("config", type=click.Path()) def cli(config, output_dir, **kwargs): config = read_config(config) runner = LocalRunner(config, output_dir=output_dir, **kwargs) runner.run() if __name__ == "__main__": cli() PK!VEK"reprobench/runners/local/worker.pyimport atexit import os from signal import signal import zmq from loguru import logger from playhouse.apsw_ext import APSWDatabase from reprobench.core.db import Run, db from reprobench.core.events import RUN_COMPLETE, RUN_REGISTER from reprobench.utils import clean_up, decode_message, import_class, send_event def execute(args): atexit.register(clean_up) run_id, config, server_address = args context = zmq.Context() socket = context.socket(zmq.DEALER) socket.connect(server_address) send_event(socket, RUN_REGISTER, run_id) run = decode_message(socket.recv()) tool = import_class(run["tool"]) context = config.copy() context["tool"] = tool context["run"] = run context["socket"] = socket logger.info(f"Processing task: {run['directory']}") for runstep in config["steps"]["run"]: logger.debug(f"Running step {runstep['module']}") step = import_class(runstep["module"]) step.execute(context, runstep.get("config", {})) send_event(socket, RUN_COMPLETE, run_id) PK!X$reprobench/runners/slurm/__init__.pyimport itertools import os import signal import subprocess import time from multiprocessing.pool import Pool from pathlib import Path from string import Template import click from loguru import logger from playhouse.apsw_ext import APSWDatabase from reprobench.core.base import Runner from reprobench.core.bootstrap import bootstrap from reprobench.core.db import Run, db from reprobench.utils import get_db_path, import_class, init_db, read_config from .utils import create_ranges DIR = os.path.dirname(os.path.realpath(__file__)) class SlurmRunner(Runner): def __init__(self, config_path, python_path, server_address, **kwargs): self.config = read_config(config_path) self.config_path = config_path self.python_path = python_path self.server_address = server_address self.output_dir = kwargs.pop("output_dir", "./output") self.resume = kwargs.pop("resume", False) self.templates = {} self.templates["server"] = kwargs.pop( "server_template_file", os.path.join(DIR, "./slurm.server.job.tpl") ) self.templates["run"] = kwargs.pop( "run_template_file", os.path.join(DIR, "./slurm.run.job.tpl") ) self.templates["compile"] = kwargs.pop( "compile_template_file", os.path.join(DIR, "./slurm.compile.job.tpl") ) self.queue = [] def populate_unfinished_runs(self): query = Run.select(Run.id).where(Run.status < Run.DONE) self.queue = [run.id for run in query] def generate_template(self, template_type): template_file = self.templates[template_type] with open(template_file) as tpl: template = Template(tpl.read()) job_str = template.safe_substitute( output_dir=self.output_dir, mem=int(1 + self.config["limits"]["memory"] / 1024 / 1024), # mb time=int(1 + (self.config["limits"]["time"] + 15) / 60), # minutes run_ids=create_ranges(self.queue), python_path=self.python_path, config_path=self.config_path, server_address=self.server_address, ) job_path = Path(self.output_dir) / f"slurm.{template_type}.job" with open(job_path, "w") as job: job.write(job_str) return str(job_path.resolve()) def run(self): init_db(get_db_path(self.output_dir)) self.populate_unfinished_runs() db.close() if len(self.queue) == 0: logger.success("No tasks remaining to run") exit(0) logger.debug("Generating templates...") templates = {t: self.generate_template(t) for t in ["run"]} logger.info("Submitting jobs to SLURM...") run_cmd = ["sbatch", "--parsable", templates["run"]] run_job = subprocess.check_output(run_cmd).decode().strip() logger.debug(f"Run job id: {run_job}") @click.command("slurm") @click.option( "-o", "--output-dir", type=click.Path(file_okay=False, writable=True, resolve_path=True), default="./output", required=True, show_default=True, ) @click.option("--run-template", type=click.Path(dir_okay=False, resolve_path=True)) @click.option("--compile-template", type=click.Path(dir_okay=False, resolve_path=True)) @click.option("-r", "--resume", is_flag=True) @click.option("-p", "--python-path", required=True, type=click.Path(resolve_path=True)) @click.option("-s", "--server", required=True) @click.argument("config", type=click.Path()) def cli(config, output_dir, python_path, server, **kwargs): runner = SlurmRunner( config_path=config, python_path=python_path, server_address=server, **kwargs ) runner.run() if __name__ == "__main__": cli() PK!Y96*reprobench/runners/slurm/slurm.run.job.tpl#!/bin/bash #SBATCH --export=all #SBATCH --array=$run_ids #SBATCH --mem=$mem #SBATCH --time=$time #SBATCH -o $output_dir/slurm-run_%a.out srun -- $python_path \ -m reprobench.runners.slurm.worker \ -c $config_path \ $server_address \ $SLURM_ARRAY_TASK_ID PK!&ı!reprobench/runners/slurm/utils.pyfrom itertools import tee, zip_longest # https://stackoverflow.com/a/3430312/9314778 def pairwise_longest(iterable): "variation of pairwise in http://docs.python.org/library/itertools.html#recipes" a, b = tee(iterable) next(b, None) return zip_longest(a, b) def takeuntil(predicate, iterable): """returns all elements before and including the one for which the predicate is true variation of http://docs.python.org/library/itertools.html#itertools.takewhile""" for x in iterable: yield x if predicate(x): break def get_range(it): "gets a range from a pairwise iterator" rng = list(takeuntil(lambda args: (args[1] is None) or (args[1] - args[0] > 1), it)) if rng: b, e = rng[0][0], rng[-1][0] return "%d-%d" % (b, e) if b != e else str(b) def create_ranges(zones): it = pairwise_longest(zones) return ",".join(iter(lambda: get_range(it), None)) PK!jϙ"reprobench/runners/slurm/worker.pyimport atexit import os import signal import time import click import zmq from loguru import logger from reprobench.core.schema import schema from reprobench.runners.local.worker import RUN_REGISTER from reprobench.utils import ( clean_up, decode_message, import_class, send_event, read_config, ) @click.command() @click.option("-c", "--config", required=True, type=click.Path()) @click.argument("server_address") @click.argument("run_id", type=int) def run(config, server_address, run_id): atexit.register(clean_up) config = read_config(config) context = zmq.Context() socket = context.socket(zmq.DEALER) socket.connect(server_address) send_event(socket, RUN_REGISTER, run_id) run = decode_message(socket.recv()) tool = import_class(run["tool"]) context = config.copy() context["socket"] = socket context["tool"] = tool context["run"] = run logger.info(f"Processing task: {run['directory']}") for runstep in config["steps"]["run"]: logger.debug(f"Running step {runstep['module']}") step = import_class(runstep["module"]) step.execute(context, runstep.get("config", {})) if __name__ == "__main__": run() PK!#reprobench/task_sources/__init__.pyPK!ϗ?reprobench/task_sources/base.pyclass BaseTaskSource(object): def __init__(self, path=None, **kwargs): self.path = path def setup(self): return [] PK!11'reprobench/task_sources/doi/__init__.pyfrom reprobench.task_sources.url import UrlSource from reprobench.task_sources.doi.zenodo import ZenodoHandler, ZenodoSandboxHandler class DOISource(UrlSource): handlers = [ZenodoHandler, ZenodoSandboxHandler] def __init__(self, doi, **kwargs): super().__init__(**kwargs) self.doi = doi for handler in self.handlers: if handler.is_compatible(self.doi): self.urls = handler.get_urls(self.doi) break else: raise NotImplementedError(f"No handler for doi: {doi}") PK!RO#reprobench/task_sources/doi/base.pyclass BaseDOIHandler(object): @classmethod def is_compatible(cls, doi): return False @classmethod def get_urls(cls, doi): return [] PK!Sʺ%reprobench/task_sources/doi/zenodo.pyimport requests from reprobench.task_sources.doi.base import BaseDOIHandler class ZenodoHandler(BaseDOIHandler): doi_prefix = "10.5281/zenodo." api_url = "https://zenodo.org/api" @classmethod def is_compatible(cls, doi): return doi.startswith(cls.doi_prefix) @classmethod def get_urls(cls, doi): record_id = doi[len(cls.doi_prefix) :] # remove doi_prefix url = "{}/records/{}".format(cls.api_url, record_id) record = requests.get(url).json() return [file["links"]["self"] for file in record["files"]] class ZenodoSandboxHandler(ZenodoHandler): doi_prefix = "10.5072/zenodo." api_url = "https://sandbox.zenodo.org/api" PK!a reprobench/task_sources/local.pyfrom pathspec import PathSpec from pathlib import Path from .base import BaseTaskSource class LocalSource(BaseTaskSource): def __init__(self, path=None, patterns="", **kwargs): super().__init__(path) self.patterns = patterns def setup(self): spec = PathSpec.from_lines("gitwildmatch", self.patterns.splitlines()) matches = spec.match_tree(self.path) return map(lambda match: Path(self.path) / match, matches) PK! DDreprobench/task_sources/url.pyfrom pathlib import Path from zipfile import ZipFile from loguru import logger from reprobench.utils import download_file from .local import LocalSource class UrlSource(LocalSource): def __init__( self, urls=[], path=None, patterns="", skip_existing=True, extract_archives=True, **kwargs, ): super().__init__(path, patterns=patterns) self.urls = urls self.extract_archives = extract_archives self.skip_existing = skip_existing def extract_zip(self, path): extract_path = Path(path) / ".." / path.stem if not extract_path.is_dir(): with ZipFile(path) as zip: zip.extractall(extract_path) def setup(self): root = Path(self.path) root.mkdir(parents=True, exist_ok=True) for url in self.urls: filename = url.split("/")[-1].split("?")[0] path = root / filename if not path.exists() or not self.skip_existing: logger.debug(f"Downloading {url} to {path}") download_file(url, path) else: logger.debug(f"Skipping already downloaded file {path}") if self.extract_archives and path.suffix == ".zip": self.extract_zip(path) return super().setup() PK!; reprobench/tools/executable.pyimport subprocess import tempfile import shutil from pathlib import Path from uuid import uuid4 from reprobench.core.base import Tool from reprobench.utils import find_executable, silent_run class ExecutableTool(Tool): name = "Basic Executable Tool" path = None PK!reprobench/tools/reprozip.pyimport subprocess import tempfile import shutil from pathlib import Path from uuid import uuid4 from reprobench.core.base import Tool from reprobench.utils import find_executable, silent_run class ReprozipTool(Tool): name = "Reprozip-based Tool" path = None runner = "directory" REQUIRED_PATHS = [ str((Path(find_executable("reprounzip")) / ".." / "..").resolve()), tempfile.gettempdir(), ] def __init__(self): self.reprounzip = find_executable("reprounzip") self.dir = f"{tempfile.gettempdir()}/reprounzip-{uuid4()}" self.base_command = [self.reprounzip, self.runner] def setup(self): silent_run(self.base_command + ["setup", self.path, self.dir]) def cmdline(self, context): return self.base_command + ["run", self.dir] def teardown(self): silent_run(self.base_command + ["destroy", self.dir]) PK!D reprobench/utils.pyimport importlib import logging import os import re import signal import subprocess import time from pathlib import Path from shutil import which import msgpack import requests import strictyaml from playhouse.apsw_ext import APSWDatabase from tqdm import tqdm from reprobench.core.db import db from reprobench.core.schema import schema from reprobench.core.exceptions import ExecutableNotFoundError log = logging.getLogger(__name__) def find_executable(executable): path = which(executable) if path is None: raise ExecutableNotFoundError return path def silent_run(command): log.debug(f"Running: {command}") return subprocess.run(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def import_class(path): module_path, tail = ".".join(path.split(".")[:-1]), path.split(".")[-1] module = importlib.import_module(module_path) return getattr(module, tail) def copyfileobj(fsrc, fdst, callback, length=16 * 1024): while True: buf = fsrc.read(length) if not buf: break fdst.write(buf) callback(len(buf)) def download_file(url, dest): r = requests.get(url, stream=True) with tqdm( total=int(r.headers["content-length"]), unit="B", unit_scale=True, unit_divisor=1024, ) as progress_bar: progress_bar.set_postfix(file=dest, refresh=False) with open(dest, "wb") as f: copyfileobj(r.raw, f, progress_bar.update) ranged_numbers_re = re.compile(r"(?P\d+)\.\.(?P\d+)(\.\.(?P\d+))?") def is_range_str(range_str): return ranged_numbers_re.match(range_str) def str_to_range(range_str): matches = ranged_numbers_re.match(range_str).groupdict() start = int(matches["start"]) end = int(matches["end"]) if matches["step"]: return range(start, end, int(matches["step"])) return range(start, end) def encode_message(obj): return msgpack.packb(obj, use_bin_type=True) def decode_message(msg): return msgpack.unpackb(msg, raw=False) def send_event(socket, event_type, payload): """ Used in the worker with a DEALER socket """ socket.send_multipart([event_type, encode_message(payload)]) def recv_event(socket): """ Used in the SUB handler """ event_type, payload, address = socket.recv_multipart() return event_type, decode_message(payload), address def clean_up(): signal.signal(signal.SIGTERM, signal.SIG_IGN) os.killpg(os.getpgid(0), signal.SIGTERM) time.sleep(1) os.killpg(os.getpgid(0), signal.SIGKILL) def get_db_path(output_dir): return str((Path(output_dir) / f"benchmark.db").resolve()) def init_db(db_path): database = APSWDatabase(db_path) db.initialize(database) def read_config(config_path): with open(config_path, "r") as f: config_text = f.read() config = strictyaml.load(config_text, schema=schema).data return config PK!HJ.:+reprobench-0.6.0.dist-info/entry_points.txtN+I/N.,()*J-(OJKΰE0r3s2PK!XB33"reprobench-0.6.0.dist-info/LICENSEMIT License Copyright (c) 2019 Rakha Kanz Kautsar Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PK!Hu)GTU reprobench-0.6.0.dist-info/WHEEL HM K-*ϳR03rOK-J,/R(O-)$qzd&Y)r$UV&UrPK!H%B#reprobench-0.6.0.dist-info/METADATA]O0+y{\9u4r%S'=4Rl@5hMȐ$:7ɫOK_x >[U 4e^b|c.y;t\BET>G,5O37Y 7`IQoр II3^NS|}{<}uE(f\2|I%7EW%c).T:ͫ1-xsg;/>ɕtp?ݿXXW}]ĿJ?^e3͕mwTcFx!Pg65C_/^HW4WjZqT.v'9#b3B!^] L9ȭ r jz˝L&G Fv\×% r-tݚC$7@)E#sD] y;ŹR %K7C zY=qt3E oƒ!dAwSܽ 8zfW8o<af׾h^'A~B0KEd41cn|XB{,6gkCDzkA"_08'RQ]TqV!;%ςj*G`Y\z/d<(c/]"5yL\p-dPs# 7!/q8uuG>pܲ>{F&ƘJ;epmտ `ob`5x%YҥkAsW5 VMx2NjF s"|Rsh3Y:@_Bǔ׷a.TP+㺏%F~s8m~Fpߒ CAeq^b&v.xO`kČ.\HPmC&.$Wyئ$(:rn0bA5IVxPJc?wJ-/}2}LQZl=$.L9] [. lԖ +p '74 q7\s cDpǔ}1vWu;4<MMc()6"A+Ay~G/?mK:ugPBf8U;HvTePr@_a52K;blek#myQa!t0{S˟PばC_}By(ā^y?F{ /SXNk2R Lt@ģy+rsv<oQXZ2j#[BC 4c:K"A i}FuUͧck.!9X y:3 ÆQU>4TQy!XnSsAp Nu#^L?[\ejwR0w8'Ve2OhQOD˅H4?'ŵ2}2DIyfT@kpwwC;~݊ܚFs1vݫzV9 |؞aF}VG^AH= 5DܘkCWSzS;6PS|0Ο2N-x!y_i/nAd-m>(bEi?[[N~22HUSң]$6quI4p.")j3wӽ.=-]nh#f`!G-E}]?юqwEhErTۏξYBQ deL|ي+bޔt2sbl;?:Ǩ: PK!8&&reprobench/__init__.pyPK!{f&&Zreprobench/console/main.pyPK!reprobench/core/base.pyPK!s   reprobench/core/bootstrap.pyPK!sc6 reprobench/core/db.pyPK!?rrx(reprobench/core/events.pyPK!"]+;77!)reprobench/core/exceptions.pyPK!od)reprobench/core/observers.pyPK!N].reprobench/core/schema.pyPK!  2reprobench/core/server.pyPK!-vI I =reprobench/core/sysinfo.pyPK!s݊tt eHreprobench/executors/__init__.pyPK!=f))Ireprobench/executors/base.pyPK!..zKreprobench/executors/db.pyPK!d,C,,Oreprobench/executors/events.pyPK!~~HPreprobench/executors/psmon.pyPK!)a a !Xreprobench/executors/runsolver.pyPK!{areprobench/runners/__init__.pyPK!AUm m $breprobench/runners/local/__init__.pyPK!VEK"mreprobench/runners/local/worker.pyPK!X$qreprobench/runners/slurm/__init__.pyPK!Y96*reprobench/runners/slurm/slurm.run.job.tplPK!&ı!Mreprobench/runners/slurm/utils.pyPK!jϙ"=reprobench/runners/slurm/worker.pyPK!#?reprobench/task_sources/__init__.pyPK!ϗ?reprobench/task_sources/base.pyPK!11'Ireprobench/task_sources/doi/__init__.pyPK!RO#reprobench/task_sources/doi/base.pyPK!Sʺ%reprobench/task_sources/doi/zenodo.pyPK!a reprobench/task_sources/local.pyPK! DDreprobench/task_sources/url.pyPK!; ,reprobench/tools/executable.pyPK!zreprobench/tools/reprozip.pyPK!D <reprobench/utils.pyPK!HJ.:+reprobench-0.6.0.dist-info/entry_points.txtPK!XB33"{reprobench-0.6.0.dist-info/LICENSEPK!Hu)GTU reprobench-0.6.0.dist-info/WHEELPK!H%B#reprobench-0.6.0.dist-info/METADATAPK!H${/ !reprobench-0.6.0.dist-info/RECORDPK'' 5