PK!8&&reprobench/__init__.pyname = "reprobench" VERSION = "0.1.0" PK!#6reprobench/console/main.py#!/usr/bin/env python import os import sys import click from loguru import logger from reprobench.core.bootstrap import cli as bootstrap_cli from reprobench.core.server import cli as server_cli from reprobench.core.worker import cli as worker_cli from reprobench.runners import cli as runner_cli from .status import benchmark_status @click.group() @click.version_option() @click.option("-q", "--quiet", is_flag=True) @click.option("--verbose", "-v", "verbosity", count=True, default=0, help="Verbosity") def cli(quiet, verbosity): sys.path.append(os.getcwd()) logger.remove() if not quiet: verbosity_levels = ["INFO", "DEBUG", "TRACE"] verbosity = min(verbosity, 2) logger.add(sys.stderr, level=verbosity_levels[verbosity]) cli.add_command(bootstrap_cli) cli.add_command(server_cli) cli.add_command(runner_cli) cli.add_command(worker_cli) cli.add_command(benchmark_status) if __name__ == "__main__": cli() PK!reprobench/console/status.pyimport time import click from tqdm import tqdm from reprobench.core.db import Run from reprobench.utils import init_db def get_total_count(): return Run.select().count() def get_done_count(): return Run.select().where(Run.status == Run.DONE).count() @click.command("status") @click.option("-d", "--database", default="./output/benchmark.db", show_default=True) @click.option("-n", "--interval", default=2, show_default=True, type=int) def benchmark_status(database, interval): init_db(database) total = get_total_count() progress = tqdm(total=total) last = get_done_count() progress.update(last) while last < total: time.sleep(interval) current = get_done_count() progress.update(current - last) last = current PK!A)reprobench/core/base.pyimport zmq.green as zmq from reprobench.utils import recv_event class Runner: def __init__(self, config): pass def run(self): pass class Observer: SUBSCRIBED_EVENTS = [] @classmethod def observe(cls, context, backend_address, reply): socket = context.socket(zmq.SUB) socket.connect(backend_address) for event in cls.SUBSCRIBED_EVENTS: socket.setsockopt(zmq.SUBSCRIBE, event) while True: event_type, payload, address = recv_event(socket) cls.handle_event(event_type, payload, reply=reply, address=address) @classmethod def handle_event(cls, event_type, payload, **kwargs): pass class Step: @classmethod def register(cls, config=None): pass @classmethod def execute(cls, context, config=None): pass class Tool: name = "Base Tool" def __init__(self, context): self.cwd = context["run"]["directory"] self.parameters = context["run"]["parameters"] self.task = context["run"]["task"] def run(self, executor): raise NotImplementedError def get_output(self): raise NotImplementedError def get_error(self): raise NotImplementedError @classmethod def setup(cls): pass @classmethod def version(cls): return "1.0.0" @classmethod def is_ready(cls): pass @classmethod def teardown(cls): pass PK!pc#reprobench/core/bootstrap.pyimport atexit import itertools import json import shutil from pathlib import Path import click from loguru import logger from tqdm import tqdm from reprobench.core.db import ( MODELS, Limit, Observer, Parameter, ParameterGroup, Run, Step, Task, TaskGroup, Tool, db, ) from reprobench.task_sources.doi import DOISource from reprobench.task_sources.local import LocalSource from reprobench.task_sources.url import UrlSource from reprobench.utils import ( get_db_path, import_class, init_db, is_range_str, read_config, str_to_range, ) def _bootstrap_db(config): logger.info("Bootstrapping db...") db.connect() db.create_tables(MODELS) Limit.insert_many( [{"key": key, "value": value} for (key, value) in config["limits"].items()] ).execute() Step.insert_many( [ { "category": key, "module": step["module"], "config": json.dumps(step.get("config", None)), } for key, steps in config["steps"].items() for step in steps ] ).execute() Observer.insert_many( [ { "module": observer["module"], "config": json.dumps(observer.get("config", None)), } for observer in config["observers"] ] ).execute() def _create_parameter_group(tool, group, parameters): ranged_enum_parameters = { key: value for key, value in parameters.items() if isinstance(parameters[key], list) } ranged_numbers_parameters = { key: str_to_range(value) for key, value in parameters.items() if isinstance(value, str) and is_range_str(value) } ranged_parameters = {**ranged_enum_parameters, **ranged_numbers_parameters} if len(ranged_parameters) == 0: parameter_group = ParameterGroup.create(name=group, tool=tool) for (key, value) in parameters.items(): Parameter.create(group=parameter_group, key=key, value=value) return constant_parameters = { key: value for key, value in parameters.items() if key not in ranged_parameters } tuples = [ [(key, value) for value in values] for key, values in ranged_parameters.items() ] for combination in itertools.product(*tuples): combination_str = ",".join(f"{key}={value}" for key, value in combination) parameter_group = ParameterGroup.create( name=f"{group}[{combination_str}]", tool=tool ) parameters = {**dict(combination), **constant_parameters} for (key, value) in parameters.items(): Parameter.create(group=parameter_group, key=key, value=value) def _bootstrap_tools(config): logger.info("Bootstrapping and running setups on tools...") for tool_name, tool in config["tools"].items(): tool_module = import_class(tool["module"]) if not tool_module.is_ready(): tool_module.setup() version = import_class(tool["module"]).version() Tool.create(name=tool_name, module=tool["module"], version=version) if "parameters" not in tool: _create_parameter_group(tool["module"], "default", {}) continue for group, parameters in tool["parameters"].items(): _create_parameter_group(tool["module"], group, parameters) def _bootstrap_tasks(config): logger.info("Bootstrapping tasks...") for (group, task) in config["tasks"].items(): task_group = TaskGroup.create(name=group) source = None # @TODO use chain of command if task["type"] == "local": source = LocalSource(**task) elif task["type"] == "url": source = UrlSource(**task) elif task["type"] == "doi": source = DOISource(**task) else: raise NotImplementedError( f"No implementation for task source {task['type']}" ) files = source.setup() for file in files: Task.create(group=task_group, path=str(file)) def _register_steps(config): logger.info("Registering steps...") for step in itertools.chain.from_iterable(config["steps"].values()): import_class(step["module"]).register(step.get("config", {})) def _bootstrap_runs(config, output_dir): parameter_groups = ParameterGroup.select().iterator() tasks = Task.select().iterator() for (parameter_group, task) in tqdm( itertools.product(parameter_groups, tasks), desc="Bootstrapping runs" ): directory = ( Path(output_dir) / parameter_group.tool_id / parameter_group.name / task.group_id / Path(task.path).name ) directory.mkdir(parents=True, exist_ok=True) Run.create( tool=parameter_group.tool_id, task=task, parameter_group=parameter_group, directory=directory, status=Run.PENDING, ) def bootstrap(config, output_dir): Path(output_dir).mkdir(parents=True, exist_ok=True) atexit.register(shutil.rmtree, output_dir) db_path = get_db_path(output_dir) init_db(db_path) _bootstrap_db(config) _bootstrap_tools(config) _bootstrap_tasks(config) _register_steps(config) _bootstrap_runs(config, output_dir) atexit.unregister(shutil.rmtree) @click.command(name="bootstrap") @click.option( "-o", "--output-dir", type=click.Path(file_okay=False, writable=True, resolve_path=True), default="./output", required=True, show_default=True, ) @click.argument("config", type=click.Path()) def cli(config, output_dir): config = read_config(config) bootstrap(config, output_dir) if __name__ == "__main__": cli() PK!'ʵ reprobench/core/db.pyfrom datetime import datetime from playhouse.apsw_ext import ( Model, Proxy, BlobField, CharField, CompositeKey, DateTimeField, FloatField, ForeignKeyField, IntegerField, AutoField, ) db = Proxy() class BaseModel(Model): class Meta: database = db class Limit(BaseModel): key = CharField(max_length=32, primary_key=True) value = CharField() class TaskGroup(BaseModel): name = CharField(primary_key=True) class Task(BaseModel): group = ForeignKeyField(TaskGroup, backref="tasks") path = CharField(primary_key=True) class Tool(BaseModel): module = CharField(primary_key=True) name = CharField() version = CharField(null=True) class ParameterGroup(BaseModel): name = CharField() tool = ForeignKeyField(Tool, backref="parameter_groups") class Meta: indexes = ((("name", "tool"), True),) class Parameter(BaseModel): group = ForeignKeyField(ParameterGroup, backref="parameters") key = CharField() value = CharField() class Meta: primary_key = CompositeKey("group", "key") class BasePlugin(BaseModel): module = CharField(index=True) config = BlobField() class Step(BasePlugin): RUN = "run" AGGREGATE = "aggregate" CATEGORY_CHOICES = ((RUN, "Single run step"), (AGGREGATE, "Aggregation step")) category = CharField(choices=CATEGORY_CHOICES, index=True) class Observer(BasePlugin): pass class Run(BaseModel): FAILED = -2 CANCELED = -1 PENDING = 0 SUBMITTED = 1 RUNNING = 2 DONE = 3 STATUS_CHOICES = ( (FAILED, "Failed"), (CANCELED, "Canceled"), (PENDING, "Pending"), (SUBMITTED, "Submitted"), (RUNNING, "Running"), (DONE, "Done"), ) created_at = DateTimeField(default=datetime.now) tool = ForeignKeyField(Tool, backref="runs") parameter_group = ForeignKeyField(ParameterGroup, backref="runs") task = ForeignKeyField(Task, backref="runs") status = IntegerField(choices=STATUS_CHOICES, default=PENDING) directory = CharField(null=True) current_step = ForeignKeyField(Step, null=True) class Meta: only_save_dirty = True MODELS = (Limit, TaskGroup, Task, Tool, ParameterGroup, Parameter, Run, Step, Observer) PK!reprobench/core/events.pySERVER_PING = b"server:ping" WORKER_JOIN = b"worker:join" WORKER_REQUEST = b"worker:request" WORKER_DONE = b"worker:done" WORKER_LEAVE = b"worker:leave" RUN_START = b"run:start" RUN_STEP = b"run:step" RUN_INTERRUPT = b"run:interrupt" RUN_FINISH = b"run:finish" PK!"]+;77reprobench/core/exceptions.pyclass ExecutableNotFoundError(RuntimeError): pass PK!reprobench/core/observers.pyfrom reprobench.core.db import Run, Step, Limit from reprobench.core.base import Observer from reprobench.core.events import ( RUN_FINISH, RUN_START, RUN_STEP, RUN_INTERRUPT, WORKER_REQUEST, ) from reprobench.utils import encode_message class CoreObserver(Observer): SUBSCRIBED_EVENTS = [WORKER_REQUEST, RUN_START, RUN_STEP, RUN_FINISH] @classmethod def get_next_pending_run(cls): run = Run.get_or_none(Run.status == Run.PENDING) if run is None: return None run.status = Run.SUBMITTED run.save() runsteps = Step.select().where(Step.category == Step.RUN) limits = {l.key: l.value for l in Limit.select()} parameters = {p.key: p.value for p in run.parameter_group.parameters} run_dict = dict( id=run.id, task=run.task_id, tool=run.tool_id, directory=run.directory, parameters=parameters, steps=list(runsteps.dicts()), limits=limits, ) return run_dict @classmethod def handle_event(cls, event_type, payload, **kwargs): reply = kwargs.pop("reply") address = kwargs.pop("address") if event_type == WORKER_REQUEST: run = cls.get_next_pending_run() if run is not None: reply.send_multipart([address, encode_message(run)]) elif event_type == RUN_INTERRUPT: Run.update(status=Run.PENDING).where(Run.id == payload).execute() elif event_type == RUN_START: Run.update(status=Run.RUNNING).where(Run.id == payload).execute() elif event_type == RUN_STEP: step = Step.get(module=payload["step"]) Run.update(current_step=step).where(Run.id == payload["run_id"]).execute() elif event_type == RUN_FINISH: Run.update(status=Run.DONE).where(Run.id == payload).execute() PK!NZ reprobench/core/schema.pyfrom strictyaml import Any, Enum, Int, Map, MapPattern, Optional, Regex, Seq, Str limits_schema = Map( { "time": Int(), Optional("memory", default=8192): Int(), Optional("output"): Int(), Optional("cores"): Int(), } ) module_schema = Regex(r"\.?\w+(\.\w+)*") plugin_schema = Map( {"module": module_schema, Optional("config"): MapPattern(Str(), Any())} ) task_sources = Enum(["local", "url"]) schema = Map( { "title": Str(), Optional("description"): Str(), "limits": limits_schema, "steps": Map( {"run": Seq(plugin_schema), Optional("compile"): Seq(plugin_schema)} ), "observers": Seq(plugin_schema), "tasks": MapPattern(Str(), MapPattern(Str(), Any())), "tools": MapPattern( Str(), Map( { "module": module_schema, Optional("parameters"): MapPattern(Str(), MapPattern(Str(), Any())), } ), ), } ) PK!ch h reprobench/core/server.pyfrom pathlib import Path import click import gevent import zmq.green as zmq from loguru import logger from playhouse.apsw_ext import APSWDatabase from reprobench.core.db import Observer, Run, db from reprobench.core.events import WORKER_JOIN, WORKER_LEAVE, WORKER_DONE, SERVER_PING from reprobench.core.observers import CoreObserver from reprobench.utils import import_class class BenchmarkServer: BACKEND_ADDRESS = "inproc://backend" def __init__(self, db_path, frontend_address, **kwargs): super().__init__() db.initialize(APSWDatabase(db_path)) self.frontend_address = frontend_address self.observers = [CoreObserver] self.observers += [ import_class(o.module) for o in Observer.select(Observer.module) ] self.serve_forever = kwargs.pop("forever", False) self.jobs_waited = 0 self.worker_count = 0 def loop(self): while True: if ( not self.serve_forever and self.jobs_waited == 0 and self.worker_count == 0 ): break address, event_type, payload = self.frontend.recv_multipart() logger.trace((address, event_type, payload)) self.backend.send_multipart([event_type, payload, address]) if event_type == SERVER_PING: self.frontend.send_multipart([address, b"pong"]) elif event_type == WORKER_JOIN: self.worker_count += 1 elif event_type == WORKER_LEAVE: self.worker_count -= 1 elif event_type == WORKER_DONE: self.jobs_waited -= 1 def run(self): self.context = zmq.Context() self.frontend = self.context.socket(zmq.ROUTER) self.frontend.bind(self.frontend_address) self.backend = self.context.socket(zmq.PUB) self.backend.bind(self.BACKEND_ADDRESS) Run.update(status=Run.PENDING).where(Run.status < Run.DONE).execute() self.jobs_waited = Run.select().where(Run.status == Run.PENDING).count() logger.info(f"Listening on {self.frontend_address}") observer_greenlets = [] for observer in self.observers: greenlet = gevent.spawn( observer.observe, self.context, backend_address=self.BACKEND_ADDRESS, reply=self.frontend, ) observer_greenlets.append(greenlet) serverlet = gevent.spawn(self.loop) serverlet.join() gevent.killall(observer_greenlets) @click.command(name="server") @click.option("-f", "--forever", help="Serve forever", is_flag=True) @click.option("-d", "--database", default="./output/benchmark.db", show_default=True) @click.option("-h", "--host", default="0.0.0.0", show_default=True) @click.option("-p", "--port", default=31313, show_default=True) def cli(database, host, port, **kwargs): db_path = str(Path(database).resolve()) frontend_address = f"tcp://{host}:{port}" server = BenchmarkServer(db_path, frontend_address, **kwargs) server.run() if __name__ == "__main__": cli() PK!E3A A reprobench/core/sysinfo.pyimport platform import psutil from cpuinfo import get_cpu_info from playhouse.apsw_ext import CharField, FloatField, ForeignKeyField, IntegerField from reprobench.core.base import Step, Observer from reprobench.core.db import BaseModel, Run, db from reprobench.utils import send_event class Node(BaseModel): hostname = CharField(primary_key=True) platform = CharField(null=True) arch = CharField(null=True) python = CharField(null=True) cpu = CharField(null=True) cpu_count = IntegerField(null=True) cpu_min_freq = FloatField(null=True) cpu_max_freq = FloatField(null=True) mem_total = IntegerField(null=True) mem_available = IntegerField(null=True) swap_total = IntegerField(null=True) swap_available = IntegerField(null=True) class RunNode(BaseModel): run = ForeignKeyField(Run, backref="run_node", primary_key=True) node = ForeignKeyField(Node, backref="runs") MODELS = (Node, RunNode) STORE_SYSINFO = b"sysinfo:store" class SystemInfoObserver(Observer): SUBSCRIBED_EVENTS = [STORE_SYSINFO] @classmethod def handle_event(cls, event_type, payload, **kwargs): if event_type == STORE_SYSINFO: node = payload["node"] run = payload["run_id"] Node.insert(**node).on_conflict("ignore").execute() RunNode.insert(run=run, node=node["hostname"]).on_conflict( "replace" ).execute() class CollectSystemInfo(Step): @classmethod def register(cls, config=None): db.create_tables(MODELS) @classmethod def _get_system_info(cls): cpu_info = get_cpu_info() cpu_freq = psutil.cpu_freq() mem_info = psutil.virtual_memory() swap_info = psutil.swap_memory() info = {} info["platform"] = platform.platform(aliased=True) info["arch"] = cpu_info["arch"] info["python"] = cpu_info["python_version"] info["cpu"] = cpu_info["brand"] info["cpu_count"] = psutil.cpu_count() info["cpu_min_freq"] = cpu_freq.min info["cpu_max_freq"] = cpu_freq.max info["mem_total"] = mem_info.total info["mem_available"] = mem_info.available info["swap_total"] = swap_info.total info["swap_available"] = swap_info.free return info @classmethod def execute(cls, context, config=None): hostname = platform.node() info = cls._get_system_info() run_id = context["run"]["id"] payload = dict(run_id=run_id, node=dict(hostname=hostname, **info)) send_event(context["socket"], STORE_SYSINFO, payload) PK!خ@ @ reprobench/core/worker.pyimport atexit import json import click import zmq from loguru import logger from reprobench.core.events import ( WORKER_JOIN, WORKER_REQUEST, WORKER_DONE, WORKER_LEAVE, RUN_STEP, RUN_INTERRUPT, ) from reprobench.utils import decode_message, import_class, send_event REQUEST_TIMEOUT = 15000 class BenchmarkWorker: """ Request for a work from server, if there's no more work, terminate. else, do the work and request for more. """ def __init__(self, server_address): self.server_address = server_address def killed(self, run_id): send_event(self.socket, RUN_INTERRUPT, run_id) send_event(self.socket, WORKER_LEAVE) def loop(self): while True: send_event(self.socket, WORKER_REQUEST) reply_count = self.socket.poll(timeout=REQUEST_TIMEOUT) if reply_count == 0: # looks like the server is dead logger.warning("Exiting because there's no reply from server.") break run = decode_message(self.socket.recv()) if run is None: # there's no more work to do logger.success("Exiting because there's no more work to do.") break atexit.register(self.killed, run["id"]) tool = import_class(run["tool"]) context = {} context["socket"] = self.socket context["tool"] = tool context["run"] = run logger.info(f"Processing task: {run['directory']}") for runstep in run["steps"]: payload = {"run_id": run["id"], "step": runstep["module"]} send_event(self.socket, RUN_STEP, payload) logger.debug(f"Running step {runstep['module']}") step = import_class(runstep["module"]) config = json.loads(runstep["config"]) step.execute(context, config) send_event(self.socket, WORKER_DONE, run["id"]) atexit.unregister(self.killed) def run(self): context = zmq.Context() self.socket = context.socket(zmq.DEALER) self.socket.connect(self.server_address) send_event(self.socket, WORKER_JOIN) self.loop() send_event(self.socket, WORKER_LEAVE) @click.command("worker") @click.option("-h", "--host", default="0.0.0.0", show_default=True) @click.option("-p", "--port", default=31313, show_default=True) def cli(host, port): worker = BenchmarkWorker(f"tcp://{host}:{port}") worker.run() if __name__ == "__main__": cli() PK!s݊tt reprobench/executors/__init__.pyfrom .base import RunStatisticObserver # from .runsolver import RunsolverExecutor from .psmon import PsmonExecutor PK!4UUreprobench/executors/base.pyfrom reprobench.core.base import Step, Observer from reprobench.executors.events import STORE_RUNSTATS from .db import RunStatistic class RunStatisticObserver(Observer): SUBSCRIBED_EVENTS = [STORE_RUNSTATS] @classmethod def handle_event(cls, event_type, payload, **kwargs): if event_type == STORE_RUNSTATS: RunStatistic.create(**payload) class Executor(Step): def run( self, cmdline, out_path=None, err_path=None, input_str=None, directory=None, **kwargs ): raise NotImplementedError @classmethod def register(cls, config=None): RunStatistic.create_table() @classmethod def execute(cls, context, config=None): tool = context["tool"] executor = cls(context, config) tool(context).run(executor) PK!%o-NNreprobench/executors/db.pyfrom datetime import datetime from reprobench.core.db import BaseModel, Run from playhouse.apsw_ext import ( ForeignKeyField, FloatField, CharField, IntegerField, DateTimeField, ) class RunStatistic(BaseModel): TIMEOUT = "TLE" MEMOUT = "MEM" RUNTIME_ERR = "RTE" OUTPUT_LIMIT = "OLE" SUCCESS = "OK" VERDICT_CHOICES = ( (TIMEOUT, "Time Limit Exceeded"), (MEMOUT, "Memory Limit Exceeded"), (RUNTIME_ERR, "Runtime Error"), (OUTPUT_LIMIT, "Output Limit Exceeded"), (SUCCESS, "Run Successfully"), ) created_at = DateTimeField(default=datetime.now) run = ForeignKeyField( Run, backref="statistics", on_delete="cascade", primary_key=True ) cpu_time = FloatField(help_text="CPU Time (s)", null=True) wall_time = FloatField(help_text="Wall Clock Time (s)", null=True) max_memory = FloatField(help_text="Max Memory Usage (KiB)", null=True) return_code = IntegerField(help_text="Process Return Code", null=True) verdict = CharField(choices=VERDICT_CHOICES, max_length=3, null=True) PK!d,C,,reprobench/executors/events.pySTORE_RUNSTATS = b"executor:store_runstats" PK!^B B reprobench/executors/psmon.pyfrom loguru import logger from psmon import ProcessMonitor from psmon.limiters import CpuTimeLimiter, MaxMemoryLimiter, WallTimeLimiter from reprobench.core.events import RUN_FINISH, RUN_START from reprobench.utils import send_event from .base import Executor from .db import RunStatistic from .events import STORE_RUNSTATS class PsmonExecutor(Executor): def __init__(self, context, config): self.socket = context["socket"] self.run_id = context["run"]["id"] if config is not None: wall_grace = config.get("wall_grace") else: wall_grace = 15 limits = context["run"]["limits"] time_limit = float(limits["time"]) MB = 1024 * 1024 self.wall_limit = time_limit + wall_grace self.cpu_limit = time_limit self.mem_limit = float(limits["memory"]) * MB def compile_stats(self, stats): verdict = None if stats["error"] == TimeoutError: verdict = RunStatistic.TIMEOUT elif stats["error"] == MemoryError: verdict = RunStatistic.MEMOUT elif stats["error"]: verdict = RunStatistic.RUNTIME_ERR else: verdict = RunStatistic.SUCCESS del stats["error"] return dict(run_id=self.run_id, verdict=verdict, **stats) def run( self, cmdline, out_path=None, err_path=None, input_str=None, directory=None, **kwargs, ): out_file = open(out_path, "wb") err_file = open(err_path, "wb") monitor = ProcessMonitor( cmdline, cwd=directory, stdout=out_file, stderr=err_file, input=input_str, freq=15, ) monitor.subscribe("wall_time", WallTimeLimiter(self.wall_limit)) monitor.subscribe("cpu_time", CpuTimeLimiter(self.cpu_limit)) monitor.subscribe("max_memory", MaxMemoryLimiter(self.mem_limit)) logger.debug(f"Running {directory}") send_event(self.socket, RUN_START, self.run_id) stats = monitor.run() send_event(self.socket, RUN_FINISH, self.run_id) logger.debug(f"Finished {directory}") out_file.close() err_file.close() payload = self.compile_stats(stats) send_event(self.socket, STORE_RUNSTATS, payload) PK!{reprobench/runners/__init__.pyimport click from .local import LocalRunner from .local import cli as local_cli from .slurm import SlurmRunner from .slurm import cli as slurm_cli @click.group("run") def cli(): pass cli.add_command(local_cli) cli.add_command(slurm_cli) PK!Վ{{reprobench/runners/base.pyimport time from pathlib import Path import zmq from loguru import logger from reprobench.core.base import Runner from reprobench.core.bootstrap import bootstrap from reprobench.core.events import SERVER_PING from reprobench.utils import get_db_path, send_event class BaseRunner(Runner): def __init__(self, config, **kwargs): self.config = config self.output_dir = kwargs.pop("output_dir") self.resume = kwargs.pop("resume", False) self.num_workers = kwargs.pop("num_workers", None) self.db_path = get_db_path(self.output_dir) self.server_address = None def prepare(self): pass def spawn_server(self): raise NotImplementedError def spawn_workers(self): raise NotImplementedError def server_ping(self): context = zmq.Context() socket = context.socket(zmq.DEALER) socket.connect(self.server_address) # should be blocking if not ready: http://api.zeromq.org/2-1:zmq-socket send_event(socket, SERVER_PING) socket.recv() def wait(self): pass def run(self): db_exist = Path(self.db_path).exists() if not db_exist: bootstrap(self.config, self.output_dir) if db_exist and not self.resume: logger.warning( f"Previous run exists in {self.output_dir}. Please use --resume, or specify a different output directory" ) exit(1) self.prepare() self.spawn_server() logger.info("Making sure the server has started...") self.server_ping() self.spawn_workers() self.wait() PK!E$reprobench/runners/local/__init__.pyimport atexit import time from multiprocessing import Process, cpu_count from pathlib import Path import click from loguru import logger from reprobench.core.server import BenchmarkServer from reprobench.core.worker import BenchmarkWorker from reprobench.runners.base import BaseRunner from reprobench.utils import read_config class LocalRunner(BaseRunner): def __init__(self, config, **kwargs): super().__init__(config, **kwargs) self.num_workers = kwargs.pop("num_workers") or cpu_count() self.start_time = None self.workers = [] port = kwargs.pop("port") host = kwargs.pop("host") self.server_address = f"tcp://{host}:{port}" def exit(self): if hasattr(self, "server_proc"): self.server_proc.terminate() self.server_proc.join() for worker in self.workers: worker.terminate() worker.join() logger.info(f"Total time elapsed: {time.perf_counter() - self.start_time}") def prepare(self): atexit.register(self.exit) self.start_time = time.perf_counter() def spawn_server(self): server = BenchmarkServer(self.db_path, self.server_address) self.server_proc = Process(target=server.run) self.server_proc.start() def spawn_workers(self): worker = BenchmarkWorker(self.server_address) for _ in range(self.num_workers): worker_proc = Process(target=worker.run) worker_proc.start() self.workers.append(worker_proc) def wait(self): self.server_proc.join() for worker in self.workers: worker.join() @click.command("local") @click.option( "-o", "--output-dir", type=click.Path(file_okay=False, writable=True, resolve_path=True), default="./output", show_default=True, ) @click.option("--resume", is_flag=True, default=False) @click.option("-w", "--num-workers", type=int) @click.option("-h", "--host", default="127.0.0.1", show_default=True) @click.option("-p", "--port", default=31313, show_default=True) @click.argument("config", type=click.Path()) def cli(config, **kwargs): config = read_config(config) runner = LocalRunner(config, **kwargs) runner.run() if __name__ == "__main__": cli() PK!~> > $reprobench/runners/slurm/__init__.pyimport math import subprocess import sys from pathlib import Path import click from loguru import logger from reprobench.core.base import Runner from reprobench.core.bootstrap import bootstrap from reprobench.core.db import Run from reprobench.runners.base import BaseRunner from reprobench.utils import get_db_path, init_db, read_config from .utils import create_ranges, get_nodelist class SlurmRunner(BaseRunner): def __init__(self, config, **kwargs): super().__init__(config, **kwargs) self.port = kwargs.pop("port") self.num_workers = kwargs.pop("num_workers", None) def prepare(self): init_db(self.db_path) limits = self.config["limits"] num_jobs = Run.select(Run.id).where(Run.status < Run.DONE).count() jobs_per_worker = int(math.ceil(1.0 * num_jobs / self.num_workers)) time_limit_minutes = int(math.ceil(limits["time"] / 60.0)) self.cpu_count = limits.get("cores", 1) # @TODO improve this self.time_limit = 2 * time_limit_minutes * jobs_per_worker self.mem_limit = 2 * limits["memory"] def spawn_server(self): logger.info("Spawning server...") server_cmd = f"{sys.exec_prefix}/bin/reprobench -vvv server --database={self.db_path} --port={self.port}" server_submit_cmd = [ "sbatch", "--parsable", f"--time={self.time_limit}", f"--job-name={self.config['title']}-benchmark-server", f"--output={self.output_dir}/slurm-server.out", "--wrap", server_cmd, ] logger.debug(server_submit_cmd) self.server_job = subprocess.check_output(server_submit_cmd).decode().strip() logger.info("Waiting for the server to be assigned...") self.server_host = get_nodelist(self.server_job) logger.info(f"Server spawned at {self.server_host}, job id: {self.server_job}") self.server_address = f"tcp://{self.server_host}:{self.port}" def spawn_workers(self): logger.info("Spawning workers...") worker_cmd = f"{sys.exec_prefix}/bin/reprobench -vvv worker --host={self.server_host} --port={self.port}" worker_submit_cmd = [ "sbatch", "--parsable", f"--ntasks={self.num_workers}", f"--time={self.time_limit}", f"--mem={self.mem_limit}", f"--cpus-per-task={self.cpu_count}", f"--job-name={self.config['title']}-benchmark-worker", f"--output={self.output_dir}/slurm-worker.out", "--wrap", f"srun {worker_cmd}", ] logger.debug(worker_submit_cmd) self.worker_job = subprocess.check_output(worker_submit_cmd).decode().strip() logger.info(f"Workers job id: {self.worker_job}") @click.command("slurm") @click.option( "-o", "--output-dir", type=click.Path(file_okay=False, writable=True, resolve_path=True), default="./output", show_default=True, ) @click.option("--resume", is_flag=True, default=False) @click.option("-w", "--num-workers", type=int, required=True) @click.option("-p", "--port", default=31313, show_default=True) @click.argument("config", type=click.Path()) def cli(config, **kwargs): config = read_config(config) runner = SlurmRunner(config, **kwargs) runner.run() if __name__ == "__main__": cli() PK!w !reprobench/runners/slurm/utils.pyimport subprocess from itertools import tee, zip_longest # https://stackoverflow.com/a/3430312/9314778 def pairwise_longest(iterable): "variation of pairwise in http://docs.python.org/library/itertools.html#recipes" a, b = tee(iterable) next(b, None) return zip_longest(a, b) def takeuntil(predicate, iterable): """returns all elements before and including the one for which the predicate is true variation of http://docs.python.org/library/itertools.html#itertools.takewhile""" for x in iterable: yield x if predicate(x): break def get_range(it): "gets a range from a pairwise iterator" rng = list(takeuntil(lambda args: (args[1] is None) or (args[1] - args[0] > 1), it)) if rng: b, e = rng[0][0], rng[-1][0] return "%d-%d" % (b, e) if b != e else str(b) def create_ranges(zones): it = pairwise_longest(zones) return ",".join(iter(lambda: get_range(it), None)) def get_nodelist(job_step): """ Blocks until job step is assigned a node """ while True: cmd = ["sacct", "-n", "--parsable2", "-j", job_step, "-o", "NodeList"] output = subprocess.check_output(cmd) if len(output) > 0 and output != b"None assigned\n": return output.decode().strip() PK!ϗ?reprobench/task_sources/base.pyclass BaseTaskSource(object): def __init__(self, path=None, **kwargs): self.path = path def setup(self): return [] PK!11'reprobench/task_sources/doi/__init__.pyfrom reprobench.task_sources.url import UrlSource from reprobench.task_sources.doi.zenodo import ZenodoHandler, ZenodoSandboxHandler class DOISource(UrlSource): handlers = [ZenodoHandler, ZenodoSandboxHandler] def __init__(self, doi, **kwargs): super().__init__(**kwargs) self.doi = doi for handler in self.handlers: if handler.is_compatible(self.doi): self.urls = handler.get_urls(self.doi) break else: raise NotImplementedError(f"No handler for doi: {doi}") PK!RO#reprobench/task_sources/doi/base.pyclass BaseDOIHandler(object): @classmethod def is_compatible(cls, doi): return False @classmethod def get_urls(cls, doi): return [] PK!Sʺ%reprobench/task_sources/doi/zenodo.pyimport requests from reprobench.task_sources.doi.base import BaseDOIHandler class ZenodoHandler(BaseDOIHandler): doi_prefix = "10.5281/zenodo." api_url = "https://zenodo.org/api" @classmethod def is_compatible(cls, doi): return doi.startswith(cls.doi_prefix) @classmethod def get_urls(cls, doi): record_id = doi[len(cls.doi_prefix) :] # remove doi_prefix url = "{}/records/{}".format(cls.api_url, record_id) record = requests.get(url).json() return [file["links"]["self"] for file in record["files"]] class ZenodoSandboxHandler(ZenodoHandler): doi_prefix = "10.5072/zenodo." api_url = "https://sandbox.zenodo.org/api" PK!< reprobench/task_sources/local.pyfrom pathspec import PathSpec from pathlib import Path from .base import BaseTaskSource class LocalSource(BaseTaskSource): def __init__(self, path=None, patterns="", **kwargs): super().__init__(path) self.patterns = patterns def setup(self): spec = PathSpec.from_lines("gitwildmatch", self.patterns.splitlines()) matches = spec.match_tree(self.path) return map(lambda match: (Path(self.path) / match).resolve(), matches) PK!,assreprobench/task_sources/url.pyfrom pathlib import Path from zipfile import ZipFile from loguru import logger from reprobench.utils import download_file, extract_archives from .local import LocalSource class UrlSource(LocalSource): def __init__( self, urls=None, path=None, patterns="", skip_existing=True, extract_archives=True, **kwargs, ): super().__init__(path, patterns=patterns) self.urls = urls or [] self.extract_archives = extract_archives self.skip_existing = skip_existing def setup(self): root = Path(self.path) root.mkdir(parents=True, exist_ok=True) for url in self.urls: filename = url.split("/")[-1].split("?")[0] path = root / filename if not path.exists() or not self.skip_existing: logger.debug(f"Downloading {url} to {path}") download_file(url, path) else: logger.debug(f"Skipping already downloaded file {path}") if self.extract_archives: extract_archives(path) return super().setup() PK!reprobench/tools/base.pyPK!Rbreprobench/tools/executable.pyfrom pathlib import Path from loguru import logger from reprobench.core.base import Tool class ExecutableTool(Tool): name = "Basic Executable Tool" path = None prefix = "--" @classmethod def is_ready(cls): return True def get_arguments(self): return [f"{self.prefix}{key}={value}" for key, value in self.parameters.items()] def get_cmdline(self): return [self.path, *self.get_arguments()] def get_out_path(self): return Path(self.cwd) / "run.out" def get_err_path(self): return Path(self.cwd) / "run.err" def get_output(self): return self.get_out_path().read_bytes() def get_error(self): return self.get_err_path().read_bytes() def run(self, executor): logger.debug([*self.get_cmdline(), self.task]) executor.run( [*self.get_cmdline(), self.task], directory=self.cwd, out_path=self.get_out_path(), err_path=self.get_err_path(), ) PK!ؙ reprobench/utils.pyimport importlib import logging import os import re import signal import subprocess import tarfile import time import zipfile from pathlib import Path from shutil import which import msgpack import requests import strictyaml from playhouse.apsw_ext import APSWDatabase from tqdm import tqdm from reprobench.core.db import db from reprobench.core.schema import schema from reprobench.core.exceptions import ExecutableNotFoundError log = logging.getLogger(__name__) def find_executable(executable): path = which(executable) if path is None: raise ExecutableNotFoundError return path def silent_run(command): log.debug(f"Running: {command}") return subprocess.run(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def import_class(path): module_path, tail = ".".join(path.split(".")[:-1]), path.split(".")[-1] module = importlib.import_module(module_path) return getattr(module, tail) def copyfileobj(fsrc, fdst, callback, length=16 * 1024): while True: buf = fsrc.read(length) if not buf: break fdst.write(buf) callback(len(buf)) def download_file(url, dest): r = requests.get(url, stream=True) with tqdm( total=int(r.headers.get("content-length", 0)), unit="B", unit_scale=True, unit_divisor=1024, ) as progress_bar: progress_bar.set_postfix(file=Path(dest).name, refresh=False) with open(dest, "wb") as f: copyfileobj(r.raw, f, progress_bar.update) ranged_numbers_re = re.compile(r"(?P\d+)\.\.(?P\d+)(\.\.(?P\d+))?") def is_range_str(range_str): return ranged_numbers_re.match(range_str) def str_to_range(range_str): matches = ranged_numbers_re.match(range_str).groupdict() start = int(matches["start"]) end = int(matches["end"]) if matches["step"]: return range(start, end, int(matches["step"])) return range(start, end) def encode_message(obj): return msgpack.packb(obj, use_bin_type=True) def decode_message(msg): return msgpack.unpackb(msg, raw=False) def send_event(socket, event_type, payload=None): """ Used in the worker with a DEALER socket """ socket.send_multipart([event_type, encode_message(payload)]) def recv_event(socket): """ Used in the SUB handler """ event_type, payload, address = socket.recv_multipart() return event_type, decode_message(payload), address def clean_up(): signal.signal(signal.SIGTERM, signal.SIG_IGN) os.killpg(os.getpgid(0), signal.SIGTERM) time.sleep(1) os.killpg(os.getpgid(0), signal.SIGKILL) def get_db_path(output_dir): return str((Path(output_dir) / f"benchmark.db").resolve()) def init_db(db_path): database = APSWDatabase(db_path) db.initialize(database) def read_config(config_path): with open(config_path, "r") as f: config_text = f.read() config = strictyaml.load(config_text, schema=schema).data return config def extract_zip(path, dest): if not dest.is_dir(): with zipfile.ZipFile(path, "r") as f: f.extractall(dest) def extract_tar(path, dest): if not dest.is_dir(): with tarfile.TarFile.open(path) as f: f.extractall(dest) def extract_archives(path): extract_path = Path(path).with_name(path.stem) if zipfile.is_zipfile(path): extract_zip(path, extract_path) elif tarfile.is_tarfile(path): extract_tar(path, extract_path) PK!HJ.:+reprobench-0.7.3.dist-info/entry_points.txtN+I/N.,()*J-(OJKΰE0r3s2PK!XB33"reprobench-0.7.3.dist-info/LICENSEMIT License Copyright (c) 2019 Rakha Kanz Kautsar Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PK!Hu)GTU reprobench-0.7.3.dist-info/WHEEL HM K-*ϳR03rOK-J,/R(O-)$qzd&Y)r$UV&UrPK!H#reprobench-0.7.3.dist-info/METADATAoO0Sd8ICctP*` 4!׹&^NYM|<].Ak}(Gn 3 Ilȹ)ò B,j+wɥy 98_9OT6;]9@L9)tʥW)vqdVXnR0L3g@3 *AyҠFIziK ;W7g0۳i~qe.y"1c4O 'Qo}%_nwT鑋zta{|kd,GA{n,Dp_Z7ky]-ݹDaV6Fqp:trh@xc*Yy_d[e^)q'W$F` ]=Fw8<K%nU}۝[ӌ*= _ВW3"I l(ξ1\UP@;KZ5]g'9#cＶis3A}.x+c<\b'K(*/hjαjDWh#6H)I&;V*LlB_tI5~ҌP='L堗G- ̵bȞ"~qM^X88ޕ;YF#n sKeI>2,m?c𱚇N VxܧN{Uwzt@V (6_ s m/],gt% lvA\bHͲ5+f|W \W"ceI J3.hA^5eDpG:e(T6Nɖ+>MQPB?$'y-cO7Xc# f{uF [>Gx Џw_PD,7\mχDAbFr3PxAѮqoCx!*q+Dn)tѪ(QM[V{Bcv Jea{Y$U󵄣w1?PK!8&&reprobench/__init__.pyPK!#6Zreprobench/console/main.pyPK!Oreprobench/console/status.pyPK!A)reprobench/core/base.pyPK!pc# reprobench/core/bootstrap.pyPK!'ʵ $reprobench/core/db.pyPK!.reprobench/core/events.pyPK!"]+;77C/reprobench/core/exceptions.pyPK!/reprobench/core/observers.pyPK!NZ z7reprobench/core/schema.pyPK!ch h ;reprobench/core/server.pyPK!E3A A dHreprobench/core/sysinfo.pyPK!خ@ @ Rreprobench/core/worker.pyPK!s݊tt T]reprobench/executors/__init__.pyPK!4UU^reprobench/executors/base.pyPK!%o-NNareprobench/executors/db.pyPK!d,C,,freprobench/executors/events.pyPK!^B B freprobench/executors/psmon.pyPK!{preprobench/runners/__init__.pyPK!Վ{{2qreprobench/runners/base.pyPK!E$wreprobench/runners/local/__init__.pyPK!~> > $&reprobench/runners/slurm/__init__.pyPK!w !reprobench/runners/slurm/utils.pyPK!ϗ?reprobench/task_sources/base.pyPK!11'reprobench/task_sources/doi/__init__.pyPK!RO#7reprobench/task_sources/doi/base.pyPK!Sʺ%reprobench/task_sources/doi/zenodo.pyPK!< reprobench/task_sources/local.pyPK!,ass0reprobench/task_sources/url.pyPK!ߡreprobench/tools/base.pyPK!Rbreprobench/tools/executable.pyPK!ؙ Kreprobench/utils.pyPK!HJ.:+:reprobench-0.7.3.dist-info/entry_points.txtPK!XB33"reprobench-0.7.3.dist-info/LICENSEPK!Hu)GTU $reprobench-0.7.3.dist-info/WHEELPK!H#reprobench-0.7.3.dist-info/METADATAPK!HoL=2 !reprobench-0.7.3.dist-info/RECORDPK%% p