PK!6reprobench/__init__.pyname = "reprobench" PK!?6 reprobench/console/decorators.pyimport functools import os import sys from pathlib import Path import click from loguru import logger def common(func): @click.option("-q", "--quiet", is_flag=True) @click.option( "--verbose", "-v", "verbosity", count=True, default=0, help="Verbosity" ) @functools.wraps(func) def wrapper(*args, **kwargs): quiet = kwargs.pop("quiet") verbosity = kwargs.pop("verbosity") sys.path.append(os.getcwd()) logger.remove() if not quiet: verbosity_levels = ["INFO", "DEBUG", "TRACE"] verbosity = min(verbosity, 2) logger.add(sys.stderr, level=verbosity_levels[verbosity]) return func(*args, **kwargs) return wrapper def server_info(func): @click.option("-a", "--address", default="tcp://127.0.0.1:31313", show_default=True) @functools.wraps(func) def wrapper(*args, **kwargs): server_address = kwargs.pop("address") return func(server_address=server_address, *args, **kwargs) return wrapper def use_tunneling(func): @click.option("-h", "--host", required=False, help="[Tunneling] SSH Host") @click.option( "-p", "--port", help="[Tunneling] Remote server port for", default=31313 ) @click.option( "-K", "--key-file", type=click.Path(exists=True), help="[Tunneling] SSH private key file", required=False, ) @click.option( "-C", "--ssh-config-file", help="[Tunneling] SSH config file", default=Path.home() / ".ssh" / "config", ) @functools.wraps(func) def wrapper(host, port, key_file, ssh_config_file, *args, **kwargs): if host is not None: tunneling = dict( host=host, port=port, key_file=key_file, ssh_config_file=ssh_config_file ) return func(tunneling=tunneling, *args, **kwargs) return func(tunneling=None, *args, **kwargs) return wrapper PK!#reprobench/console/main.py#!/usr/bin/env python import click from .status import benchmark_status @click.group() @click.version_option() def cli(): pass try: from reprobench.core.server import cli as server_cli cli.add_command(server_cli) cli.add_command(benchmark_status) except ImportError: pass try: from reprobench.core.worker import cli as worker_cli cli.add_command(worker_cli) except ImportError: pass try: from reprobench.managers import cli as manager_cli cli.add_command(manager_cli) except ImportError: pass try: from reprobench.core.analyzer import cli as analyzer_cli cli.add_command(analyzer_cli) except ImportError: pass if __name__ == "__main__": cli() PK!28)22reprobench/console/status.pyimport time import click from tqdm import tqdm from reprobench.utils import init_db try: from reprobench.core.db import Run except ImportError: Run = None def get_total_count(): return Run.select().count() def get_done_count(): return Run.select().where(Run.status == Run.DONE).count() @click.command("status") @click.option("-d", "--database", default="./output/benchmark.db", show_default=True) @click.option("-n", "--interval", default=2, show_default=True, type=int) def benchmark_status(database, interval): init_db(database) total = get_total_count() last = get_done_count() progress = tqdm(total=total, initial=last) while last < total: time.sleep(interval) current = get_done_count() progress.update(current - last) last = current PK!Iفreprobench/core/analyzer.pyimport click from loguru import logger from playhouse.apsw_ext import APSWDatabase from reprobench.console.decorators import common from reprobench.core.db import Step, db from reprobench.utils import get_db_path, import_class, init_db, read_config class BenchmarkAnalyzer(object): def __init__(self, output_dir, config, **kwargs): self.output_dir = output_dir self.config = read_config(config) self.db_path = get_db_path(output_dir) init_db(self.db_path) def run(self): steps = self.config["steps"]["analysis"] context = dict(output_dir=self.output_dir, db_path=self.db_path) for step in steps: logger.debug(f"Running {step['module']}") module = import_class(step["module"]) module.execute(context, step["config"]) @click.command(name="analyze") @click.option( "-d", "--output-dir", type=click.Path(), default="./output", show_default=True ) @click.argument("config", type=click.Path(), default="./benchmark.yml") @common def cli(**kwargs): analyzer = BenchmarkAnalyzer(**kwargs) analyzer.run() if __name__ == "__main__": cli() PK!,AAreprobench/core/base.pyfrom reprobench.utils import recv_event try: import zmq.green as zmq except ImportError: pass class Observer: SUBSCRIBED_EVENTS = [] @classmethod def observe(cls, context, backend_address, reply): observe_args = (context, backend_address, reply) socket = context.socket(zmq.SUB) socket.connect(backend_address) for event in cls.SUBSCRIBED_EVENTS: socket.setsockopt(zmq.SUBSCRIBE, event) while True: event_type, payload, address = recv_event(socket) cls.handle_event( event_type, payload, reply=reply, address=address, observe_args=observe_args, ) @classmethod def handle_event(cls, event_type, payload, **kwargs): pass class Step: @classmethod def register(cls, config=None): pass @classmethod def execute(cls, context, config=None): pass class Tool: name = "Base Tool" def __init__(self, context): self.cwd = context["run"]["id"] self.parameters = context["run"]["parameters"] self.task = context["run"]["task"] def run(self, executor): raise NotImplementedError def get_output(self): raise NotImplementedError def get_error(self): raise NotImplementedError @classmethod def setup(cls): pass @classmethod def version(cls): return "1.0.0" @classmethod def is_ready(cls): pass @classmethod def teardown(cls): pass PK! dd%reprobench/core/bootstrap/__init__.pyfrom .client import bootstrap as bootstrap_client from .server import bootstrap as bootstrap_server PK!|--#reprobench/core/bootstrap/client.pyfrom loguru import logger from reprobench.task_sources.doi import DOISource from reprobench.task_sources.file import FileSource from reprobench.task_sources.url import UrlSource from reprobench.utils import import_class def bootstrap_tasks(config): logger.info("Bootstrapping tasks...") available_sources = (FileSource, UrlSource, DOISource) logger.trace(config) task_groups = {} for (group, task) in config["tasks"].items(): logger.trace(f"Processing task group: {group}") for TaskSource in available_sources: if task["type"] == TaskSource.TYPE: source = TaskSource(**task) break else: raise NotImplementedError( f"No implementation for task source {task['type']}" ) tasks = source.setup() task_groups[group] = [str(task) for task in tasks] return task_groups def bootstrap_tools(config): logger.info("Setting up tools...") tools = {} for tool_name, tool in config["tools"].items(): tools[tool_name] = dict( module=tool["module"], parameters=tool.get("parameters") ) return tools def bootstrap(config): tasks = bootstrap_tasks(config) tools = bootstrap_tools(config) return dict(tasks=tasks, tools=tools) PK!QBB#reprobench/core/bootstrap/server.pyimport atexit import itertools import json import shutil from pathlib import Path import gevent from loguru import logger from peewee import chunked from tqdm import tqdm from reprobench.core.db import ( MODELS, Limit, Observer, Parameter, ParameterGroup, Run, Step, Task, TaskGroup, Tool, db, ) from reprobench.utils import ( check_valid_config_space, get_db_path, import_class, init_db, is_range_str, parse_pcs_parameters, str_to_range, ) try: from ConfigSpace.read_and_write import pcs except ImportError: pcs = None def bootstrap_db(output_dir): db_path = get_db_path(output_dir) init_db(db_path) db.connect() db.create_tables(MODELS, safe=True) def bootstrap_limits(config): # TODO: handle limit changes query = Limit.insert_many( [{"key": key, "value": value} for (key, value) in config["limits"].items()] ).on_conflict("ignore") query.execute() def bootstrap_steps(config): count = Step.select().count() new_steps = config["steps"]["run"][count:] if len(new_steps) > 0: query = Step.insert_many( [ { "category": "run", "module": step["module"], "config": json.dumps(step.get("config", None)), } for step in new_steps ] ) query.execute() def bootstrap_observers(config, observe_args): count = Observer.select().count() new_observers = config["observers"][count:] if len(new_observers) > 0: query = Observer.insert_many( [ { "module": observer["module"], "config": json.dumps(observer.get("config", None)), } for observer in new_observers ] ) query.execute() for observer in new_observers: observer_class = import_class(observer["module"]) gevent.spawn(observer_class.observe, *observe_args) def register_steps(config): logger.info("Registering steps...") for step in itertools.chain.from_iterable(config["steps"].values()): import_class(step["module"]).register(step.get("config", {})) def bootstrap_tasks(config): for (name, tasks) in config["tasks"].items(): TaskGroup.insert(name=name).on_conflict("ignore").execute() with db.atomic(): for batch in chunked(tasks, 100): query = Task.insert_many( [{"path": task, "group": name} for task in batch] ).on_conflict("ignore") query.execute() def create_parameter_group(tool, group, parameters): PCS_KEY = "__pcs" pcs_parameters = {} use_pcs = PCS_KEY in parameters config_space = None if use_pcs: pcs_text = parameters.pop(PCS_KEY) lines = pcs_text.split("\n") config_space = pcs.read(lines) pcs_parameters = parse_pcs_parameters(lines) ranged_enum_parameters = { key: value for key, value in parameters.items() if isinstance(parameters[key], list) } ranged_numbers_parameters = { key: str_to_range(value) for key, value in parameters.items() if isinstance(value, str) and is_range_str(value) } ranged_parameters = { **pcs_parameters, **ranged_enum_parameters, **ranged_numbers_parameters, } if len(ranged_parameters) == 0: parameter_group, _ = ParameterGroup.get_or_create(name=group, tool=tool) for (key, value) in parameters.items(): query = Parameter.insert( group=parameter_group, key=key, value=value ).on_conflict("replace") query.execute() return constant_parameters = { key: value for key, value in parameters.items() if key not in ranged_parameters } tuples = [ [(key, value) for value in values] for key, values in ranged_parameters.items() ] for combination in itertools.product(*tuples): parameters = {**dict(combination), **constant_parameters} if use_pcs: check_valid_config_space(config_space, parameters) combination_str = ",".join(f"{key}={value}" for key, value in combination) group_name = f"{group}[{combination_str}]" parameter_group, _ = ParameterGroup.get_or_create(name=group_name, tool=tool) for (key, value) in parameters.items(): query = Parameter.insert( group=parameter_group, key=key, value=value ).on_conflict("replace") query.execute() def bootstrap_tools(config): logger.info("Bootstrapping tools...") for tool_name, tool in config["tools"].items(): query = Tool.insert(name=tool_name, module=tool["module"]).on_conflict( "replace" ) query.execute() if "parameters" not in tool: create_parameter_group(tool_name, "default", {}) continue for group, parameters in tool["parameters"].items(): create_parameter_group(tool_name, group, parameters) def bootstrap_runs(config, output_dir, repeat=1): parameter_groups = ParameterGroup.select().iterator() tasks = Task.select().iterator() total = ParameterGroup.select().count() * Task.select().count() with db.atomic(): for (parameter_group, task) in tqdm( itertools.product(parameter_groups, tasks), desc="Bootstrapping runs", total=total, ): for iteration in range(repeat): directory = ( Path(output_dir) / parameter_group.tool_id / parameter_group.name / task.group_id / Path(task.path).name / str(iteration) ) query = Run.insert( id=directory, tool=parameter_group.tool_id, task=task, parameter_group=parameter_group, status=Run.PENDING, iteration=iteration, ).on_conflict("ignore") query.execute() def bootstrap(config=None, output_dir=None, repeat=1, observe_args=None): Path(output_dir).mkdir(parents=True, exist_ok=True) bootstrap_db(output_dir) bootstrap_limits(config) bootstrap_steps(config) bootstrap_observers(config, observe_args) register_steps(config) bootstrap_tasks(config) bootstrap_tools(config) bootstrap_runs(config, output_dir, repeat) PK!reprobench/core/db.pyfrom datetime import datetime from playhouse.apsw_ext import ( Model, Proxy, CharField, CompositeKey, DateTimeField, ForeignKeyField, IntegerField, TextField, ) db = Proxy() class BaseModel(Model): class Meta: database = db class Limit(BaseModel): key = CharField(max_length=32, primary_key=True) value = CharField() class TaskGroup(BaseModel): name = CharField(primary_key=True) class Task(BaseModel): group = ForeignKeyField(TaskGroup, backref="tasks") path = CharField(primary_key=True) class Tool(BaseModel): module = CharField() name = CharField(primary_key=True) class ParameterGroup(BaseModel): name = CharField() tool = ForeignKeyField(Tool, backref="parameter_groups") class Meta: indexes = ((("name", "tool"), True),) class Parameter(BaseModel): group = ForeignKeyField(ParameterGroup, backref="parameters") key = CharField() value = CharField() class Meta: primary_key = CompositeKey("group", "key") class BasePlugin(BaseModel): module = CharField(index=True) config = TextField() class Step(BasePlugin): RUN = "run" ANALYSIS = "analysis" CATEGORY_CHOICES = ((RUN, "Single run step"), (ANALYSIS, "Analysis step")) category = CharField(choices=CATEGORY_CHOICES, index=True) class Observer(BasePlugin): pass class Run(BaseModel): FAILED = -2 CANCELED = -1 PENDING = 0 SUBMITTED = 1 RUNNING = 2 DONE = 3 STATUS_CHOICES = ( (FAILED, "Failed"), (CANCELED, "Canceled"), (PENDING, "Pending"), (SUBMITTED, "Submitted"), (RUNNING, "Running"), (DONE, "Done"), ) id = CharField(null=True, primary_key=True) created_at = DateTimeField(default=datetime.now) tool = ForeignKeyField(Tool, backref="runs") tool_version = CharField(null=True) parameter_group = ForeignKeyField(ParameterGroup, backref="runs") task = ForeignKeyField(Task, backref="runs") status = IntegerField(choices=STATUS_CHOICES, default=PENDING) last_step = ForeignKeyField(Step, null=True) iteration = IntegerField(default=0) MODELS = (Limit, TaskGroup, Task, Tool, ParameterGroup, Parameter, Run, Step, Observer) PK!|0_reprobench/core/events.pyBOOTSTRAP = b"core:bootstrap" WORKER_JOIN = b"worker:join" WORKER_LEAVE = b"worker:leave" RUN_START = b"run:start" RUN_STEP = b"run:step" RUN_INTERRUPT = b"run:interrupt" RUN_FINISH = b"run:finish" PK!a_hhreprobench/core/exceptions.pyclass ExecutableNotFoundError(RuntimeError): pass class NotSupportedError(RuntimeError): pass PK! reprobench/core/observers.pyfrom functools import lru_cache from peewee import fn from reprobench.core.base import Observer from reprobench.core.bootstrap.server import bootstrap from reprobench.core.db import Limit, Run, Step from reprobench.core.events import ( BOOTSTRAP, RUN_FINISH, RUN_INTERRUPT, RUN_START, RUN_STEP, WORKER_JOIN, ) from reprobench.utils import encode_message class CoreObserver(Observer): SUBSCRIBED_EVENTS = (BOOTSTRAP, WORKER_JOIN, RUN_START, RUN_STEP, RUN_FINISH) @classmethod @lru_cache(maxsize=1) def get_limits(cls): return {l.key: l.value for l in Limit.select()} @classmethod def get_next_pending_run(cls): try: run = Run.select().where(Run.status == Run.PENDING).limit(1).get() except Run.DoesNotExist: return None run.status = Run.SUBMITTED run.save() last_step = run.last_step_id or 0 runsteps = Step.select().where( (Step.category == Step.RUN) & (Step.id > last_step) ) limits = cls.get_limits() parameters = {p.key: p.value for p in run.parameter_group.parameters} run_dict = dict( id=run.id, task=run.task_id, tool=run.tool.module, parameters=parameters, steps=list(runsteps.dicts()), limits=limits, ) return run_dict @classmethod def get_pending_runs(cls): last_step = ( Step.select(fn.MAX(Step.id)).where(Step.category == Step.RUN).scalar() ) Run.update(status=Run.PENDING).where( (Run.status < Run.DONE) | (Run.last_step_id != last_step) ).execute() pending_runs = Run.select(Run.id).where(Run.status == Run.PENDING).count() return pending_runs @classmethod def handle_event(cls, event_type, payload, **kwargs): reply = kwargs.pop("reply") address = kwargs.pop("address") observe_args = kwargs.pop("observe_args") if event_type == BOOTSTRAP: bootstrap(observe_args=observe_args, **payload) pending_runs = cls.get_pending_runs() reply.send_multipart([address, encode_message(pending_runs)]) elif event_type == WORKER_JOIN: run = cls.get_next_pending_run() reply.send_multipart([address, encode_message(run)]) elif event_type == RUN_INTERRUPT: Run.update(status=Run.PENDING).where(Run.id == payload).execute() elif event_type == RUN_START: run_id = payload.pop("run_id") Run.update(status=Run.RUNNING, **payload).where(Run.id == run_id).execute() elif event_type == RUN_STEP: step = Step.get(module=payload["step"]) Run.update(last_step=step).where(Run.id == payload["run_id"]).execute() elif event_type == RUN_FINISH: Run.update(status=Run.DONE).where(Run.id == payload).execute() PK!~~reprobench/core/schema.pyfrom strictyaml import Any, Enum, Int, Map, MapPattern, Optional, Regex, Seq, Str limits_schema = Map( { "time": Int(), Optional("memory", default=8192): Int(), Optional("output"): Int(), Optional("cores"): Int(), } ) module_schema = Regex(r"\.?\w+(\.\w+)*") plugin_schema = Map( {"module": module_schema, Optional("config"): MapPattern(Str(), Any())} ) task_sources = Enum(["local", "url"]) schema = Map( { "title": Str(), Optional("description"): Str(), "limits": limits_schema, "steps": Map( {"run": Seq(plugin_schema), Optional("analysis"): Seq(plugin_schema)} ), "observers": Seq(plugin_schema), "tasks": MapPattern(Str(), MapPattern(Str(), Any())), "tools": MapPattern( Str(), Map( { "module": module_schema, Optional("parameters"): MapPattern(Str(), MapPattern(Str(), Any())), } ), ), } ) PK!$JJreprobench/core/server.pyfrom pathlib import Path import click import gevent import zmq.green as zmq from loguru import logger from reprobench.console.decorators import common, server_info from reprobench.core.bootstrap.server import bootstrap from reprobench.core.db import Observer from reprobench.core.events import BOOTSTRAP from reprobench.core.observers import CoreObserver from reprobench.utils import decode_message, import_class class BenchmarkServer(object): BACKEND_ADDRESS = "inproc://backend" def __init__(self, frontend_address, **kwargs): self.frontend_address = frontend_address def receive_event(self): address, event_type, payload = self.frontend.recv_multipart() logger.trace((address, event_type, decode_message(payload))) return address, event_type, payload def loop(self): while True: address, event_type, payload = self.receive_event() self.backend.send_multipart([event_type, payload, address]) def run(self): self.context = zmq.Context() self.frontend = self.context.socket(zmq.ROUTER) self.frontend.bind(self.frontend_address) self.backend = self.context.socket(zmq.PUB) self.backend.bind(self.BACKEND_ADDRESS) core_observer_greenlet = gevent.spawn( CoreObserver.observe, self.context, backend_address=self.BACKEND_ADDRESS, reply=self.frontend, ) logger.info(f"Listening on {self.frontend_address}...") serverlet = gevent.spawn(self.loop) logger.info(f"Ready to receive events...") serverlet.join() core_observer_greenlet.kill() @click.command(name="server") @server_info @common def cli(server_address, **kwargs): server = BenchmarkServer(server_address, **kwargs) server.run() if __name__ == "__main__": cli() PK!; reprobench/core/sysinfo.pyimport platform from playhouse.apsw_ext import CharField, FloatField, ForeignKeyField, IntegerField from reprobench.core.base import Step, Observer from reprobench.core.db import BaseModel, Run, db from reprobench.utils import send_event try: import psutil from cpuinfo import get_cpu_info except ImportError: psutil = None get_cpu_info = None class Node(BaseModel): hostname = CharField(primary_key=True) platform = CharField(null=True) arch = CharField(null=True) python = CharField(null=True) cpu = CharField(null=True) cpu_count = IntegerField(null=True) cpu_min_freq = FloatField(null=True) cpu_max_freq = FloatField(null=True) mem_total = IntegerField(null=True) mem_available = IntegerField(null=True) swap_total = IntegerField(null=True) swap_available = IntegerField(null=True) class RunNode(BaseModel): run = ForeignKeyField(Run, backref="run_node", primary_key=True) node = ForeignKeyField(Node, backref="runs") MODELS = (Node, RunNode) STORE_SYSINFO = b"sysinfo:store" class SystemInfoObserver(Observer): SUBSCRIBED_EVENTS = (STORE_SYSINFO,) @classmethod def handle_event(cls, event_type, payload, **kwargs): if event_type == STORE_SYSINFO: node = payload["node"] run = payload["run_id"] Node.insert(**node).on_conflict("ignore").execute() RunNode.insert(run=run, node=node["hostname"]).on_conflict( "replace" ).execute() class CollectSystemInfo(Step): @classmethod def register(cls, config=None): db.create_tables(MODELS) @classmethod def _get_system_info(cls): cpu_info = get_cpu_info() cpu_freq = psutil.cpu_freq() mem_info = psutil.virtual_memory() swap_info = psutil.swap_memory() info = {} info["platform"] = platform.platform(aliased=True) info["arch"] = cpu_info["arch"] info["python"] = cpu_info["python_version"] info["cpu"] = cpu_info["brand"] info["cpu_count"] = psutil.cpu_count() info["cpu_min_freq"] = cpu_freq.min info["cpu_max_freq"] = cpu_freq.max info["mem_total"] = mem_info.total info["mem_available"] = mem_info.available info["swap_total"] = swap_info.total info["swap_available"] = swap_info.free return info @classmethod def execute(cls, context, config=None): hostname = platform.node() info = cls._get_system_info() run_id = context["run"]["id"] payload = dict(run_id=run_id, node=dict(hostname=hostname, **info)) send_event(context["socket"], STORE_SYSINFO, payload) PK!cV V reprobench/core/worker.pyimport sys import atexit import json from pathlib import Path import click import zmq from sshtunnel import SSHTunnelForwarder from loguru import logger from reprobench.console.decorators import common, server_info, use_tunneling from reprobench.core.events import ( RUN_FINISH, RUN_INTERRUPT, RUN_START, RUN_STEP, WORKER_JOIN, WORKER_LEAVE, ) from reprobench.utils import decode_message, import_class, send_event REQUEST_TIMEOUT = 15000 class BenchmarkWorker: def __init__(self, server_address, tunneling): self.server_address = server_address if tunneling is not None: self.server = SSHTunnelForwarder( tunneling["host"], remote_bind_address=("127.0.0.1", tunneling["port"]), ssh_pkey=tunneling["key_file"], ssh_config_file=tunneling["ssh_config_file"], ) # https://github.com/pahaz/sshtunnel/issues/138 if sys.version_info[0] > 3 or ( sys.version_info[0] == 3 and sys.version_info[1] >= 7 ): self.server.daemon_forward_servers = True self.server.start() self.server_address = f"tcp://127.0.0.1:{self.server.local_bind_port}" logger.info(f"Tunneling established at {self.server_address}") atexit.register(self.stop_tunneling) def killed(self, run_id): send_event(self.socket, RUN_INTERRUPT, run_id) send_event(self.socket, WORKER_LEAVE) def stop_tunneling(self): self.server.stop() def run(self): context = zmq.Context() self.socket = context.socket(zmq.DEALER) logger.debug(f"Connecting to {self.server_address}") self.socket.connect(self.server_address) send_event(self.socket, WORKER_JOIN) run = decode_message(self.socket.recv()) self.run_id = run["id"] atexit.register(self.killed, self.run_id) tool = import_class(run["tool"]) if not tool.is_ready(): tool.setup() context = {} context["socket"] = self.socket context["tool"] = tool context["run"] = run logger.info(f"Processing task: {run['id']}") directory = Path(run["id"]) directory.mkdir(parents=True, exist_ok=True) payload = dict(tool_version=tool.version(), run_id=self.run_id) send_event(self.socket, RUN_START, payload) for runstep in run["steps"]: logger.debug(f"Running step {runstep['module']}") step = import_class(runstep["module"]) config = json.loads(runstep["config"]) step.execute(context, config) payload = {"run_id": self.run_id, "step": runstep["module"]} send_event(self.socket, RUN_STEP, payload) send_event(self.socket, RUN_FINISH, self.run_id) atexit.unregister(self.killed) send_event(self.socket, WORKER_LEAVE, self.run_id) @click.command("worker") @server_info @use_tunneling @common def cli(**kwargs): worker = BenchmarkWorker(**kwargs) worker.run() if __name__ == "__main__": cli() PK!s݊tt reprobench/executors/__init__.pyfrom .base import RunStatisticObserver # from .runsolver import RunsolverExecutor from .psmon import PsmonExecutor PK!(Xrreprobench/executors/base.pyfrom reprobench.core.base import Step, Observer from reprobench.executors.events import STORE_RUNSTATS from .db import RunStatistic class RunStatisticObserver(Observer): SUBSCRIBED_EVENTS = (STORE_RUNSTATS,) @classmethod def handle_event(cls, event_type, payload, **kwargs): if event_type == STORE_RUNSTATS: RunStatistic.insert(**payload).on_conflict("replace").execute() class Executor(Step): def __init__(self, *args, **kwargs): pass def run( self, cmdline, out_path=None, err_path=None, input_str=None, directory=None, **kwargs ): raise NotImplementedError @classmethod def register(cls, config=None): RunStatistic.create_table() @classmethod def execute(cls, context, config=None): tool = context["tool"] executor = cls(context, config) tool(context).run(executor) PK!%o-NNreprobench/executors/db.pyfrom datetime import datetime from reprobench.core.db import BaseModel, Run from playhouse.apsw_ext import ( ForeignKeyField, FloatField, CharField, IntegerField, DateTimeField, ) class RunStatistic(BaseModel): TIMEOUT = "TLE" MEMOUT = "MEM" RUNTIME_ERR = "RTE" OUTPUT_LIMIT = "OLE" SUCCESS = "OK" VERDICT_CHOICES = ( (TIMEOUT, "Time Limit Exceeded"), (MEMOUT, "Memory Limit Exceeded"), (RUNTIME_ERR, "Runtime Error"), (OUTPUT_LIMIT, "Output Limit Exceeded"), (SUCCESS, "Run Successfully"), ) created_at = DateTimeField(default=datetime.now) run = ForeignKeyField( Run, backref="statistics", on_delete="cascade", primary_key=True ) cpu_time = FloatField(help_text="CPU Time (s)", null=True) wall_time = FloatField(help_text="Wall Clock Time (s)", null=True) max_memory = FloatField(help_text="Max Memory Usage (KiB)", null=True) return_code = IntegerField(help_text="Process Return Code", null=True) verdict = CharField(choices=VERDICT_CHOICES, max_length=3, null=True) PK!d,C,,reprobench/executors/events.pySTORE_RUNSTATS = b"executor:store_runstats" PK![ I I reprobench/executors/psmon.pyfrom loguru import logger try: from psmon import ProcessMonitor from psmon.limiters import CpuTimeLimiter, MaxMemoryLimiter, WallTimeLimiter except ImportError: logger.warning( "You may need to install the `psmon` extra to run with this executor." ) from reprobench.utils import send_event from .base import Executor from .db import RunStatistic from .events import STORE_RUNSTATS class PsmonExecutor(Executor): def __init__(self, context, config): self.socket = context["socket"] self.run_id = context["run"]["id"] if config is None: config = {} wall_grace = config.get("wall_grace", 15) self.nonzero_as_rte = config.get("nonzero_rte", True) limits = context["run"]["limits"] time_limit = float(limits["time"]) MB = 1024 * 1024 self.wall_limit = time_limit + wall_grace self.cpu_limit = time_limit self.mem_limit = float(limits["memory"]) * MB def compile_stats(self, stats): verdict = None if stats["error"] == TimeoutError: verdict = RunStatistic.TIMEOUT elif stats["error"] == MemoryError: verdict = RunStatistic.MEMOUT elif stats["error"] or (self.nonzero_as_rte and stats["return_code"] != 0): verdict = RunStatistic.RUNTIME_ERR else: verdict = RunStatistic.SUCCESS del stats["error"] return dict( run_id=self.run_id, verdict=verdict, cpu_time=stats["cpu_time"], wall_time=stats["wall_time"], max_memory=stats["max_memory"], return_code=stats["return_code"], ) def run( self, cmdline, out_path=None, err_path=None, input_str=None, directory=None, **kwargs, ): out_file = open(out_path, "wb") err_file = open(err_path, "wb") monitor = ProcessMonitor( cmdline, cwd=directory, stdout=out_file, stderr=err_file, input=input_str, freq=15, ) monitor.subscribe("wall_time", WallTimeLimiter(self.wall_limit)) monitor.subscribe("cpu_time", CpuTimeLimiter(self.cpu_limit)) monitor.subscribe("max_memory", MaxMemoryLimiter(self.mem_limit)) logger.debug(f"Running {directory}") stats = monitor.run() logger.debug(f"Finished {directory}") out_file.close() err_file.close() payload = self.compile_stats(stats) send_event(self.socket, STORE_RUNSTATS, payload) PK!Greprobench/managers/__init__.pyimport click from .local import LocalManager from .local import cli as local_cli from .slurm import SlurmManager from .slurm import cli as slurm_cli @click.group("manage") def cli(): pass cli.add_command(local_cli) cli.add_command(slurm_cli) PK!errreprobench/managers/base.pyimport zmq from loguru import logger from reprobench.core.events import BOOTSTRAP from reprobench.core.bootstrap import bootstrap_client from reprobench.utils import decode_message, read_config, send_event class BaseManager(object): def __init__(self, config, server_address, tunneling, **kwargs): self.server_address = server_address self.tunneling = tunneling self.config = read_config(config, resolve_files=True) self.output_dir = kwargs.pop("output_dir") self.repeat = kwargs.pop("repeat") context = zmq.Context() self.socket = context.socket(zmq.DEALER) def prepare(self): pass def spawn_workers(self): raise NotImplementedError def bootstrap(self): self.socket.connect(self.server_address) client_results = bootstrap_client(self.config) bootstrapped_config = {**self.config, **client_results} logger.info("Sending bootstrap event to server") payload = dict( config=bootstrapped_config, output_dir=self.output_dir, repeat=self.repeat ) send_event(self.socket, BOOTSTRAP, payload) self.pending = decode_message(self.socket.recv()) def wait(self): pass def stop(self): pass def run(self): self.prepare() self.bootstrap() self.spawn_workers() self.wait() PK!A?&&%reprobench/managers/local/__init__.pyfrom multiprocessing import cpu_count import click from loguru import logger from reprobench.console.decorators import server_info, common, use_tunneling from .manager import LocalManager @click.command("local") @click.option("-w", "--num-workers", type=int, default=cpu_count(), show_default=True) @click.option( "-d", "--output-dir", type=click.Path(), default="./output", show_default=True ) @click.option("-r", "--repeat", type=int, default=1) @click.argument("command", type=click.Choice(("run",))) @click.argument("config", type=click.Path(), default="./benchmark.yml") @server_info @use_tunneling @common def cli(command, **kwargs): manager = LocalManager(**kwargs) if command == "run": manager.run() # TODO: add run_with_server if __name__ == "__main__": cli() PK!Z$reprobench/managers/local/manager.pyimport atexit import time import sys from multiprocessing import Pool from sshtunnel import SSHTunnelForwarder from loguru import logger from tqdm import tqdm from reprobench.core.worker import BenchmarkWorker from reprobench.managers.base import BaseManager class LocalManager(BaseManager): def __init__(self, **kwargs): super().__init__(**kwargs) self.num_workers = kwargs.pop("num_workers") self.start_time = None self.workers = [] def exit(self): for worker in self.workers: worker.terminate() worker.join() logger.info(f"Total time elapsed: {time.perf_counter() - self.start_time}") def prepare(self): atexit.register(self.exit) self.start_time = time.perf_counter() if self.tunneling is not None: self.server = SSHTunnelForwarder( self.tunneling["host"], remote_bind_address=("127.0.0.1", self.tunneling["port"]), ssh_pkey=self.tunneling["key_file"], ssh_config_file=self.tunneling["ssh_config_file"], ) # https://github.com/pahaz/sshtunnel/issues/138 if sys.version_info[0] > 3 or ( sys.version_info[0] == 3 and sys.version_info[1] >= 7 ): self.server.daemon_forward_servers = True self.server.start() self.server_address = f"tcp://127.0.0.1:{self.server.local_bind_port}" logger.info(f"Tunneling established at {self.server_address}") @staticmethod def spawn_worker(server_address): worker = BenchmarkWorker(server_address) worker.run() def spawn_workers(self): self.pool = Pool(self.num_workers) jobs = (self.server_address for _ in range(self.pending)) self.pool_iterator = self.pool.imap_unordered(self.spawn_worker, jobs) self.pool.close() def wait(self): progress_bar = tqdm(desc="Executing runs", total=self.pending) for _ in self.pool_iterator: progress_bar.update() progress_bar.close() self.pool.join() if self.tunneling is not None: self.server.stop() PK!v]%reprobench/managers/slurm/__init__.pyimport click from reprobench.console.decorators import common, server_info, use_tunneling from reprobench.utils import read_config from .manager import SlurmManager @click.command("slurm") @click.option( "-d", "--output-dir", type=click.Path(), default="./output", show_default=True ) @click.option("-r", "--repeat", type=int, default=1) @click.argument("command", type=click.Choice(("run", "stop"))) @click.argument("config", type=click.Path(), default="./benchmark.yml") @server_info @use_tunneling @common def cli(command, **kwargs): manager = SlurmManager(**kwargs) if command == "run": manager.run() elif command == "stop": manager.stop() if __name__ == "__main__": cli() PK!gR R $reprobench/managers/slurm/manager.pyimport math import subprocess import sys from pathlib import Path from loguru import logger from sshtunnel import SSHTunnelForwarder from reprobench.managers.base import BaseManager from reprobench.utils import read_config from .utils import to_comma_range class SlurmManager(BaseManager): def prepare(self): Path(self.output_dir).mkdir(parents=True, exist_ok=True) limits = self.config["limits"] time_limit_minutes = int(math.ceil(limits["time"] / 60.0)) self.cpu_count = limits.get("cores", 1) # @TODO improve this self.time_limit = 2 * time_limit_minutes self.mem_limit = 2 * limits["memory"] if self.tunneling is not None: self.server = SSHTunnelForwarder( self.tunneling["host"], remote_bind_address=("127.0.0.1", self.tunneling["port"]), ssh_pkey=self.tunneling["key_file"], ssh_config_file=self.tunneling["ssh_config_file"], ) # https://github.com/pahaz/sshtunnel/issues/138 if sys.version_info[0] > 3 or ( sys.version_info[0] == 3 and sys.version_info[1] >= 7 ): self.server.daemon_forward_servers = True self.server.start() self.server_address = f"tcp://127.0.0.1:{self.server.local_bind_port}" logger.info(f"Tunneling established at {self.server_address}") def stop(self): subprocess.run(["scancel", f"--name={self.config['title']}-benchmark-worker"]) def spawn_workers(self): logger.info("Spawning workers...") address_args = f"--address={self.server_address}" if self.tunneling is not None: address_args = f"-h {self.tunneling['host']} -p {self.tunneling['port']} -K {self.tunneling['key_file']}" worker_cmd = f"{sys.exec_prefix}/bin/reprobench worker {address_args} -vv" worker_submit_cmd = [ "sbatch", "--parsable", f"--array=1-{self.pending}", f"--time={self.time_limit}", f"--mem={self.mem_limit}", f"--cpus-per-task={self.cpu_count}", f"--job-name={self.config['title']}-benchmark-worker", f"--output={self.output_dir}/slurm-worker_%a.out", "--wrap", f"srun {worker_cmd}", ] logger.trace(worker_submit_cmd) self.worker_job = subprocess.check_output(worker_submit_cmd).decode().strip() logger.info(f"Worker job array id: {self.worker_job}") def wait(self): if self.tunneling is not None: self.server.stop() PK!m"reprobench/managers/slurm/utils.pyimport subprocess from itertools import groupby from operator import itemgetter def consecutive_groups(it): for _, g in groupby(enumerate(it), lambda tup: tup[0] - tup[1]): yield tuple(map(itemgetter(1), g)) def to_comma_range(it): return ",".join( "-".join(map(str, (g[0], g[-1])[: len(g)])) for g in consecutive_groups(it) ) def get_nodelist(job_step): """ Blocks until job step is assigned a node """ while True: cmd = ["sacct", "-n", "--parsable2", "-j", job_step, "-o", "NodeList"] output = subprocess.check_output(cmd) if len(output) > 0 and output != b"None assigned\n": return output.decode().strip() PK!p'reprobench/statistics/plots/__init__.pyfrom .cactus import CactusPlot PK!=6#reprobench/statistics/plots/base.pyfrom pathlib import Path from reprobench.core.base import Step try: import papermill as pm except ImportError: pass class NotebookExecutor(Step): INPUT_NOTEBOOK = None DEFAULT_OUTPUT = None @classmethod def execute(cls, context, config=None): if config is None: config = {} output_dir = context.get("output_dir", None) output = Path(output_dir) / config.get("output", cls.DEFAULT_OUTPUT) output.parent.mkdir(parents=True, exist_ok=True) parameters = dict(db_path=context.get("db_path"), **config) pm.execute_notebook(cls.INPUT_NOTEBOOK, str(output), parameters=parameters) PK!8V.reprobench/statistics/plots/cactus/__init__.pyimport os from reprobench.statistics.plots.base import NotebookExecutor DIR = os.path.dirname(__file__) class CactusPlot(NotebookExecutor): DEFAULT_OUTPUT = "output/statistics/cactus.ipynb" INPUT_NOTEBOOK = os.path.join(DIR, "template.ipynb") PK! 1reprobench/statistics/plots/cactus/template.ipynb{ "cells": [ { "cell_type": "code", "source": [ "db_path = \"benchmark.db\"\n", "measure = \"cpu_time\"" ], "outputs": [], "execution_count": null, "metadata": { "collapsed": false, "outputHidden": false, "inputHidden": false, "tags": [ "parameters" ] } }, { "cell_type": "code", "source": [ "from reprobench.utils import init_db\n", "init_db(db_path)" ], "outputs": [], "execution_count": null, "metadata": { "collapsed": false, "outputHidden": false, "inputHidden": false } }, { "cell_type": "code", "source": [ "import itertools\n", "\n", "import pandas as pd\n", "import seaborn as sns\n", "\n", "from reprobench.core.db import Run, Tool, ParameterGroup\n", "from reprobench.executors.db import RunStatistic\n", "\n\n", "def cactus_plot(measure, **kwargs):\n", " cactus_df = pd.DataFrame()\n", " \n", " for group in ParameterGroup.select():\n", " tool_name = f\"{group.tool_id}_{group.name}\"\n", " measure_field = getattr(RunStatistic, measure)\n", " values_query = (\n", " RunStatistic\n", " .select(measure_field)\n", " .join(Run)\n", " .where(Run.tool_id == group.tool_id)\n", " .where(Run.parameter_group_id == group.id)\n", " .order_by(measure_field)\n", " )\n", " series = pd.Series(\n", " data=[*itertools.chain.from_iterable(values_query.tuples())],\n", " name=tool_name,\n", " ).sort_values()\n", " cactus_df = cactus_df.append(series, sort=False)\n", " \n", " cactus_df = cactus_df.transpose().reset_index(drop=True)\n", " \n", " return sns.scatterplot(data=cactus_df, **kwargs)" ], "outputs": [], "execution_count": null, "metadata": { "collapsed": false, "outputHidden": false, "inputHidden": false } }, { "cell_type": "code", "source": [ "import matplotlib.pyplot as plt\n", "import matplotlib.ticker as ticker\n", "\n", "fig, ax = plt.subplots(figsize=(8, 6))\n", "plt.xticks()\n", "plt.xlabel(\"Instance solved\")\n", "plt.ylabel(\"Time (s)\")\n", "cactus_plot(measure, ax=ax)\n", "plt.show()" ], "outputs": [], "execution_count": null, "metadata": { "collapsed": false, "outputHidden": false, "inputHidden": false } } ], "metadata": { "kernel_info": { "name": "python3" }, "language_info": { "name": "python", "version": "3.7.2", "mimetype": "text/x-python", "codemirror_mode": { "name": "ipython", "version": 3 }, "pygments_lexer": "ipython3", "nbconvert_exporter": "python", "file_extension": ".py" }, "kernelspec": { "name": "python3", "language": "python", "display_name": "Python 3" }, "nteract": { "version": "0.12.3" } }, "nbformat": 4, "nbformat_minor": 4 }PK!&++(reprobench/statistics/tables/__init__.pyfrom .run import RunTable, RunSummaryTable PK!Qk[aa$reprobench/statistics/tables/base.pyfrom pathlib import Path from reprobench.core.base import Step class PandasExporter(Step): @classmethod def get_dataframe(cls, config): raise NotImplementedError @classmethod def save_df(cls, df, output): if output.endswith(".csv"): df.to_csv(output) elif output.endswith(".json"): df.to_json(output) else: raise NotImplementedError @classmethod def execute(cls, context, config=None): if config is None: config = {} output_dir = context.get("output_dir", None) output = Path(output_dir) / config.pop("output") output.parent.mkdir(parents=True, exist_ok=True) df = cls.get_dataframe(config) # remove duplicated columns df = df.loc[:, ~df.columns.duplicated()] cls.save_df(df, str(output)) PK!r#reprobench/statistics/tables/run.pyfrom reprobench.core.db import ParameterGroup, Run, db from reprobench.executors.db import RunStatistic from reprobench.utils import import_class from .base import PandasExporter try: import pandas as pd except ImportError: pass class RunTable(PandasExporter): @classmethod def get_dataframe(cls, config): joins = config.get("joins", []) query = Run.select() for model_class in joins: model = import_class(model_class) query = query.join_from(Run, model).select_extend( *model._meta.fields.values() ) sql, params = query.sql() return pd.read_sql_query(sql, db, params=params) class RunSummaryTable(PandasExporter): DEFAULT_COLUMNS = ("cpu_time", "wall_time", "max_memory") @classmethod def get_dataframe(cls, config): columns = config.get("columns", cls.DEFAULT_COLUMNS) tool_names = [ f"{group.tool_id}_{group.name}" for group in ParameterGroup.select() ] multiindex = pd.MultiIndex.from_product((tool_names, columns)) df = pd.DataFrame(index=multiindex).transpose() for group in ParameterGroup.select(): tool_name = f"{group.tool_id}_{group.name}" query = ( RunStatistic.select() .join(Run) .where(Run.tool_id == group.tool_id) .where(Run.parameter_group_id == group.id) ) sql, params = query.sql() tool_df = pd.read_sql(sql, db, params=params) for col in columns: df.loc(axis=1)[tool_name, col] = tool_df[col] return df.describe() PK!?reprobench/task_sources/base.pyclass BaseTaskSource(object): TYPE = None def __init__(self, path=None, **kwargs): self.path = path def setup(self): return [] PK!6BB'reprobench/task_sources/doi/__init__.pyfrom reprobench.task_sources.url import UrlSource from reprobench.task_sources.doi.zenodo import ZenodoHandler, ZenodoSandboxHandler class DOISource(UrlSource): TYPE = "doi" handlers = [ZenodoHandler, ZenodoSandboxHandler] def __init__(self, doi, **kwargs): super().__init__(**kwargs) self.doi = doi for handler in self.handlers: if handler.is_compatible(self.doi): self.urls = handler.get_urls(self.doi) break else: raise NotImplementedError(f"No handler for doi: {doi}") PK!RO#reprobench/task_sources/doi/base.pyclass BaseDOIHandler(object): @classmethod def is_compatible(cls, doi): return False @classmethod def get_urls(cls, doi): return [] PK!Sʺ%reprobench/task_sources/doi/zenodo.pyimport requests from reprobench.task_sources.doi.base import BaseDOIHandler class ZenodoHandler(BaseDOIHandler): doi_prefix = "10.5281/zenodo." api_url = "https://zenodo.org/api" @classmethod def is_compatible(cls, doi): return doi.startswith(cls.doi_prefix) @classmethod def get_urls(cls, doi): record_id = doi[len(cls.doi_prefix) :] # remove doi_prefix url = "{}/records/{}".format(cls.api_url, record_id) record = requests.get(url).json() return [file["links"]["self"] for file in record["files"]] class ZenodoSandboxHandler(ZenodoHandler): doi_prefix = "10.5072/zenodo." api_url = "https://sandbox.zenodo.org/api" PK!#9 reprobench/task_sources/file.pyfrom pathspec import PathSpec from pathlib import Path from .base import BaseTaskSource class FileSource(BaseTaskSource): TYPE = "file" def __init__(self, path=None, patterns="", **kwargs): super().__init__(path) self.patterns = patterns def setup(self): spec = PathSpec.from_lines("gitwildmatch", self.patterns.splitlines()) matches = spec.match_tree(self.path) return map(lambda match: Path(self.path).resolve() / match, matches) PK!-ffreprobench/task_sources/url.pyfrom pathlib import Path from loguru import logger from reprobench.utils import download_file, extract_archives from .file import FileSource class UrlSource(FileSource): TYPE = "url" def __init__( self, urls=None, path=None, patterns="", skip_existing=True, extract_archives=True, **kwargs, ): super().__init__(path, patterns=patterns) self.urls = urls or [] self.extract_archives = extract_archives self.skip_existing = skip_existing def setup(self): root = Path(self.path) root.mkdir(parents=True, exist_ok=True) for url in self.urls: filename = url.split("/")[-1].split("?")[0] path = root / filename if not path.exists() or not self.skip_existing: logger.debug(f"Downloading {url} to {path}") download_file(url, path) else: logger.debug(f"Skipping already downloaded file {path}") if self.extract_archives: extract_archives(path) return super().setup() PK!P!*reprobench/tools/executable.pyfrom pathlib import Path from loguru import logger from reprobench.core.base import Tool class ExecutableTool(Tool): name = "Basic Executable Tool" path = None prefix = "--" @classmethod def is_ready(cls): return True def get_arguments(self): return [f"{self.prefix}{key}={value}" for key, value in self.parameters.items()] def get_cmdline(self): return [self.path, *self.get_arguments(), self.task] def get_out_path(self): return Path(self.cwd) / "run.out" def get_err_path(self): return Path(self.cwd) / "run.err" def get_output(self): return self.get_out_path().read_bytes() def get_error(self): return self.get_err_path().read_bytes() def run(self, executor): logger.debug([*self.get_cmdline(), self.task]) executor.run( self.get_cmdline(), directory=self.cwd, out_path=self.get_out_path(), err_path=self.get_err_path(), ) PK!}J((reprobench/utils.py"""Various utilities""" import importlib import re import tarfile import zipfile from ast import literal_eval from collections.abc import Iterable from pathlib import Path from shutil import which import numpy import requests import strictyaml from reprobench.core.exceptions import ExecutableNotFoundError, NotSupportedError from reprobench.core.schema import schema from retrying import retry from tqdm import tqdm try: import msgpack from playhouse.apsw_ext import APSWDatabase from reprobench.core.db import db except ImportError: APSWDatabase = None db = None def find_executable(executable): """Find an executable path from its name Similar to `/usr/bin/which`, this function find the path of an executable by its name, for example by finding it in the PATH environment variable. Args: executable (str): The executable name Returns: str: Path of the executable Raises: ExecutableNotFoundError: If no path for `executable` is found. """ path = which(executable) if path is None: raise ExecutableNotFoundError return path def import_class(path): """Import a class by its path Args: path (str): the path to the class, in similar notation as modules Returns: class: the specified class Examples: >>> import_class("reprobench.core.server.BenchmarkServer") """ module_path, tail = ".".join(path.split(".")[:-1]), path.split(".")[-1] module = importlib.import_module(module_path) return getattr(module, tail) def _copy_file_obj(source, destination, callback, length=16 * 1024): """Modified version of shutil.copyfileobj with callback""" while True: buf = source.read(length) if not buf: break destination.write(buf) callback(len(buf)) def download_file(url, dest): """Download a file by the specified URL Args: url (str): URL for the file to download dest (str): Destination path for saving the file """ r = requests.get(url, stream=True) with tqdm( total=int(r.headers.get("content-length", 0)), unit="B", unit_scale=True, unit_divisor=1024, ) as progress_bar: progress_bar.set_postfix(file=Path(dest).name, refresh=False) with open(dest, "wb") as f: _copy_file_obj(r.raw, f, progress_bar.update) ranged_numbers_re = re.compile(r"(?P\d+)\.\.(?P\d+)(\.\.(?P\d+))?") def is_range_str(range_str): """Check if a string is in range notation Args: range_str (str): The string to check Returns: bool: if the string is in range notation Examples: >>> is_range_str("1..2") True >>> is_range_str("1..5..2") True >>> is_range_str("1") False """ return ranged_numbers_re.match(range_str) is not None def str_to_range(range_str): """Generate range from a string with range notation Args: range_str (str): The string with range notation Returns: range: The generated range Examples: >>> str_to_range("1..3") range(1, 4) >>> str_to_range("1..5..2") range(1, 6, 2) >>> [*str_to_range("1..3")] [1, 2, 3] """ matches = ranged_numbers_re.match(range_str).groupdict() start = int(matches["start"]) end = int(matches["end"]) + 1 if matches["step"]: return range(start, end, int(matches["step"])) return range(start, end) def encode_message(obj): """Encode an object for transport This method serialize the object with msgpack for network transportation. Args: obj: serializable object Returns: bin: binary string of the encoded object """ return msgpack.packb(obj, use_bin_type=True) def decode_message(msg): """Decode an encoded object This method deserialize the encoded object from `encode_message(obj)`. Args: bin: binary string of the encoded object Returns: obj: decoded object """ return msgpack.unpackb(msg, raw=False) @retry(wait_exponential_multiplier=500) def send_event(socket, event_type, payload=None): """Used in the worker with a DEALER socket to send events to the server. Args: socket (zmq.Socket): the socket for sending the event event_type (str): event type agreed between the parties payload (any, optional): the payload for the event """ event = [event_type, encode_message(payload)] socket.send_multipart(event) def recv_event(socket): """Receive published event for the observers Args: socket (zmq.Socket): SUB socket for receiving the event Returns: (event_type, payload, address): Tuple for received events """ event_type, payload, address = socket.recv_multipart() return event_type, decode_message(payload), address def get_db_path(output_dir): """Get the database path from the given output directory Args: output_dir (str): path to the output directory Returns: str: database path """ return str((Path(output_dir) / f"benchmark.db").resolve()) def init_db(db_path): """Initialize the given database Args: db_path (str): path to the database """ database = APSWDatabase(db_path, pragmas=(("journal_mode", "wal"),)) db.initialize(database) def resolve_files_uri(root): """Resolve all `file://` URIs in a dictionary to its content Args: root (dict): Root dictionary of the configuration Examples: >>> d = dict(test="file://./test.txt") >>> resolve_files_uri(d) >>> d {'a': 'this is the content of test.txt\\n'} """ protocol = "file://" iterator = None if isinstance(root, dict): iterator = root elif isinstance(root, list) or isinstance(root, tuple): iterator = range(len(root)) for k in iterator: if isinstance(root[k], str) and root[k].startswith(protocol): root[k] = Path(root[k][len(protocol) :]).read_text() elif isinstance(root[k], Iterable) and not isinstance(root[k], str): resolve_files_uri(root[k]) def read_config(config_path, resolve_files=False): """Read a YAML configuration from a path Args: config_path (str): Configuration file path (YAML) resolve_files (bool, optional): Should files be resolved to its content? Defaults to False. Returns: dict: Configuration """ with open(config_path, "r") as f: config_text = f.read() config = strictyaml.load(config_text, schema=schema).data if resolve_files: resolve_files_uri(config) return config def extract_zip(path, dest): """Extract a ZIP file Args: path (str): Path to ZIP file dest (str): Destination for extraction """ if not dest.is_dir(): with zipfile.ZipFile(path, "r") as f: f.extractall(dest) def extract_tar(path, dest): """Extract a TAR file Args: path (str): Path to TAR file dest (str): Destination for extraction """ if not dest.is_dir(): with tarfile.TarFile.open(path) as f: f.extractall(dest) def extract_archives(path): """Extract archives based on its extension Args: path (str): Path to the archive file """ extract_path = Path(path).with_name(path.stem) if zipfile.is_zipfile(path): extract_zip(path, extract_path) elif tarfile.is_tarfile(path): extract_tar(path, extract_path) def get_pcs_parameter_range(parameter_str, is_categorical): """Generate a range from specified pcs range notation Args: parameter_str (str): specified pcs parameter is_categorical (bool): is the range categorical Raises: NotSupportedError: If there is no function for resolving the range Returns: range: Generated range """ functions = dict( range=range, arange=numpy.arange, linspace=numpy.linspace, logspace=numpy.logspace, geomspace=numpy.geomspace, ) function_re = re.compile(r"(?P[A-Za-z_]+)\((?P.*)\)") match = function_re.match(parameter_str) parameter_range = None if match: function = match.group("function") if function not in functions: raise NotSupportedError(f"Declaring range with {function} is not supported") args = literal_eval(match.group("arguments")) parameter_range = functions[function](*args) else: parameter_range = literal_eval(parameter_str) if not isinstance(parameter_range, Iterable) or isinstance( parameter_range, str ): parameter_range = (parameter_range,) if is_categorical: parameter_range = map(str, parameter_range) return parameter_range def parse_pcs_parameters(lines): """Parse parameters from a pcs file content Args: lines ([str]): pcs file content Returns: dict: generated parameters """ parameter_range_indicator = "-->" parameters = {} parameter_key = None is_categorical = False for line in lines: if ("{" in line or "[" in line) and not line.startswith("#"): parameter_key = line[: line.find(" ")] is_categorical = "{" in line if "#" not in line or parameter_range_indicator not in line: continue comment_pos = line.find("#") pos = line.find(parameter_range_indicator, comment_pos) parameter_str = line[pos + len(parameter_range_indicator) :].strip() parameter_range = get_pcs_parameter_range(parameter_str, is_categorical) parameters[parameter_key] = parameter_range return parameters def check_valid_config_space(config_space, parameters): """Check if the parameters is valid based on a configuration space Args: config_space (ConfigSpace): configuration space parameters (dict): parameters dictionary Raises: ValueError: If there is invalid values """ base = config_space.get_default_configuration() for key, value in parameters.items(): if key in base: base[key] = value PK!HJ.:,reprobench-0.11.0.dist-info/entry_points.txtN+I/N.,()*J-(OJKΰE0r3s2PK!XB33#reprobench-0.11.0.dist-info/LICENSEMIT License Copyright (c) 2019 Rakha Kanz Kautsar Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PK!Hu)GTU!reprobench-0.11.0.dist-info/WHEEL HM K-*ϳR03rOK-J,/R(O-)$qzd&Y)r$UV&UrPK!HNC^?x $reprobench-0.11.0.dist-info/METADATAVr6}Wv'Ϙ.lVSΤ$'NLdž2_)ɲ 8<{`,eſuBnt$cϣnvݖR2[ȧIK.&jKVZA!o?~ af9#?w.bL(h6kd!@'x(TE?t]91dd4"oȹAsH/rcufBe=SYIߋ'"-V1Ŋ vx!@i'ڊ:x 5O_ QqexxvD;?/5T{)/Mv, "BȜaChz09SA-:?|)y x{o(I8Nd 'pL*ykY iwПv^ݣc>Ox0O~fpeUӥ0΄IQ (eEM6a"S㺸Z%sD3o+/'ēp!3rEЉyskgk 6[^M'=w>]m~ծWv~a խiyNV?W KY|[ߑ_l/ l%Bfx6ܫBs| `~ĦtaoY{G?zM@ jߵSV5&ejnqMCd=Ѭ䊒'VBxR>O[`"u9ly d;Qy([@(߰O=h$kfd,a;w؝:h Naq>y-IA*s̹_L: ɭ1)&aWΕy~[up[[22{v  H$TjU_ c T@OYcy*6ph3CYw%o;_M:2ٷ9LoU]=e=q:^y  q=KZiھclXZ SyFdBkۿ4ri?y fMۚ7X=&~IKHBJ\5/`H 8`N;\@;=n"# Hn9׎M GmJdg) ZtIu9Eя H ݆XE(bqYAtKbM@WqOO5)~L.CmpCnQT³e 3?PE:D,WtȆQ|5Bf8T"@~#I4q^8t[ă<:?:5v<ꊛ[f$QN)P@9r/!{ Ӣ4YO2q| 7>ZSf=Y0ODUj:ɀCw lU_=QS ;]Ƹ[Z>*b|KFِN) u+AEYՏn@v;>CBΧ6:h K9es=ob zٛ yLL >o- ȦPIQ hee=NDQ Od`ϩ N%HmP Fyk b؊yt#]%Y\m/T1ooY%MCYhC蓯{nH~,93l3:6^v8AiS+yzg׮xtѬ1M»X@ƖX!fR~y&)^%<-9U&} Ԋ'aHVߏUU=lojanngYYj&=4v]Z􏩺8_m!e$R`;ʙ%.>F|O߲=IlR ʬRҮQ*^>ɑWׂ9#K|3'pgWK@*G[-Esy끅Nbto9k -7,PK!6reprobench/__init__.pyPK!?6 Hreprobench/console/decorators.pyPK!#Freprobench/console/main.pyPK!28)22L reprobench/console/status.pyPK!Iفreprobench/core/analyzer.pyPK!,AArreprobench/core/base.pyPK! dd%reprobench/core/bootstrap/__init__.pyPK!|--#reprobench/core/bootstrap/client.pyPK!QBB#reprobench/core/bootstrap/server.pyPK!:reprobench/core/db.pyPK!|0_Creprobench/core/events.pyPK!a_hhDreprobench/core/exceptions.pyPK! 5Ereprobench/core/observers.pyPK!~~Preprobench/core/schema.pyPK!$JJIUreprobench/core/server.pyPK!; \reprobench/core/sysinfo.pyPK!cV V greprobench/core/worker.pyPK!s݊tt treprobench/executors/__init__.pyPK!(Xrtreprobench/executors/base.pyPK!%o-NNxreprobench/executors/db.pyPK!d,C,,=}reprobench/executors/events.pyPK![ I I }reprobench/executors/psmon.pyPK!G)reprobench/managers/__init__.pyPK!errareprobench/managers/base.pyPK!A?&&% reprobench/managers/local/__init__.pyPK!Z$ureprobench/managers/local/manager.pyPK!v]%Zreprobench/managers/slurm/__init__.pyPK!gR R $mreprobench/managers/slurm/manager.pyPK!m"reprobench/managers/slurm/utils.pyPK!p'reprobench/statistics/plots/__init__.pyPK!=6#\reprobench/statistics/plots/base.pyPK!8V.6reprobench/statistics/plots/cactus/__init__.pyPK! 1reprobench/statistics/plots/cactus/template.ipynbPK!&++(Preprobench/statistics/tables/__init__.pyPK!Qk[aa$reprobench/statistics/tables/base.pyPK!r#dreprobench/statistics/tables/run.pyPK!?;reprobench/task_sources/base.pyPK!6BB'reprobench/task_sources/doi/__init__.pyPK!RO#reprobench/task_sources/doi/base.pyPK!Sʺ%reprobench/task_sources/doi/zenodo.pyPK!#9 reprobench/task_sources/file.pyPK!-ffreprobench/task_sources/url.pyPK!P!*Freprobench/tools/executable.pyPK!}J((yreprobench/utils.pyPK!HJ.:,~reprobench-0.11.0.dist-info/entry_points.txtPK!XB33#reprobench-0.11.0.dist-info/LICENSEPK!Hu)GTU!j reprobench-0.11.0.dist-info/WHEELPK!HNC^?x $ reprobench-0.11.0.dist-info/METADATAPK!H"reprobench-0.11.0.dist-info/RECORDPK11