PK!benchpress/__init__.pyPK!$ >>benchpress/cli/__init__.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. PK!#benchpress/cli/commands/__init__.pyPK!1&&"benchpress/cli/commands/command.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. from abc import ABCMeta, abstractmethod class BenchpressCommand(object, metaclass=ABCMeta): @abstractmethod def populate_parser(self, parser): pass @abstractmethod def run(self, args, jobs): pass PK!dbenchpress/cli/commands/list.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. from benchpress.cli.commands.command import BenchpressCommand class ListCommand(BenchpressCommand): def populate_parser(self, subparsers): parser = subparsers.add_parser("list", help="list all configured jobs") parser.set_defaults(command=self) def run(self, args, jobs): for job in jobs.values(): print("{}: {}".format(job.name, job.description)) PK!씘benchpress/cli/commands/run.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. import logging from datetime import datetime, timezone from benchpress.cli.commands.command import BenchpressCommand from benchpress.lib.reporter_factory import ReporterFactory logger = logging.getLogger(__name__) class RunCommand(BenchpressCommand): def populate_parser(self, subparsers): parser = subparsers.add_parser("run", help="run job(s)") parser.set_defaults(command=self) parser.add_argument("jobs", nargs="*", default=[], help="jobs to run") def run(self, args, jobs): reporter = ReporterFactory.create("default") if len(args.jobs) > 0: for name in args.jobs: if name not in jobs: logger.error('No job "{}" found'.format(name)) exit(1) jobs = {name: jobs[name] for name in args.jobs} jobs = jobs.values() print("Will run {} job(s)".format(len(jobs))) for job in jobs: print('Running "{}": {}'.format(job.name, job.description)) metrics = job.run() reporter.report(job, metrics) reporter.close() PK!(B B benchpress/cli/main.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. import argparse import logging import sys import yaml from benchpress.cli.commands.list import ListCommand from benchpress.cli.commands.run import RunCommand from benchpress.lib.job import Job from benchpress.lib.reporter import StdoutReporter from benchpress.lib.reporter_factory import ReporterFactory def setup_parser(): """Setup the commands and command line parser. Returns: setup parser (argparse.ArgumentParser) """ commands = [ListCommand(), RunCommand()] parser = argparse.ArgumentParser() parser.add_argument( "-b", "--benchmarks", default="benchmarks.yml", metavar="benchmarks file", help="path to benchmarks file", ) parser.add_argument( "-j", "--jobs", default="jobs/jobs.yml", dest="jobs_file", metavar="job configs file", help="path to job configs file", ) subparsers = parser.add_subparsers(dest="command", help="subcommand to run") for command in commands: command.populate_parser(subparsers) subparsers.required = True parser.add_argument("--verbose", "-v", action="count", default=0) return parser # ignore sys.argv[0] because that is the name of the program def main(args=sys.argv[1:]): # register reporter plugins before setting up the parser ReporterFactory.register("stdout", StdoutReporter) parser = setup_parser() args = parser.parse_args(args) # warn is 30, should default to 30 when verbose=0 # each level below warning is 10 less than the previous log_level = args.verbose * (-10) + 30 logging.basicConfig(format="%(levelname)s:%(name)s: %(message)s", level=log_level) logger = logging.getLogger(__name__) logger.info('Loading benchmarks from "{}"'.format(args.benchmarks)) with open(args.benchmarks) as tests_file: benchmarks = yaml.load(tests_file) logger.info('Loading jobs from "{}"'.format(args.jobs_file)) with open(args.jobs_file) as jobs_file: job_configs = yaml.load(jobs_file) jobs = [Job(j, benchmarks[j["benchmark"]]) for j in job_configs if "tests" not in j] jobs = {j.name: j for j in jobs} logger.info("Loaded {} benchmarks and {} jobs".format(len(benchmarks), len(jobs))) args.command.run(args, jobs) PK!benchpress/lib/__init__.pyPK!M4benchpress/lib/factory.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. class BaseFactory(object): """Factory to construct instances of classes based on name. Attributes: base_class (class): base class that registered classes must subclass """ def __init__(self, base_class): """Create a BaseFactory with base_class as the supertype.""" self.base_class = base_class self.classes = {} @property def registered_names(self): """list of str: class names registered with the factory.""" return list(self.classes.keys()) def create(self, name): """Find the subclass with the correct name and instantiates it. Args: name (str): name of the item """ if name not in self.classes: raise KeyError( 'No type "{}". ' "Did you forget to register() it?".format(name) ) return self.classes[name]() def register(self, name, subclass): """Registers a class with the factory. Args: name (str): name of the class subclass (class): concrete subclass of base_class """ assert issubclass(subclass, self.base_class) self.classes[name] = subclass PK!.ccbenchpress/lib/hook.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. from abc import ABCMeta, abstractmethod class Hook(object, metaclass=ABCMeta): """Hook allows jobs to run some Python code before/after a job runs.""" @abstractmethod def before_job(self, opts, job): """Do something to setup before this job. Args: opts (dict): user-defined options for this hook """ @abstractmethod def after_job(self, opts, job): """Do something to teardown after this job. Args: opts (dict): user-defined options for this hook """ PK!"P44benchpress/lib/hook_factory.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. from benchpress.lib.factory import BaseFactory from benchpress.lib.hook import Hook from benchpress.plugins.hooks import register_hooks HookFactory = BaseFactory(Hook) # register third-party hooks with the factory register_hooks(HookFactory) PK!rrbenchpress/lib/job.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. import errno import logging import subprocess import sys from subprocess import CalledProcessError, TimeoutExpired from benchpress.lib.hook_factory import HookFactory from benchpress.lib.parser_factory import ParserFactory logger = logging.getLogger(__name__) class Job(object): """Holds the run configuration for an individual job. A Job starts it's default config based on the benchmark configuration that it references. The binary defined in the benchmark is run according to the configuration of this job. Attributes: name (str): short name to identify job description (str): longer description to state intent of job tolerances (dict): percentage tolerance around the mean of historical results config (dict): raw configuration dictionary """ def __init__(self, job_config, benchmark_config): """Create a Job with the default benchmark_config and the specific job config Args: config (dict): job config benchmark_config (dict): benchmark (aka default) config """ # start with the config being the benchmark config and then update it # with the job config so that a job can override any options in the # benchmark config config = benchmark_config # TODO(vmagro) should there be some basic sanity check that a job_config # contains certain fields? config.update(job_config) self.config = config self.name = config["name"] self.description = config["description"] self.binary = config["path"] self.parser = ParserFactory.create(config["parser"]) self.check_returncode = config.get("check_returncode", True) self.timeout = config.get("timeout", None) self.timeout_is_pass = config.get("timeout_is_pass", False) # if tee_output is True, the stdout and stderr commands of the child # process will be copied onto the stdout and stderr of benchpress # if this option is a string, the output will be written to the file # named by this value self.tee_output = config.get("tee_output", False) self.hooks = config.get("hooks", []) self.hooks = [ (HookFactory.create(h["hook"]), h.get("options", None)) for h in self.hooks ] # self.hooks is list of (hook, options) self.tolerances = config.get("tolerances", {}) self.args = self.arg_list(config["args"]) @staticmethod def arg_list(args): """Convert argument definitions to a list suitable for subprocess. """ if isinstance(args, list): return args l = [] for key, val in args.items(): l.append("--" + key) if val is not None: l.append(str(val)) return l def run(self): """Run the benchmark and return the metrics that are reported. """ # take care of preprocessing setup via hook logger.info('Running setup hooks for "{}"'.format(self.name)) for hook, opts in self.hooks: logger.info("Running %s %s", hook, opts) hook.before_job(opts, self) try: logger.info('Starting "{}"'.format(self.name)) cmd = [self.binary] + self.args try: process = subprocess.run( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, timeout=self.timeout, encoding="utf-8", ) except TimeoutExpired as e: stdout, stderr = e.stdout, e.stderr if not self.timeout_is_pass: logger.error( "Job timed out\n" "stdout:\n{}\nstderr:\n{}".format(stdout, stderr) ) raise returncode = 0 else: stdout, stderr = process.stdout, process.stderr returncode = process.returncode if self.check_returncode and returncode != 0: output = "stdout:\n{}\nstderr:\n{}".format(stdout, stderr) cmd = " ".join(cmd) raise CalledProcessError(process.returncode, cmd, output) # optionally copy stdout/err of the child process to our own if self.tee_output: # default to stdout if no filename given tee = sys.stdout # if a file was specified, write to that file instead if isinstance(self.tee_output, str): tee = open(self.tee_output, "w") # do this so each line is prefixed with stdout for line in stdout.splitlines(): tee.write(f"stdout: {line}\n") for line in stderr.splitlines(): tee.write(f"stderr: {line}\n") # close the output if it was a file if tee != sys.stdout: tee.close() logger.info('Parsing results for "{}"'.format(self.name)) try: return self.parser.parse( stdout.splitlines(), stderr.splitlines(), returncode ) except Exception: logger.error( "Failed to parse results, this might mean the" " benchmark failed" ) logger.error("stdout:\n{}".format(stdout)) logger.error("stderr:\n{}".format(stderr)) raise except OSError as e: logger.error('"{}" failed ({})'.format(self.name, e)) if e.errno == errno.ENOENT: logger.error("Binary not found, did you forget to install it?") raise # make sure it passes the exception up the chain except CalledProcessError as e: logger.error(e.output) raise # make sure it passes the exception up the chain finally: logger.info('Running cleanup hooks for "{}"'.format(self.name)) # run hooks in reverse this time so it operates like a stack for hook, opts in reversed(self.hooks): hook.after_job(opts, self) @property def safe_name(self): return self.name.replace(" ", "_") PK!گ[[benchpress/lib/parser.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. from abc import ABCMeta, abstractmethod class Parser(object, metaclass=ABCMeta): """Parser is the link between benchmark output and the rest of the system. A Parser is given the benchmark's stdout and stderr and returns the exported metrics. """ @abstractmethod def parse(self, stdout, stderr, returncode): """Take stdout/stderr and convert it to a dictionary of metrics. Args: stdout (list of str): stdout of benchmark process split on newline stderr (list of str): stderr of benchmark process split on newline returncode (int): subprocess return code Returns: (dict): metrics mapping name -> value - keys can be nested or flat with dot-separated names """ pass PK!uFF benchpress/lib/parser_factory.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. from benchpress.lib.factory import BaseFactory from benchpress.lib.parser import Parser from benchpress.plugins.parsers import register_parsers ParserFactory = BaseFactory(Parser) # register third-party parsers with the factory register_parsers(ParserFactory) PK!J66benchpress/lib/reporter.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. import json import sys from abc import ABCMeta, abstractmethod class Reporter(object, metaclass=ABCMeta): """A Reporter is used to record job results in your infrastructure. """ @abstractmethod def report(self, job, metrics): """Save job metrics somewhere in existing monitoring infrastructure. Args: job (Job): job that was run metrics (dict): metrics that were exported by job """ pass @abstractmethod def close(self): """Do whatever necessary cleanup is required after all jobs are finished. """ pass class StdoutReporter(Reporter): """Default reporter implementation, logs a JSON object to stdout.""" def report(self, job, metrics): """Log JSON report to stdout. Attempt to detect whether a real person is running the program then pretty print the JSON, otherwise print it without linebreaks and unsorted keys. """ # use isatty as a proxy for if a real human is running this if sys.stdout.isatty(): json.dump(metrics, sys.stdout, sort_keys=True, indent=2) else: json.dump(metrics, sys.stdout) sys.stdout.write("\n") def close(self): pass PK!"benchpress/lib/reporter_factory.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. from benchpress.lib.factory import BaseFactory from benchpress.lib.reporter import Reporter ReporterFactory = BaseFactory(Reporter) PK!-benchpress/lib/util.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. import sys def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) PK!benchpress/plugins/__init__.pyPK!1";zz$benchpress/plugins/hooks/__init__.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. from benchpress.plugins.hooks.cpu_limit import CpuLimit from benchpress.plugins.hooks.file import FileHook from benchpress.plugins.hooks.shell import ShellHook def register_hooks(factory): factory.register("cpu-limit", CpuLimit) factory.register("file", FileHook) factory.register("shell", ShellHook) PK!t  %benchpress/plugins/hooks/cpu_limit.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. import logging from benchpress.lib.hook import Hook logger = logging.getLogger(__name__) class CpuLimit(Hook): """CpuLimit hook allows you to limit the benchmark to a set of CPUs using `taskset`. The only option is a hex string bitmask that is the CPU mask passed to `taskset`, for each bit, if there is a 1 the CPU is enabled for the benchmark process, otherwise it's disabled. """ def before_job(self, opts, job): mask = str(opts) # try to parse the mask as a hex string as a basic sanity check try: int(mask, 16) except ValueError: raise ValueError("{} is not a valid CPU mask".format(mask)) # modify the job config to run taskset with the given mask instead of # directly running the benchmark binary binary = job.config["path"] job.args = [mask, binary] + job.args job.binary = "taskset" def after_job(self, opts, job): pass PK!& benchpress/plugins/hooks/file.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. import errno import logging import os import shutil from benchpress.lib.hook import Hook logger = logging.getLogger(__name__) class FileHook(Hook): """FileHook provides the ability to create and delete files/directories. Options are specified as a list of dictionaries - each dictionary must have a 'type' and a 'path', 'type' is either 'dir' or 'file', and path is where it will live on the filesystem. Files/directories are created before the job runs and destroyed after. """ def before_job(self, opts, job): for opt in opts: path = opt["path"] logger.info('Creating "{}"'.format(path)) if opt["type"] == "dir": try: os.makedirs(path) except OSError as e: if e.errno == errno.EEXIST: logger.warning( '"{}" already exists, proceeding anyway'.format(path) ) else: # other errors should be fatal raise if opt["type"] == "file": os.mknod(path) def after_job(self, opts, job): for opt in opts: path = opt["path"] logger.info('Deleting "{}"'.format(path)) if opt["type"] == "dir": shutil.rmtree(path) if opt["type"] == "file": os.unlink(path) PK!!O{{!benchpress/plugins/hooks/shell.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. import logging import os import shlex import subprocess from benchpress.lib.hook import Hook logger = logging.getLogger(__name__) class ShellHook(Hook): """ShellHook provides the ability to run arbitrary shell commands before/after a job Options are a dictioanry of 'before' and 'after' lists with a string for each command to run. Commands are not run in a shell, so 'cd's are converted to os.chdir A 'cd' can be executed and will change the working directory of the running test binary, and is reverted to the previous working directory during the post hook. """ def __init__(self): self.original_dir = os.getcwd() @staticmethod def run_commands(cmds): for cmd in cmds: # running with shell=True means we should give command as a string # and not pre-process it split = shlex.split(cmd) if split[0] == "cd": assert len(split) == 2 dst = split[1] logger.info('Switching to dir "%s"', dst) os.chdir(dst) else: logger.info('Running "%s"', cmd) subprocess.check_call( cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) def before_job(self, opts, job=None): self.original_dir = os.getcwd() if "before" in opts: self.run_commands(opts["before"]) def after_job(self, opts, job=None): if "after" in opts: self.run_commands(opts["after"]) # cd back to the original dir in case a command changed it if os.getcwd() != self.original_dir: logger.info('Returning to "%s"', self.original_dir) os.chdir(self.original_dir) PK!X&benchpress/plugins/parsers/__init__.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. from benchpress.plugins.parsers.fio import FioParser from benchpress.plugins.parsers.generic import JSONParser from benchpress.plugins.parsers.ltp import LtpParser from benchpress.plugins.parsers.packetdrill_parser import PacketdrillParser from benchpress.plugins.parsers.returncode import ReturncodeParser from benchpress.plugins.parsers.schbench import SchbenchParser from benchpress.plugins.parsers.silo import SiloParser from benchpress.plugins.parsers.xfstests_parser import XfstestsParser def register_parsers(factory): factory.register("fio", FioParser) factory.register("json", JSONParser) factory.register("ltp", LtpParser) factory.register("returncode", ReturncodeParser) factory.register("schbench", SchbenchParser) factory.register("silo", SiloParser) factory.register("packetdrill", PacketdrillParser) factory.register("xfstests", XfstestsParser) PK!ENfȶ!benchpress/plugins/parsers/fio.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. import json from benchpress.lib.parser import Parser class FioParser(Parser): def parse(self, stdout, stderr, returncode): metrics = {} stdout = "".join(stdout) results = json.loads(stdout) results = results["jobs"] for job in results: name = job["jobname"] metrics[name] = job return metrics PK!y>%benchpress/plugins/parsers/generic.py#!/usr/bin/env python3 # Copyright (c) 2018-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. import json import logging import re from benchpress.lib.parser import Parser JSON_LIKE_REGEX = r"\s*([{\[].*?[}\]]\s*[}\]]*)\s*" JSON_LIKE_MATCHER = re.compile(JSON_LIKE_REGEX) class JSONParser(Parser): def parse(self, stdout, stderr, returncode): """Converts JSON output from either stdout or stderr into a dict. Assumes that either stdout or stderr contains a section of valid JSON, as expected by the `json` module. Returns only first match of JSON. It will try to scan for JSON-like string sections, REGEX is too simple could miss some contrived cases. Args: stdout (list[str]): Process's line-by-line stdout output. stderr (list[str]): Process's line-by-line stderr output. returncode (int): Process's exit status code. Returns: metrics (dict): Representation of either stdout or stderr. Raises: ValueError: When neither stdout nor stderr could be parsed as JSON. """ err_msg = "Failed to parse {1} as JSON: {0}" for (output, kind) in [(stdout, "stdout"), (stderr, "stderr")]: process_output = " ".join(output) possible_json_matches = JSON_LIKE_MATCHER.findall(process_output) for m in possible_json_matches: try: return json.loads(m) except ValueError: pass else: logging.warning(err_msg.format(ValueError(), kind)) msg = "Couldn't not find or parse JSON from either stdout or stderr" raise ValueError(msg) PK!pN>!benchpress/plugins/parsers/ltp.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. import re from benchpress.lib.parser import Parser # : test_format_re = re.compile("\\w+\\s+\\d+\\s+T(FAIL|PASS|BROK|WARN|INFO).*") class LtpParser(Parser): def parse(self, stdout, stderr, returncode): # ltp run in quiet mode produces lines that are mostly a single line per # test with the test name and a status and optional message metrics = {} for line in stdout: # make sure that the line matches the format of a test if not test_format_re.match(line): continue line = line.split() # combine 0 and 1 because sometimes the first name string isn't # unique but the following number is name = line[0] + "_" + line[1] status = line[2] # test failure conditions if status in ("TFAIL", "TBROK", "TWARN"): status = False elif status == "TPASS": status = True else: # if status is not one of these, just skip it continue # pragma: no cover metrics[name] = status return metrics PK!u0benchpress/plugins/parsers/packetdrill_parser.py#!/usr/bin/env python3 from benchpress.lib.parser import Parser class TestStatus(object): PASSED = 1 FAILED = 2 class PacketdrillParser(Parser): """ Packetdrill test output is very simple (for now). One row for each test: test_name return_value So the parsing is simple: "test_name 0" => "test_name PASS" "test_name none-zero" => "test_name FAIL" """ def __init__(self): super().__init__() def parse(self, stdout, stderr, returncode): metrics = {} for line in stdout: items = line.split() if len(items) != 2: continue test_name = items[0] test_metrics = {} if items[1] == "0": test_metrics["status"] = TestStatus.PASSED else: test_metrics["status"] = TestStatus.FAILED metrics[test_name] = test_metrics return metrics PK!'< nn(benchpress/plugins/parsers/returncode.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. from benchpress.lib.parser import Parser class ReturncodeParser(Parser): """Returncode parser outputs one metric 'success' that is True if job binary had a 0 exit code, and False all other times.""" def parse(self, stdout, stderr, returncode): return {"success": returncode == 0} PK!t8DD&benchpress/plugins/parsers/schbench.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. from benchpress.lib.parser import Parser class SchbenchParser(Parser): def parse(self, stdout, stderr, returncode): stdout = stderr # schbench writes it output on stderr metrics = {"latency": {}} latency_percs = ["p50", "p75", "p90", "p95", "p99", "p99_5", "p99_9"] # this is gross - there should be some error handling eventually for key, line in zip(latency_percs, stdout[1:]): metrics["latency"][key] = float(line.split(":")[-1]) return metrics PK!"benchpress/plugins/parsers/silo.py#!/usr/bin/env python3 # Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. import re from benchpress.lib.parser import Parser AGG_TPUT_REGEX = r"(agg_throughput):\s+(\d+\.?\d*e?[+-]?\d*)\s+([a-z/]+)" PER_CORE_TPUT_REGEX = r"(avg_per_core_throughput):\s+(\d+\.?\d*e?[+-]?\d*)\s+([a-z/]+)" LAT_REGEX = r"(avg_latency):\s+(\d+\.?\d*e?[+-]?\d*)\s+([a-z]+)" class SiloParser(Parser): def parse(self, stdout, stderr, returncode): output = "".join(stderr) # Results output in stderr metrics = {"throughput": {}, "latency": {}} tput_metrics = [ re.findall(AGG_TPUT_REGEX, output)[0], re.findall(PER_CORE_TPUT_REGEX, output)[0], ] lat_metrics = [re.findall(LAT_REGEX, output)[0]] for tput_metric in tput_metrics: metrics["throughput"][tput_metric[0]] = float(tput_metric[1]) for lat_metric in lat_metrics: metrics["latency"][lat_metric[0]] = float(lat_metric[1]) return metrics PK!Ϩ(-benchpress/plugins/parsers/xfstests_parser.py#!/usr/bin/env python3 import difflib import os.path import re from benchpress.lib.parser import Parser class TestStatus(object): PASSED = 1 FAILED = 2 class XfstestsParser(Parser): def __init__(self): super().__init__() self.tests_dir = "xfstests/tests" self.results_dir = "xfstests/results" def parse(self, stdout, stderr, returncode): excluded = {} # The exclude list is one test per line optionally followed by a # comment explaining why the test is excluded. exclude_list_re = re.compile( r"\s*(?P[^\s#]+)\s*(?:#\s*(?P.*))?\s*" ) try: with open("exclude_list", "r", errors="backslashreplace") as f: for line in f: match = exclude_list_re.match(line) if match: reason = match.group("reason") if reason is None: reason = "" excluded[match.group("test_name")] = reason except OSError: pass test_regex = re.compile( r"^(?P\w+/\d+)\s+(?:\d+s\s+\.\.\.\s+)?(?P.*)" ) metrics = {} for line in stdout: match = test_regex.match(line) if match: test_name = match.group("test_name") test_metrics = {} status = match.group("status") duration_match = re.fullmatch("(\d+(?:\.\d+)?)s", status) if duration_match: test_metrics["status"] = TestStatus.PASSED test_metrics["duration_secs"] = float(duration_match.group(1)) elif status.startswith("[not run]"): test_metrics["status"] = TestStatus.PASSED test_metrics["details"] = self.not_run_details(test_name) elif status.startswith("[expunged]"): test_metrics["status"] = TestStatus.PASSED test_metrics["details"] = self.excluded_details(excluded, test_name) else: test_metrics["status"] = TestStatus.FAILED test_metrics["details"] = self.run_details(test_name) metrics[test_name] = test_metrics return metrics def not_run_details(self, test_name): try: notrun = os.path.join(self.results_dir, test_name + ".notrun") with open(notrun, "r", errors="backslashreplace") as f: return "Not run: " + f.read().strip() except OSError: return "Not run" @staticmethod def excluded_details(excluded, test_name): try: return "Excluded: " + excluded[test_name] except KeyError: return "Excluded" def run_details(self, test_name): details = [] self.append_diff(test_name, details) self.append_full_output(test_name, details) self.append_dmesg(test_name, details) return "".join(details) def append_diff(self, test_name, details): try: out_path = os.path.join(self.tests_dir, test_name + ".out") with open(out_path, "r", errors="backslashreplace") as f: out = f.readlines() out_bad_path = os.path.join(self.results_dir, test_name + ".out.bad") with open(out_bad_path, "r", errors="backslashreplace") as f: out_bad = f.readlines() except OSError: return diff = difflib.unified_diff(out, out_bad, out_path, out_bad_path) details.extend(diff) def append_full_output(self, test_name, details): full_path = os.path.join(self.results_dir, test_name + ".full") try: # There are some absurdly large full results. if os.path.getsize(full_path) < 100_000: with open(full_path, "r", errors="backslashreplace") as f: if details: details.append("--\n") details.append(f"{full_path}:\n") details.append(f.read()) except OSError: pass def append_dmesg(self, test_name, details): dmesg_path = os.path.join(self.results_dir, test_name + ".dmesg") try: with open(dmesg_path, "r", errors="backslashreplace") as f: if details: details.append("--\n") details.append(f"{dmesg_path}:\n") details.append(f.read()) except OSError: pass PK!Hs29.fb_benchpress-0.1.0.dist-info/entry_points.txtN+I/N.,()JJK((J-.E0s2RJ* 3JPK!HڽTU#fb_benchpress-0.1.0.dist-info/WHEEL1 0 нRZzq+|Pu}︁;-*nrSK(u >>F6_^>C-gfܖDOYTYPK!H&fb_benchpress-0.1.0.dist-info/METADATAQMO@ﯘH6@#F#b#e(uwKo7͛a@%5DR$aLqy~EIJFktG,(Gr*C 5ƐcOV$ oK~ORd?C)al<뿜(/W++.g]p;Gb (CsE]uD*= PK!HL $fb_benchpress-0.1.0.dist-info/RECORDɒH{= d@b]0}$ӏuuf}FQޏ4E[@Wd&qo%+._A)UHf3ҵ^u_惍`VTy| 5`ayDYu^}QN7]5{1&h?=l]VJ=ث(u_& =#ܳ1[BQbeyKå{jS[4{Y/qn7jn??yfyL֭#LԒهLmbĔG[qlKyG5 irukKv{zB 4V3 hԡ\o|bOb^̻piDI^| "k#ZƆ-+%f>My͈qTГsKi[4L { MW.2GSHxnNp=WB:{f8% IڵA끍 $<⌖38/x9ɱΘ,YʭI70VrI'[NB{wLn?)Ձ9FǤOXwJ(5 z z;G8bp^&afx9sd\:DHHzΊϮ?,׸:<ڠV?R6Bhs7+,W%t3=gb9S{$Nxܯ rl%> li3ܖ"wq90_=ICvŌk-ל&PTjjF|03ŧqH)KXorjƽȡ곩M(Z6+^vs:| -BC"Q64 ܧ|M" l+/\?ͤ.@)jJDGK_ERר {|7CVuC\  W&8'p^ &>Ϭ;Ed0} G;jbe3b $BB(ް ZiWqh 6\E e<bVP[ < X ʱMGOǝyqzfQwɤEu 6M-TKFW^78&#5Tg)s{q:\=>c[{DPK!benchpress/__init__.pyPK!$ >>4benchpress/cli/__init__.pyPK!#benchpress/cli/commands/__init__.pyPK!1&&"benchpress/cli/commands/command.pyPK!dQbenchpress/cli/commands/list.pyPK!씘Xbenchpress/cli/commands/run.pyPK!(B B , benchpress/cli/main.pyPK!benchpress/lib/__init__.pyPK!M4benchpress/lib/factory.pyPK!.ccbenchpress/lib/hook.pyPK!"P44!benchpress/lib/hook_factory.pyPK!rr$benchpress/lib/job.pyPK!گ[[>benchpress/lib/parser.pyPK!uFF 7Cbenchpress/lib/parser_factory.pyPK!J66Ebenchpress/lib/reporter.pyPK!")Lbenchpress/lib/reporter_factory.pyPK!-.Nbenchpress/lib/util.pyPK!Obenchpress/plugins/__init__.pyPK!1";zz$3Pbenchpress/plugins/hooks/__init__.pyPK!t  %Rbenchpress/plugins/hooks/cpu_limit.pyPK!& ;Xbenchpress/plugins/hooks/file.pyPK!!O{{!g_benchpress/plugins/hooks/shell.pyPK!X&!hbenchpress/plugins/parsers/__init__.pyPK!ENfȶ!"mbenchpress/plugins/parsers/fio.pyPK!y>%pbenchpress/plugins/parsers/generic.pyPK!pN>!wbenchpress/plugins/parsers/ltp.pyPK!u0!~benchpress/plugins/parsers/packetdrill_parser.pyPK!'< nn((benchpress/plugins/parsers/returncode.pyPK!t8DD&܄benchpress/plugins/parsers/schbench.pyPK!"dbenchpress/plugins/parsers/silo.pyPK!Ϩ(-zbenchpress/plugins/parsers/xfstests_parser.pyPK!Hs29.ܟfb_benchpress-0.1.0.dist-info/entry_points.txtPK!HڽTU#Zfb_benchpress-0.1.0.dist-info/WHEELPK!H&fb_benchpress-0.1.0.dist-info/METADATAPK!HL $Ofb_benchpress-0.1.0.dist-info/RECORDPK## Z