PK!VH??lhub_integ/__init__.pyfrom lhub_integ.decorators import action, connection_validator PK!_!lhub_integ/bundle_requirements.pyimport os import tempfile import subprocess """ Bundle all the dependencies in requirements.txt into a tar.bz2 by downloading and building wheels for all the dependencies in requirements.txt See https://pip.pypa.io/en/stable/user_guide/#installation-bundles for details """ def main(): with tempfile.TemporaryDirectory() as tmpdir: if not os.path.exists("requirements.txt"): print("Run in a directory with requirements.txt") exit(1) subprocess.run( [ "pip", "wheel", "-r", "requirements.txt", "--build-option", "--python-tag=py3", "--wheel-dir", tmpdir, ] ) subprocess.run(["cp", "requirements.txt", tmpdir]) cwd = os.getcwd() os.chdir(tmpdir) subprocess.run(["tar", "-cjvf", f"{cwd}/bundle.tar.bz2", *os.listdir(tmpdir)]) os.chdir(cwd) PK!l1 lhub_integ/decorators.pyfrom lhub_integ import util """ You can annotate a function with either @action or @action("action name"). Adding a name will set the name, otherwise it will default to the entrypoint. Part of the hairiness is from accepting both versions since they have totally different """ class action: actions = {} # add_action is called by all code paths for an action def add_action(self, f): # See https://www.python.org/dev/peps/pep-3155/ if f.__name__ != f.__qualname__: util.invalid_integration( code="invalid_action", error="actions must be top level functions" ) self.function = f self.entrypoint = f"{f.__module__}.{f.__name__}" if not self.name: self.name = self.entrypoint self.actions[self.entrypoint] = self def __init__(self, name=None, validator=None): # these will be set in add_action self.function = None self.entrypoint = None self.name = None self.validator = None # if the first argument is callable, the first argument is actually the function if name and callable(name): self.add_action(name) # otherwise we're in the @action(string) case where the given argument is the name else: self.name = name self.validator = validator def __call__(self, f, *args, **kwargs): # if @action case __call__ is called when the function is called, act as only a pass-through if self.function: return self.function(f, *args, **kwargs) # in the @action(string) case __call__ is called with the function itself and should return a function else: if not f: util.invalid_integration( code="invalid_action", error="@action must be called with a function", ) else: self.add_action(f) return f @classmethod def all(cls): return cls.actions class connection_validator: validator = None def __init__(self, f): self.f = f if connection_validator.validator is not None: util.invalid_integration( code="too_many_validators", error="Only one function can be annotated with @connection_validator", ) connection_validator.validator = self def __call__(self, *args, **kwargs): return self.f(*args, **kwargs) PK!<77lhub_integ/env.py""" Environment Variable Wrappers By wrapping environment variables in our classes, we can find what environment variables the integration uses at parse time. User-code should only use `EnvVar` -- `PrefixedEnvVar` subclasses are for internal use only. """ import os from lhub_integ.util import exit_with_instantiation_errors from abc import ABCMeta, abstractmethod class __EnvVar: """ __EnvVar should not be instantiated by Integrations! Use ConnectionParam or ActionParam instead """ def __init__(self, id, default=None, optional=False): """ :param id: ID of the environment variable """ self.id = id self.default = default self.optional = optional or default is not None def get_not_empty(self): env = os.environ.get(self.id) if env and len(env) > 0: return env else: return None def valid(self): return self.get_not_empty() or self.optional def read(self): value = self.get_not_empty() or self.default if value is None and not self.optional: exit_with_instantiation_errors( 1, [{"message": f"Environment variable {self.id} must be defined"}] ) else: return value def __str__(self): read = self.read() if not read: return "" return read class PrefixedEnvVar(__EnvVar, metaclass=ABCMeta): """ PrefixedEnvVar wraps Environment variables use for internal signalling to lhub_integ Specifically: Internal environment variables are prefixed with `__` Environment variables that map input columns to parameter ids are prefixed with `___` """ def __init__(self, id, *args, **kwargs): super().__init__(self.prefix() + id, *args, **kwargs) @classmethod @abstractmethod def prefix(cls): pass class InternalEnvVar(PrefixedEnvVar): @classmethod def prefix(cls): return "__" class MappedColumnEnvVar(PrefixedEnvVar): @classmethod def prefix(cls): return "___" PK!}lhub_integ/generate_metadata.py""" `generate_metadata` will load a Python module and generate a metadata JSON that the module system can use to build an integration descriptor. Usage: python -m lhub_integ.generate_metadata """ import inspect import json from dataclasses import dataclass from typing import List, Optional from dataclasses_json import dataclass_json from docstring_parser import parse from lhub_integ import util, action from lhub_integ.params import ( ConnectionParam, ActionParam, DataType, InputType, JinjaTemplatedStr, ) from lhub_integ.util import print_result, InvalidIntegration @dataclass @dataclass_json class Param: id: str label: str data_type: str input_type: str optional: bool = False description: Optional[str] = None default: Optional[str] = None options: Optional[List[str]] = None @dataclass @dataclass_json class Action: name: str entrypoint: str args: List[Param] params: List[Param] description: str errors: List[str] @dataclass @dataclass_json class IntegrationMetadata: connection_params: List[Param] integration_description: str actions: List[Action] errors: List[str] ok: bool def generate_metadata() -> IntegrationMetadata: # Side-effectful action of importing every file in the current working directory. # This will populate the internal class-storage for `action`, `ConnectionParam` and `ActionParam` toplevel_errors = [] parse_failure = False errors, docstrings = util.import_workdir() for error in errors: parse_failure = True if isinstance(error, SyntaxError): toplevel_errors.append(f"Failed to import module: {error}") elif isinstance(error, ImportError): toplevel_errors.append( f"Failed to import module (did you run bundle-integrations?): {error}" ) elif isinstance(error, InvalidIntegration): toplevel_errors.append(f"Integration was invalid: {error}") else: toplevel_errors.append( f"Failed to import module (Unexpected error): {error}" ) actions = [] for action_object in action.all().values(): entrypoint = action_object.entrypoint processing_function = action_object.function action_name = action_object.name # Parse the docstring docs = processing_function.__doc__ parsed = parse(docs) errors = [] # Any docstring on the function itself will be used as the description for the integration if parsed.short_description or parsed.long_description: function_description = parsed.short_description or parsed.long_description else: function_description = None # Build a map of the actual function arguments to compare with the docstrings args, varargs, kwargs = inspect.getargs(processing_function.__code__) type_annotations = util.type_annotations(processing_function) if varargs: errors.append("Varargs are not supported") if kwargs: errors.append("Kwargs are not supported") # default of a label is the id, may update it below when iterating over metadata arg_map = { arg: {"id": arg, "label": arg, "type": type_annotations[arg]} for arg in args } # load labels for column parameters for meta in parsed.meta: if meta.args[0] == "label" and len(meta.args) == 2: arg_map[meta.args[1]]["label"] = meta.description # Augment arg_map with information from the docstrings for param in parsed.params: if param.arg_name not in arg_map: errors.append( f"Docstring referenced {param.arg_name} but there were no matching arguments" ) else: if param.description is not None: arg_map[param.arg_name]["description"] = param.description if param.type_name is not None: arg_map[param.arg_name]["tpe"] = param.type_name params = [ Param( id=e.id, label=e.label, description=e.description, default=e.default, optional=e.optional, data_type=e.data_type.value, options=e.options, input_type=e.input_type.value, ) for e in ActionParam.for_action(action_object) ] args = [arg_map[arg] for arg in args] args = [ Param( id=arg["id"], label=arg["label"], description=arg.get("description"), data_type=JinjaTemplatedStr.data_type().value if arg["type"] == JinjaTemplatedStr else DataType.COLUMN.value, input_type=JinjaTemplatedStr.input_type().value if arg["type"] == JinjaTemplatedStr else InputType.COLUMN_SELECT.value, ) for arg in args ] actions.append( Action( name=action_name, entrypoint=entrypoint, args=args, params=params, description=function_description, errors=errors, ) ) connection_params = [ Param( id=e.id, label=e.label, description=e.description, default=e.default, optional=e.optional, data_type=e.data_type.value, options=e.options, input_type=e.input_type.value, ) for e in ConnectionParam.all() ] if not actions and not parse_failure: toplevel_errors.append("No actions found. Did you forget to use @action?") if docstrings: integration_description = docstrings[0].strip() elif len(actions) == 1: integration_description = actions[0].description else: integration_description = "No description provided" metadata = IntegrationMetadata( connection_params=connection_params, integration_description=integration_description, actions=actions, errors=toplevel_errors, ok=len(toplevel_errors) == 0, ) return metadata if __name__ == "__main__": import traceback try: print_result(generate_metadata().to_json()) except Exception: print_result(json.dumps(dict(errors=[traceback.format_exc()]))) exit(1) PK!Llhub_integ/params.pyimport inspect import json from abc import ABCMeta from collections import defaultdict from typing import NamedTuple, Dict, Callable, Any from lhub_integ.decorators import action as action_decorator from lhub_integ.env import __EnvVar from jinja2 import Template from enum import Enum TRUTHY = {"y", "true", "yes", "1"} FALSY = {"n", "false", "no", "0"} # pulled from forms.model.DataType class DataType(Enum): STRING = "string" COLUMN = "column" NUMBER = "number" # no node datatype because integrations can only pull from one node # Unsupported by UI but validated by custom integrations BOOL = "bool" INT = "int" JSON = "json" JINJA = "jinja" def coerce(self, inp): if self == self.NUMBER: return float(inp) elif self == self.INT: return int(float(inp)) elif self == self.BOOL: if isinstance(inp, bool): return inp if inp.lower() in TRUTHY: return True if inp.lower() in FALSY: return False raise ValueError( "Expected boolean input but {input} could not be coerced into a boolean value" ) elif self == self.JSON: if isinstance(inp, dict): return inp return json.loads(inp) else: return inp # pulled from forms.model.InputType class InputType(Enum): TEXT = "text" TEXT_AREA = "textarea" EMAIL = "email" PASSWORD = "password" SELECT = "select" COLUMN_SELECT = "columnSelect" class __Param(__EnvVar, metaclass=ABCMeta): def __init__( self, id, description=None, label=None, default=None, optional=False, options=None, data_type=DataType.STRING, input_type=InputType.TEXT, ): super().__init__(id, default, optional) if label: self.label = label else: self.label = id self.description = description self.default = default self.data_type = data_type self.options = options if data_type == DataType.COLUMN: self.input_type = InputType.InputType.COLUMN_SELECT elif options is not None and len(options) > 1: self.input_type = InputType.SELECT else: self.input_type = input_type def read(self): raw = super().read() return self.data_type.coerce(raw) class JinjaTemplatedStr(str): @classmethod def input_type(cls) -> InputType: return InputType.TEXT_AREA @classmethod def data_type(cls) -> DataType: return DataType.JINJA """ We take most of the param information from our Form.Input case class. We don't enable a dependsOn field because if the dataType is a column then it will auto depends on its parent. """ class ConnectionParam(__Param, metaclass=ABCMeta): """ ConnectionParam provides a parameter specified by the connection Example usage: API_KEY = ConnectionParam('api_key') def process_row(url): requests.get(url, params={api_key: API_KEY.get()}) """ _all = set() def __init__(self, *args, **kwargs): from lhub_integ import util super().__init__(*args, **kwargs) for conn_param in self._all: if conn_param.id == self.id: util.invalid_integration( "duplication_connection_param", f"You can't have 2 connection parameters with the same id ({self.id})", ) self._all.add(self) @classmethod def all(cls): return sorted(cls._all, key=lambda var: var.label) class ActionParam(__Param, metaclass=ABCMeta): """ ActionParam provides a parameter specified by the action Example usage: API_KEY = ActionParam('api_key', action='process_row') import requests def process_row(url): requests.get(url, params={api_key: API_KEY.get()}) """ action_map = defaultdict(set) def __init__(self, *args, action: str, **kwargs): super().__init__(*args, **kwargs) from lhub_integ import util caller = inspect.currentframe().f_back entrypoint = f"{caller.f_globals['__name__']}.{action}" if entrypoint in self.action_map: for action_param in self.action_map[entrypoint]: if action_param.id == self.id: util.invalid_integration( "duplicate_action_param", f"You can't have 2 action parameters with the same id ({action})", ) self.action_map[entrypoint].add(self) @classmethod def for_action(cls, action: action_decorator): return sorted(cls.action_map[action.entrypoint], key=lambda var: var.label) class ValidationError(NamedTuple): message: str param: "__EnvVar" def to_json(self): return {"message": self.message, "inputId": self.param.id} CONVERTIBLE_TYPES = [int, str, float, bool, JinjaTemplatedStr] def convert(c): def do_convert(raw: Dict[str, Any], column): if c == JinjaTemplatedStr: template = Template(column) return template.render(**raw) else: value = raw[column] if c in [str, float]: return c(value) if c == int: return int(float(value)) elif c == bool: return DataType.BOOL.coerce(value) return do_convert def get_input_converter(entrypoint_fn) -> Dict[str, Callable[[str], Any]]: """ Build the input_conversion map to allow promotion from String to to int, float, and bool :param entrypoint_fn: :return: Dict from the name of the function arguments to a converter function. """ sig = inspect.signature(entrypoint_fn) converter = {} from lhub_integ.util import exit_with_instantiation_errors for param in sig.parameters: annot = sig.parameters[param].annotation # The annotation is the Python class -- in these simple cases we can just call # the class constructor if annot in CONVERTIBLE_TYPES: converter[param] = convert(annot) elif annot == inspect.Parameter.empty: converter[param] = lambda raw, column: raw[column] else: exit_with_instantiation_errors( 1, [ f"Unsupported type annotation: {annot}. Valid annotations are: {CONVERTIBLE_TYPES}" ], ) return converter PK! M M lhub_integ/shim_exec.py""" Wrap a custom Python script to enable it to work with the integration machinery Loads ENTRYPOINT from `__ENTRYPOINT` and the column-argument mapping from `___XXXXX` Usage: This module is intended to be invoked by `PythonEnvironment`. See `test_shim.py` for a usage example python -m lhub_integ.shim_exec -e """ import json import sys import traceback import click from lhub_integ import util, action, params from lhub_integ.env import MappedColumnEnvVar from lhub_integ.params import JinjaTemplatedStr def run_integration(entrypoint_fn): type_mapper = params.get_input_converter(entrypoint_fn) annotations = util.type_annotations(entrypoint_fn) # Build a mapping from the columns in the input data to the arguments in our integration argument_column_mapping = { input_argument: MappedColumnEnvVar(input_argument).read() for input_argument in type_mapper } required_columns = { k: v for k, v in argument_column_mapping.items() if annotations.get(k) != JinjaTemplatedStr } for row in sys.stdin.readlines(): as_dict = json.loads(row)["row"] lhub_id = as_dict.get(util.LHUB_ID) missing_keys = required_columns.values() - as_dict.keys() for column in missing_keys: util.print_error(f"Column '{column}' is not present in the data") if missing_keys: continue try: # Load the arguments from the input def coerce(arg, column): try: return type_mapper[arg](as_dict, column) except ValueError as ex: util.print_error( f"Invalid value for column {column}: [{ex}]", data=as_dict ) raise EndOfLoop() arguments = { arg: coerce(arg, column) for arg, column in argument_column_mapping.items() } # Run the integration result = entrypoint_fn(**arguments) # Print the results if result: if isinstance(result, list): util.print_each_result_in_list(result, lhub_id) else: util.print_result(json.dumps(result), lhub_id) except EndOfLoop: pass except Exception: util.print_error(traceback.format_exc(), data=as_dict) class EndOfLoop(Exception): pass @click.command() @click.option("--entrypoint", "-e", required=True) def main(entrypoint): errors, _ = util.import_workdir() if errors: util.hard_exit_from_instantiation(str(errors[0])) entrypoint_fn = action.all().get(entrypoint).function assert entrypoint_fn is not None run_integration(entrypoint_fn) if __name__ == "__main__": main() PK!L%&55lhub_integ/shim_validate.py""" Validation Shim for custom integrations """ import inspect from traceback import format_exc import click from lhub_integ import action, util, connection_validator from lhub_integ.env import MappedColumnEnvVar from lhub_integ.params import ConnectionParam, ActionParam, ValidationError from lhub_integ.util import ( exit_with_instantiation_errors, print_successful_validation_result, ) @click.command() @click.option("--entrypoint", "-e", required=False) @click.option( "--validate-connections/--no-validate-connections", "check_connections", default=False, ) def main(entrypoint, check_connections): validation_errors = [] try: errors, _ = util.import_workdir() if errors: util.hard_exit_from_instantiation(f"Failed to parse integration: {errors}") if not validation_errors: if entrypoint is not None: validation_errors += validate_entrypoint(entrypoint) if check_connections: validation_errors += validate_connections() validation_errors = [v.to_json() for v in validation_errors] except Exception: util.hard_exit_from_instantiation( message=f"Unexpected exception: {format_exc()}" ) if validation_errors: exit_with_instantiation_errors( 1, validation_errors, message="Integration validation failed." ) else: print_successful_validation_result() def validate_param(param): if not param.valid(): return [ValidationError(message="Parameter must be defined", param=param)] try: param.read() except Exception as ex: return [ValidationError(message=str(ex), param=param)] return [] def validate_connections(): errors = [] try: for var in ConnectionParam.all(): errors += validate_param(var) if not errors and connection_validator.validator is not None: return normalize_validation_result(connection_validator.validator()) except Exception as ex: util.print_error(str(ex)) exit(1) return errors def normalize_validation_result(res): if res is None: return [] if not isinstance(res, list): util.hard_exit_from_instantiation( f"Validator functions must return a list of ValidationError (got {res})" ) else: return res def validate_entrypoint(entrypoint): module_name = ".".join(entrypoint.split(".")[:-1]) function_name = entrypoint.split(".")[-1] if module_name == "" or function_name == "": util.hard_exit_from_instantiation( "Bad entrypoint format. `Expected filename.functionname`" ) # Try to import the world and find the entrypoint action_object = action.all().get(entrypoint) method = action_object.function if method is None: util.hard_exit_from_instantiation( f"No matching action found. Is your action annotated with @action?" ) errors = [] # Read the arguments and environment variables we expect and make sure they've all been defined args, _, _ = inspect.getargs(method.__code__) for arg in args: if not MappedColumnEnvVar(arg).valid(): errors.append( ValidationError( message="Column name cannot be empty", param=MappedColumnEnvVar(arg) ) ) env_vars = list(ConnectionParam.all()) + list(ActionParam.for_action(action_object)) for var in env_vars: errors += validate_param(var) if errors: return errors if action_object.validator is not None: try: return normalize_validation_result(action_object.validator()) except Exception as ex: util.hard_exit_from_instantiation(str(ex)) return [] if __name__ == "__main__": main() PK!4H  lhub_integ/test_data_type.pyimport pytest from lhub_integ.params import DataType def test_numeric_coercion(): assert DataType.INT.coerce("123") == 123 assert DataType.INT.coerce("123.5") == 123 assert DataType.NUMBER.coerce("123.5") == 123.5 def test_bool_coercion(): assert DataType.BOOL.coerce("false") is False assert DataType.BOOL.coerce("true") is True with pytest.raises(ValueError): DataType.BOOL.coerce("cat") assert DataType.BOOL.coerce(True) is True assert DataType.BOOL.coerce(False) is False assert DataType.BOOL.coerce("True") is True assert DataType.BOOL.coerce("False") is False def test_json_coercion(): assert DataType.JSON.coerce("[1,2,3]") == [1, 2, 3] with pytest.raises(ValueError): DataType.JSON.coerce("1,2,3") PK!I/m lhub_integ/util.pyfrom __future__ import print_function import fileinput import importlib import inspect import json import sys from pathlib import Path from typing import List, Tuple LHUB_ID = "lhub_id" def read_all_data(): """ If an argument is provided, fileinput.input() will iterate over the lines in that file. Otherwise, it will read from stdin. See https://docs.python.org/3/library/fileinput.html :return: """ return fileinput.input() def hard_exit_from_instantiation(message): exit_with_instantiation_errors(2, [], message) def exit_with_instantiation_errors( code, errors, message="Integration validation failed." ): error_wrapper = {"errors": errors, "message": message} print(f"[result] {json.dumps(error_wrapper)}", file=sys.stderr) sys.exit(code) def invalid_integration(code, error): raise InvalidIntegration(code, error) def print_error(message: str, data=None): error = {"has_error": True, "error": message, "data": data} print_result(json.dumps(error)) def print_result(msg, original_lhub_id=None): if original_lhub_id is not None: print_correlated_result(msg, original_lhub_id) else: print(f"[result] {msg}") def print_correlated_result(msg, original_lhub_id): meta_data_dict = {"original_lhub_id": original_lhub_id} meta_json = json.dumps(meta_data_dict) print(f"[result][meta]{meta_json}[/meta] {msg}") def print_successful_validation_result(): print_result("{}") def print_each_result_in_list(results, original_lhub_id=None): if not results: return print_result( json.dumps({"noResults": "no results returned"}), original_lhub_id=original_lhub_id, ) for result in results: print_result(json.dumps(result), original_lhub_id=original_lhub_id) def import_workdir() -> Tuple[List[Exception], List[str]]: """ Attempt to import all the files in the working directory :return: A list of errors """ errors = [] docstrings = [] for file in Path(".").iterdir(): if file.suffix == ".py": as_module = file.name[: -len(".py")] if "." in as_module: errors.append(f"Python files cannot contain dots: {as_module}") continue try: module = importlib.import_module(as_module) doc = module.__doc__ if doc: docstrings.append(doc) except Exception as ex: errors.append(ex) return errors, docstrings def get_entrypoint_fn(entrypoint): module_name = ".".join(entrypoint.split(".")[:-1]) function_name = entrypoint.split(".")[-1] module = importlib.import_module(module_name) return getattr(module, function_name) def get_module_docstring(entrypoint): module_name = ".".join(entrypoint.split(".")[:-1]) module = importlib.import_module(module_name) if module.__doc__: return module.__doc__.strip() return None def type_annotations(fn): sig = inspect.signature(fn) return {param: sig.parameters[param].annotation for param in sig.parameters} def deser_output(output): prefix = "[result]" if output.startswith(prefix): output = output[len(prefix) :] try: return json.loads(output) except json.decoder.JSONDecodeError: raise Exception(f"Could not parse JSON: {output}") class InvalidIntegration(Exception): def __init__(self, key, message): super().__init__(f"{message} [{key}]") PK!H?K+lhub_integ-0.1.5.dist-info/entry_points.txtN+I/N.,()J*KI-J-,,JM+)(M+IM׃H#K[&fqqPK!H STT lhub_integ-0.1.5.dist-info/WHEEL 1 0 нR. \I$ơ7.ZON `h6oi14m,b4>4ɛpK>X;baP>PK!HN8N#lhub_integ-0.1.5.dist-info/METADATATnH } `-% EŦM .OxD[H3ڹU{93/iC&Cpޣ%w<Z0`u嗙Tl=7 7] F (\ᎀ0z f-_Sʹuͅn؍,y_;6ڔ1N71xc)[u B[3l 0=AuN^eͭ+EkxݶF?b}rg4u4Rᆫn> _?!+uI`(H{G $ G= ZrՅG [T%*!ѲO3Vn+Iz q^ez:XzUR*%/=H{9sM+-ã=䌱PK!H)"jb!lhub_integ-0.1.5.dist-info/RECORD}rH{? -Fb1[J-CD/L\Ӹl _$qX$ f:.[_,MeCb0NI_B// x@{9d\ω-Opn>#:[7K [+ھ%J솄tO"ń-l@G3[Q{8a"՛bF^}$d\W!}yrd&O;uB W5D,!ʅ[>q~spZ×a1p m=D!iqAys5vtcB)sS.vp6G~a<Ƣlb4!XѫJ瓅|h eپmwb%Hp V5ҁdǼCQR/cϽe]#$h$c2k #8yg(?O'Kdy'%^#iH0bN)TU:_! 6&eH6e{^D9D{>}$c`{vJ;e sAf R|MAO Y=7 虙֮Zri4/N$'/\oy:ܫ4c!0t"lcV bd;j9EPK!VH??lhub_integ/__init__.pyPK!_!slhub_integ/bundle_requirements.pyPK!l1 lhub_integ/decorators.pyPK!<77lhub_integ/env.pyPK!}lhub_integ/generate_metadata.pyPK!L1lhub_integ/params.pyPK! M M >Klhub_integ/shim_exec.pyPK!L%&55Vlhub_integ/shim_validate.pyPK!4H  .flhub_integ/test_data_type.pyPK!I/m uilhub_integ/util.pyPK!H?K+wlhub_integ-0.1.5.dist-info/entry_points.txtPK!H STT xlhub_integ-0.1.5.dist-info/WHEELPK!HN8N#xlhub_integ-0.1.5.dist-info/METADATAPK!H)"jb!(|lhub_integ-0.1.5.dist-info/RECORDPK