PK!VH??lhub_integ/__init__.pyfrom lhub_integ.decorators import action, connection_validator PK! wd]LL!lhub_integ/bundle_requirements.pyimport os import tempfile import subprocess """ Bundle all the dependencies in requirements.txt into a tar.bz2 by downloading and building wheels for all the dependencies in requirements.txt See https://pip.pypa.io/en/stable/user_guide/#installation-bundles for details """ def main(): with tempfile.TemporaryDirectory() as tmpdir: if not os.path.exists("requirements.txt"): print("Run in a directory with requirements.txt") exit(1) subprocess.run( ["pip", "wheel", "-r", "requirements.txt", "--build-option", "--python-tag=py3", "--wheel-dir", tmpdir] ) subprocess.run(["cp", "requirements.txt", tmpdir]) cwd = os.getcwd() os.chdir(tmpdir) subprocess.run(["tar", "-cjvf", f"{cwd}/bundle.tar.bz2", *os.listdir(tmpdir)]) os.chdir(cwd) PK!W0  lhub_integ/decorators.pyfrom lhub_integ import util """ You can annotate a function with either @action or @action("action name"). Adding a name will set the name, otherwise it will default to the entrypoint. Part of the hairiness is from accepting both versions since they have totally different """ class action: actions = {} # add_action is called by all code paths for an action def add_action(self, f): # See https://www.python.org/dev/peps/pep-3155/ if f.__name__ != f.__qualname__: util.exit_with_instantiation_errors( code="invalid_action", errors=["actions must be top level functions"] ) self.function = f self.entrypoint = f"{f.__module__}.{f.__name__}" if not self.name: self.name = self.entrypoint self.actions[self.entrypoint] = self def __init__(self, name=None, validator=None): # these will be set in add_action self.function = None self.entrypoint = None self.name = None self.validator = None # if the first argument is callable, the first argument is actually the function if name and callable(name): self.add_action(name) # otherwise we're in the @action(string) case where the given argument is the name else: self.name = name self.validator = validator def __call__(self, f, *args, **kwargs): # if @action case __call__ is called when the function is called, act as only a pass-through if self.function: self.function(f, *args, **kwargs) # in the @action(string) case __call__ is called with the function itself and should return a function else: if not f: util.exit_with_instantiation_errors( code="invalid_action", errors=["@action is not called with a function"], ) else: self.add_action(f) return f @classmethod def all(cls): return cls.actions class connection_validator: validator = None def __init__(self, f): self.f = f if connection_validator.validator is not None: util.exit_with_instantiation_errors( code="too_many_validators", errors=[ "Only one function can be annotated with @connection_validator" ], ) connection_validator.validator = self def __call__(self, *args, **kwargs): self.f(*args, **kwargs) PK!<77lhub_integ/env.py""" Environment Variable Wrappers By wrapping environment variables in our classes, we can find what environment variables the integration uses at parse time. User-code should only use `EnvVar` -- `PrefixedEnvVar` subclasses are for internal use only. """ import os from lhub_integ.util import exit_with_instantiation_errors from abc import ABCMeta, abstractmethod class __EnvVar: """ __EnvVar should not be instantiated by Integrations! Use ConnectionParam or ActionParam instead """ def __init__(self, id, default=None, optional=False): """ :param id: ID of the environment variable """ self.id = id self.default = default self.optional = optional or default is not None def get_not_empty(self): env = os.environ.get(self.id) if env and len(env) > 0: return env else: return None def valid(self): return self.get_not_empty() or self.optional def read(self): value = self.get_not_empty() or self.default if value is None and not self.optional: exit_with_instantiation_errors( 1, [{"message": f"Environment variable {self.id} must be defined"}] ) else: return value def __str__(self): read = self.read() if not read: return "" return read class PrefixedEnvVar(__EnvVar, metaclass=ABCMeta): """ PrefixedEnvVar wraps Environment variables use for internal signalling to lhub_integ Specifically: Internal environment variables are prefixed with `__` Environment variables that map input columns to parameter ids are prefixed with `___` """ def __init__(self, id, *args, **kwargs): super().__init__(self.prefix() + id, *args, **kwargs) @classmethod @abstractmethod def prefix(cls): pass class InternalEnvVar(PrefixedEnvVar): @classmethod def prefix(cls): return "__" class MappedColumnEnvVar(PrefixedEnvVar): @classmethod def prefix(cls): return "___" PK!~mlhub_integ/generate_metadata.py""" `generate_metadata` will load a Python module and generate a metadata JSON that the module system can use to build an integration descriptor. Usage: python -m lhub_integ.generate_metadata """ import inspect import json from dataclasses import dataclass from typing import List, Optional from dataclasses_json import dataclass_json from docstring_parser import parse from lhub_integ import util, action from lhub_integ.params import ConnectionParam, ActionParam, DataType, InputType from lhub_integ.util import print_result @dataclass @dataclass_json class Param: id: str label: str data_type: str input_type: str optional: bool = False description: Optional[str] = None default: Optional[str] = None options: Optional[List[str]] = None @dataclass @dataclass_json class Action: name: str entrypoint: str args: List[Param] params: List[Param] description: str errors: List[str] @dataclass @dataclass_json class IntegrationMetadata: connection_params: List[Param] integration_description: str actions: List[Action] errors: List[str] ok: bool def generate_metadata() -> IntegrationMetadata: # Side-effectful action of importing every file in the current working directory. # This will populate the internal class-storage for `action`, `ConnectionParam` and `ActionParam` toplevel_errors = [] parse_failure = False errors, docstrings = util.import_workdir() for error in errors: parse_failure = True if isinstance(error, SyntaxError): toplevel_errors.append(f"Failed to import module: {error}") elif isinstance(error, ImportError): toplevel_errors.append( f"Failed to import module (did you run bundle-integrations?): {error}" ) else: toplevel_errors.append( f"Failed to import module (Unexpected error): {error}" ) actions = [] for action_object in action.all().values(): entrypoint = action_object.entrypoint processing_function = action_object.function action_name = action_object.name # Parse the docstring docs = processing_function.__doc__ parsed = parse(docs) errors = [] # Any docstring on the function itself will be used as the description for the integration if parsed.short_description or parsed.long_description: function_description = parsed.short_description or parsed.long_description else: function_description = None # Build a map of the actual function arguments to compare with the docstrings args, varargs, kwargs = inspect.getargs(processing_function.__code__) if varargs: errors.append("Varargs are not supported") if kwargs: errors.append("Kwargs are not supported") # default of a label is the id, may update it below when iterating over metadata arg_map = {arg: {"id": arg, "label": arg} for arg in args} # load labels for column parameters for meta in parsed.meta: if meta.args[0] == "label" and len(meta.args) == 2: arg_map[meta.args[1]]["label"] = meta.description # Augment arg_map with information from the docstrings for param in parsed.params: if param.arg_name not in arg_map: errors.append( f"Docstring referenced {param.arg_name} but there were no matching arguments" ) else: if param.description is not None: arg_map[param.arg_name]["description"] = param.description if param.type_name is not None: arg_map[param.arg_name]["tpe"] = param.type_name params = [ Param( id=e.id, label=e.label, description=e.description, default=e.default, optional=e.optional, data_type=e.data_type.value, options=e.options, input_type=e.input_type.value, ) for e in ActionParam.for_action(action_object) ] args = [arg_map[arg] for arg in args] args = [ Param( id=arg["id"], label=arg["label"], description=arg.get("description"), data_type=DataType.COLUMN.value, input_type=InputType.COLUMN_SELECT.value, ) for arg in args ] actions.append( Action( name=action_name, entrypoint=entrypoint, args=args, params=params, description=function_description, errors=errors, ) ) connection_params = [ Param( id=e.id, label=e.label, description=e.description, default=e.default, optional=e.optional, data_type=e.data_type.value, options=e.options, input_type=e.input_type.value, ) for e in ConnectionParam.all() ] if not actions and not parse_failure: toplevel_errors.append("No actions found. Did you forget to use @action?") if docstrings: integration_description = docstrings[0].strip() elif len(actions) == 1: integration_description = actions[0].description else: integration_description = "No description provided" metadata = IntegrationMetadata( connection_params=connection_params, integration_description=integration_description, actions=actions, errors=toplevel_errors, ok=len(toplevel_errors) == 0, ) return metadata if __name__ == "__main__": import traceback try: print_result(generate_metadata().to_json()) except Exception: print_result(json.dumps(dict(errors=[traceback.format_exc()]))) exit(1) PK!gyIL L lhub_integ/params.pyimport inspect from abc import ABCMeta from collections import defaultdict from lhub_integ.decorators import action as action_decorator from lhub_integ.env import __EnvVar, InternalEnvVar from enum import Enum # pulled from forms.model.DataType class DataType(Enum): STRING = "string" COLUMN = "column" NUMBER = "number" # no node datatype because integrations can only pull from one node # pulled from forms.model.InputType class InputType(Enum): TEXT = "text" TEXT_AREA = "textarea" EMAIL = "email" PASSWORD = "password" SELECT = "select" COLUMN_SELECT = "columnSelect" class __Param(__EnvVar, metaclass=ABCMeta): def __init__( self, id, description=None, label=None, default=None, optional=False, options=None, data_type=DataType.STRING, input_type=InputType.TEXT, ): super().__init__(id, default, optional) if label: self.label = label else: self.label = id self.description = description self.default = default self.data_type = data_type self.options = options if data_type == DataType.COLUMN: self.input_type = InputType.InputType.COLUMN_SELECT elif options is not None and len(options) > 1: self.input_type = InputType.SELECT else: self.input_type = input_type """ We take most of the param information from our Form.Input case class. We don't enable a dependsOn field because if the dataType is a column then it will auto depends on its parent. """ class ConnectionParam(__Param, metaclass=ABCMeta): """ ConnectionParam provides a parameter specified by the connection Example usage: API_KEY = ConnectionParam('api_key') def process_row(url): requests.get(url, params={api_key: API_KEY.get()}) """ _all = set() def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._all.add(self) @classmethod def all(cls): return sorted(cls._all, key=lambda var: var.label) class ActionParam(__Param, metaclass=ABCMeta): """ ActionParam provides a parameter specified by the action Example usage: API_KEY = ConnectionParam('api_key', action='process_row') import requests def process_row(url): requests.get(url, params={api_key: API_KEY.get()}) """ action_map = defaultdict(set) def __init__(self, *args, action: str, **kwargs): super().__init__(*args, **kwargs) caller = inspect.currentframe().f_back entrypoint = f"{caller.f_globals['__name__']}.{action}" self.action_map[entrypoint].add(self) @classmethod def for_action(cls, action: action_decorator): return sorted(cls.action_map[action.entrypoint], key=lambda var: var.label) PK!U``lhub_integ/shim_exec.py""" Wrap a custom Python script to enable it to work with the integration machinery Loads ENTRYPOINT from `__ENTRYPOINT` and the column-argument mapping from `___XXXXX` Usage: This module is intended to be invoked by `PythonEnvironment`. See `test_shim.py` for a usage example python -m lhub_integ.shim_exec -e """ import json import sys import traceback import click from lhub_integ import util, action from lhub_integ.env import MappedColumnEnvVar def run_integration(entrypoint_fn): type_mapper = util.get_input_converter(entrypoint_fn) # Build a mapping from the columns in the input data to the arguments in our integration argument_column_mapping = { input_argument: MappedColumnEnvVar(input_argument).read() for input_argument in type_mapper } for row in sys.stdin.readlines(): as_dict = json.loads(row)["row"] lhub_id = as_dict.get(util.LHUB_ID) missing_keys = argument_column_mapping.values() - as_dict.keys() for column in missing_keys: util.print_error(f"Column '{column}' is not present in the data") if missing_keys: continue try: # Load the arguments from the input arguments = { arg: type_mapper[arg](as_dict[column]) for arg, column in argument_column_mapping.items() } # Run the integration result = entrypoint_fn(**arguments) # Print the results if result: if isinstance(result, list): util.print_each_result_in_list(result, lhub_id) else: util.print_result(json.dumps(result), lhub_id) except Exception: util.print_error(traceback.format_exc(), data=as_dict) @click.command() @click.option("--entrypoint", "-e", required=True) def main(entrypoint): util.import_workdir() entrypoint_fn = action.all().get(entrypoint).function assert entrypoint_fn is not None run_integration(entrypoint_fn) if __name__ == "__main__": main() PK!$33lhub_integ/shim_validate.py""" Validate that all the required environment variables have been set to execute this integration. If this fails, it indicates a bug in the PythonEnvironment machinery. Currently, this doesn't offer the ability for the user to run custom validations but we could add it here. eg: We could look for a method like `module.entrypoint_validate()` and run it. Currently we don't """ import inspect import traceback import click from lhub_integ import action, util, connection_validator from lhub_integ.env import MappedColumnEnvVar from lhub_integ.params import ConnectionParam, ActionParam from lhub_integ.util import ( exit_with_instantiation_errors, print_successful_validation_result, ) @click.command() @click.option("--entrypoint", "-e", required=False) @click.option( "--validate-connections/--no-validate-connections", "check_connections", default=False, ) def main(entrypoint, check_connections): validation_errors = [] try: try: util.import_workdir() except ImportError as ex: validation_errors += [{"message": f"Can't import {module_name}: {ex}"}] if entrypoint is not None: validation_errors += validate_entrypoint(entrypoint) if check_connections: validation_errors += validate_connections() except Exception as ex: validation_errors = [ { "message": f"Unexpected exception: {ex}", "stack_trace": traceback.format_exc(), } ] if validation_errors: exit_with_instantiation_errors( 1, validation_errors, message="Integration validation failed." ) else: print_successful_validation_result() def validate_connections(): errors = [] try: for var in ConnectionParam.all(): if not var.valid(): errors.append( {"message": "Parameter must be defined", "inputId": var.id} ) if not errors and connection_validator.validator is not None: connection_validator.validator() except Exception as ex: errors += [{"message": str(ex)}] return errors def validate_entrypoint(entrypoint): module_name = ".".join(entrypoint.split(".")[:-1]) function_name = entrypoint.split(".")[-1] if module_name == "" or function_name == "": return [{"message": "Bad entrypoint format. `Expected filename.functionname`"}] # Try to import the world and find the entrypoint action_object = action.all().get(entrypoint) method = action_object.function if method is None: return [ { "message": f"No matching action found. Is your action annotated with @action?" } ] errors = [] # Read the arguments and environment variables we expect and make sure they've all been defined args, _, _ = inspect.getargs(method.__code__) for arg in args: if not MappedColumnEnvVar(arg).valid(): errors.append({"message": "Column name cannot be empty", "inputId": MappedColumnEnvVar(arg).id}) env_vars = list(ConnectionParam.all()) + list(ActionParam.for_action(action_object)) for var in env_vars: if not var.valid(): errors.append({"message": "Parameter cannot be empty", "inputId": var.id}) if errors: return errors if action_object.validator is not None: try: action_object.validator() except Exception as ex: errors.append({"message": str(ex)}) return errors if __name__ == "__main__": main() PK!sZCClhub_integ/util.pyfrom __future__ import print_function import importlib import inspect import json import sys import fileinput from pathlib import Path from typing import Dict, Callable, Any, List, Tuple LHUB_ID = "lhub_id" def read_all_data(): """ If an argument is provided, fileinput.input() will iterate over the lines in that file. Otherwise, it will read from stdin. See https://docs.python.org/3/library/fileinput.html :return: """ return fileinput.input() def exit_with_instantiation_errors( code, errors, message="Integration validation failed." ): error_wrapper = {"errors": errors, "message": message} print(f"[result] {json.dumps(error_wrapper)}", file=sys.stderr) sys.exit(code) def print_error(message: str, data=None): error = {"has_error": True, "error": message, "data": data} print_result(json.dumps(error)) def print_result(msg, original_lhub_id=None): if original_lhub_id is not None: print_correlated_result(msg, original_lhub_id) else: print(f"[result] {msg}") def print_correlated_result(msg, original_lhub_id): meta_data_dict = {"original_lhub_id": original_lhub_id} meta_json = json.dumps(meta_data_dict) print(f"[result][meta]{meta_json}[/meta] {msg}") def print_successful_validation_result(): print_result("{}") def print_each_result_in_list(results, original_lhub_id=None): if not results: return print_result( json.dumps({"noResults": "no results returned"}), original_lhub_id=original_lhub_id, ) for result in results: print_result(json.dumps(result), original_lhub_id=original_lhub_id) def import_workdir() -> Tuple[List[Exception], List[str]]: """ Attempt to import all the files in the working directory :return: A list of errors """ errors = [] docstrings = [] for file in Path(".").iterdir(): if file.suffix == ".py": as_module = file.name[: -len(".py")] if "." in as_module: errors.append(f"Python files cannot contain dots: {as_module}") continue try: module = importlib.import_module(as_module) doc = module.__doc__ if doc: docstrings.append(doc) except Exception as ex: errors.append(ex) return errors, docstrings def get_entrypoint_fn(entrypoint): module_name = ".".join(entrypoint.split(".")[:-1]) function_name = entrypoint.split(".")[-1] module = importlib.import_module(module_name) return getattr(module, function_name) def get_module_docstring(entrypoint): module_name = ".".join(entrypoint.split(".")[:-1]) module = importlib.import_module(module_name) if module.__doc__: return module.__doc__.strip() return None CONVERTIBLE_TYPES = [int, str, float, bool] def get_input_converter(entrypoint_fn) -> Dict[str, Callable[[str], Any]]: """ Build the input_conversion map to allow promotion from String to to int, float, and bool :param entrypoint_fn: :return: Dict from the name of the function arguments to a converter function. """ sig = inspect.signature(entrypoint_fn) converter = {} for param in sig.parameters: annot = sig.parameters[param].annotation # The annotation is the Python class -- in these simple cases we can just call # the class constructor if annot in CONVERTIBLE_TYPES: converter[param] = lambda inp: annot(inp) elif annot == inspect.Parameter.empty: converter[param] = lambda x: x else: exit_with_instantiation_errors( 1, [ f"Unsupported type annotation: {annot}. Valid annotations are: {CONVERTIBLE_TYPES}" ], ) return converter def deser_output(output): prefix = "[result]" if output.startswith(prefix): output = output[len(prefix) :] try: return json.loads(output) except json.decoder.JSONDecodeError: raise Exception(f"Could not parse JSON: {output}") PK!H?K+lhub_integ-0.1.4.dist-info/entry_points.txtN+I/N.,()J*KI-J-,,JM+)(M+IM׃H#K[&fqqPK!H STT lhub_integ-0.1.4.dist-info/WHEEL 1 0 нR. \I$ơ7.ZON `h6oi14m,b4>4ɛpK>X;baP>PK!H:C#lhub_integ-0.1.4.dist-info/METADATATn@}߯)HmE^dB E$ D6I5{i0gvΥ/3sf̙}+*,?exԕ_dR9\$?onfµ^IAP Paz f-_Qʹuͅnص,y^ڔ1N71xc/).tfpY`^K6{:yt?f5V.%RΞLE}eO ^{㦑j\SȐtTPݱm,nl@l^kpt)el۰X4ҵEvΠ464x-p75#6Y#hHB+*,i Ns0c(6]_KE :n(;aO׾~Aqo4GB㭃peV^~XWo$bnV!E׾1 }IU$&tZ5zo'Fo+J$9ovʥn>LJ g$5^Oo51zS/Ρ tGQ:(*zA#( %JTBeO3VnKIz qNfx:XxUR*كޒxùQjroPK!H<`g!lhub_integ-0.1.4.dist-info/RECORD}ɲ@}nY0hTD7C344 <}Lns}^oA.9q[NwR5F.lD gaJ(`/0!"բ<Ƞm%zWfR0!l{ˋŊKɂ078$ {A78De?_Kgl\#4T&ʼnc& $)_U8+=~Zud 6viBq,֜A_$LB܄ɰ\c$5lTYua2jmC\܊ܰuzO\]H?XPXu<_4Nw~H'ެofvWu.}aUF=lb@6)/}yS_IQͨ&Woǹ nUE~^XB !|$EGkA ︹=|tMdH5*ɇ :JW턡&$~y 1Yjp[vd_H8[Dn3AƋ@88ʧ|F^Z{9m|F[T8+)/PK!VH??lhub_integ/__init__.pyPK! wd]LL!slhub_integ/bundle_requirements.pyPK!W0  lhub_integ/decorators.pyPK!<77Llhub_integ/env.pyPK!~mlhub_integ/generate_metadata.pyPK!gyIL L .lhub_integ/params.pyPK!U``.:lhub_integ/shim_exec.pyPK!$33Blhub_integ/shim_validate.pyPK!sZCC/Qlhub_integ/util.pyPK!H?K+alhub_integ-0.1.4.dist-info/entry_points.txtPK!H STT *blhub_integ-0.1.4.dist-info/WHEELPK!H:C#blhub_integ-0.1.4.dist-info/METADATAPK!H<`g!@flhub_integ-0.1.4.dist-info/RECORDPK h