PK!RG))lhub_integ/__init__.pyfrom lhub_integ.decorators import action PK!á!lhub_integ/bundle_requirements.pyimport os import tempfile import subprocess """ Bundle all the dependencies in requirements.txt into a tar.bz2 by downloading and building wheels for all the dependencies in requirements.txt See https://pip.pypa.io/en/stable/user_guide/#installation-bundles for details """ def main(): with tempfile.TemporaryDirectory() as tmpdir: if not os.path.exists('requirements.txt'): print("Run in a directory with requirements.txt") exit(1) subprocess.run(['pip', 'wheel', '-r', 'requirements.txt', '--wheel-dir', tmpdir]) subprocess.run(['cp', 'requirements.txt', tmpdir]) cwd = os.getcwd() os.chdir(tmpdir) subprocess.run(['tar', '-cjvf', f'{cwd}/bundle.tar.bz2', *os.listdir(tmpdir)]) os.chdir(cwd) PK!g4EElhub_integ/decorators.pyfrom lhub_integ import util class action: actions = {} def __init__(self, f): # See https://www.python.org/dev/peps/pep-3155/ if f.__name__ != f.__qualname__: util.exit_with_instantiation_errors( code="invalid_action", errors=["actions must be top level functions"] ) entrypoint = f"{f.__module__}.{f.__name__}" self.actions[entrypoint] = f self.f = f def __call__(self, *args, **kwargs): self.f(*args, **kwargs) @classmethod def all(cls): return cls.actions PK!oʷlhub_integ/env.py""" Environment Variable Wrappers By wrapping environment variables in our classes, we can find what environment variables the integration uses at parse time. User-code should only use `EnvVar` -- `PrefixedEnvVar` subclasses are for internal use only. """ import os from lhub_integ.util import exit_with_instantiation_errors from abc import ABCMeta, abstractmethod class __EnvVar: """ __EnvVar should not be instantiated by Integrations! Use ConnectionParam or ActionParam instead """ def __init__(self, name, default=None, optional=False): """ :param name: Name of the environment variable """ self.name = name self.default = default self.optional = optional or default is not None def get_not_empty(self): env = os.environ.get(self.name) if env and len(env) > 0: return env else: return None def valid(self): return self.get_not_empty() or self.optional def read(self): value = self.get_not_empty() or self.default if value is None and not self.optional: exit_with_instantiation_errors( 1, [{"message": f"Environment variable {self.name} must be defined"}] ) else: return value class PrefixedEnvVar(__EnvVar, metaclass=ABCMeta): """ PrefixedEnvVar wraps Environment variables use for internal signalling to lhub_integ Specifically: Internal environment variables are prefixed with `__` Environment variables that map input columns to parameter names are prefixed with `___` """ def __init__(self, name): super().__init__(self.prefix() + name) @classmethod @abstractmethod def prefix(cls): pass class InternalEnvVar(PrefixedEnvVar): @classmethod def prefix(cls): return "__" class MappedColumnEnvVar(PrefixedEnvVar): @classmethod def prefix(cls): return "___" PK!_L=lhub_integ/generate_metadata.py""" `generate_metadata` will load a Python module and generate a metadata JSON that the module system can use to build an integration descriptor. Usage: python -m lhub_integ.generate_metadata """ import inspect import json from dataclasses import dataclass from typing import List, Optional from dataclasses_json import dataclass_json from docstring_parser import parse from lhub_integ import util, action from lhub_integ.params import ConnectionParam, ActionParam, DataType, InputType from lhub_integ.util import print_result @dataclass @dataclass_json class Param: name: str data_type: str input_type: str optional: bool = False description: Optional[str] = None default: Optional[str] = None options: Optional[List[str]] = None @dataclass @dataclass_json class Action: entrypoint: str args: List[Param] params: List[Param] description: str errors: List[str] @dataclass @dataclass_json class IntegrationMetadata: connection_params: List[Param] integration_description: str actions: List[Action] errors: List[str] ok: bool def generate_metadata() -> IntegrationMetadata: # Side-effectful action of importing every file in the current working directory. # This will populate the internal class-storage for `action`, `ConnectionParam` and `ActionParam` toplevel_errors = [] parse_failure = False errors, docstrings = util.import_workdir() for error in errors: parse_failure = True if isinstance(error, SyntaxError): toplevel_errors.append(f"Failed to import module: {error}") elif isinstance(error, ImportError): toplevel_errors.append( f"Failed to import module (did you run bundle-integrations?): {error}" ) else: toplevel_errors.append( f"Failed to import module (Unexpected error): {error}" ) actions = [] for entrypoint, processing_function in action.all().items(): # Parse the docstring docs = processing_function.__doc__ parsed = parse(docs) errors = [] # Any docstring on the function itself will be used as the description for the integration if parsed.short_description or parsed.long_description: function_description = parsed.short_description or parsed.long_description else: function_description = None # Build a map of the actual function arguments to compare with the docstrings args, varargs, kwargs = inspect.getargs(processing_function.__code__) if varargs: errors.append("Varargs are not supported") if kwargs: errors.append("Kwargs are not supported") arg_map = {arg: {"name": arg} for arg in args} # Augment arg_map with information from the docstrings for param in parsed.params: if param.arg_name not in arg_map: errors.append( f"Docstring referenced {param.arg_name} but there were no matching arguments" ) else: if param.description is not None: arg_map[param.arg_name]["description"] = param.description if param.type_name is not None: arg_map[param.arg_name]["tpe"] = param.type_name params = [ Param( name=e.name, description=e.description, default=e.default, optional=e.optional, data_type=e.data_type.value, options=e.options, input_type=e.input_type.value, ) for e in ActionParam.for_action(processing_function.__name__) ] args = [arg_map[arg] for arg in args] args = [ Param( name=arg["name"], description=arg.get("description"), data_type=DataType.COLUMN.value, input_type=InputType.COLUMN_SELECT.value, ) for arg in args ] actions.append( Action( entrypoint=entrypoint, args=args, params=params, description=function_description, errors=errors, ) ) connection_params = [ Param( name=e.name, description=e.description, default=e.default, optional=e.optional, data_type=e.data_type.value, options=e.options, input_type=e.input_type.value, ) for e in ConnectionParam.all() ] if not actions and not parse_failure: toplevel_errors.append("No actions found. Did you forget to use @action?") if docstrings: integration_description = docstrings[0].strip() elif len(actions) == 1: integration_description = actions[0].description else: integration_description = "No description provided" metadata = IntegrationMetadata( connection_params=connection_params, integration_description=integration_description, actions=actions, errors=toplevel_errors, ok=len(toplevel_errors) == 0, ) return metadata if __name__ == "__main__": import traceback try: print_result(generate_metadata().to_json()) except Exception: print_result(json.dumps(dict(errors=[traceback.format_exc()]))) exit(1) PK!^{ lhub_integ/params.pyfrom abc import ABCMeta from collections import defaultdict from lhub_integ.env import __EnvVar from enum import Enum # pulled from forms.model.DataType class DataType(Enum): STRING = "string" COLUMN = "column" NUMBER = "number" # no node datatype because integrations can only pull from one node # pulled from forms.model.InputType class InputType(Enum): TEXT = "text" TEXT_AREA = "textarea" EMAIL = "email" PASSWORD = "password" SELECT = "select" COLUMN_SELECT = "columnSelect" class __Param(__EnvVar, metaclass=ABCMeta): def __init__( self, name, description=None, default=None, optional=False, options=None, data_type=DataType.STRING, input_type=InputType.TEXT, ): super().__init__(name, default, optional) self.description = description self.default = default self.data_type = data_type self.options = options if data_type == DataType.COLUMN: self.input_type = InputType.InputType.COLUMN_SELECT elif options is not None and len(options) > 1: self.input_type = InputType.SELECT else: self.input_type = input_type """ We take most of the param information from our Form.Input case class. We don't enable a dependsOn field because if the dataType is a column then it will auto depends on its parent. """ class ConnectionParam(__Param, metaclass=ABCMeta): """ ConnectionParam provides a parameter specified by the connection Example usage: API_KEY = ConnectionParam('api_key') def process_row(url): requests.get(url, params={api_key: API_KEY.get()}) """ _all = set() def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._all.add(self) @classmethod def all(cls): return sorted(cls._all, key=lambda var: var.name) class ActionParam(__Param, metaclass=ABCMeta): """ ActionParam provides a parameter specified by the action Example usage: API_KEY = ConnectionParam('api_key', action='process_row') import requests def process_row(url): requests.get(url, params={api_key: API_KEY.get()}) """ action_map = defaultdict(set) def __init__(self, *args, action: str, **kwargs): super().__init__(*args, **kwargs) self.action_map[action].add(self) @classmethod def for_action(cls, action): return sorted(cls.action_map[action], key=lambda var: var.name) PK!շ]lhub_integ/shim_exec.py""" Wrap a custom Python script to enable it to work with the integration machinery Loads ENTRYPOINT from `__ENTRYPOINT` and the column-argument mapping from `___XXXXX` Usage: This module is intended to be invoked by `PythonEnvironment`. See `test_shim.py` for a usage example python -m lhub_integ.shim_exec -e """ import fileinput import json import sys import traceback import click from lhub_integ import util, action from lhub_integ.env import MappedColumnEnvVar def run_integration(entrypoint_fn): type_mapper = util.get_input_converter(entrypoint_fn) # Build a mapping from the columns in the input data to the arguments in our integration argument_column_mapping = { input_argument: MappedColumnEnvVar(input_argument).read() for input_argument in type_mapper } for row in sys.stdin.readlines(): as_dict = json.loads(row)["row"] lhub_id = as_dict.get(util.LHUB_ID) try: # Load the arguments from the input arguments = { arg: type_mapper[arg](as_dict[column]) for arg, column in argument_column_mapping.items() } # Run the integration result = entrypoint_fn(**arguments) # Print the results if result: if isinstance(result, list): util.print_each_result_in_list(result, lhub_id) else: util.print_result(json.dumps(result), lhub_id) except Exception: util.print_error(traceback.format_exc(), data=as_dict) @click.command() @click.option("--entrypoint", "-e", required=True) def main(entrypoint): util.import_workdir() entrypoint_fn = action.all().get(entrypoint) assert entrypoint_fn is not None run_integration(entrypoint_fn) if __name__ == "__main__": main() PK!N lhub_integ/shim_validate.py""" Validate that all the required environment variables have been set to execute this integration. If this fails, it indicates a bug in the PythonEnvironment machinery. Currently, this doesn't offer the ability for the user to run custom validations but we could add it here. eg: We could look for a method like `module.entrypoint_validate()` and run it. Currently we don't """ import inspect import click from lhub_integ import action, util from lhub_integ.env import MappedColumnEnvVar from lhub_integ.params import ConnectionParam, ActionParam from lhub_integ.util import ( exit_with_instantiation_errors, print_successful_validation_result, ) @click.command() @click.option("--entrypoint", "-e", required=True) def main(entrypoint): try: validation_errors = validate_entrypoint(entrypoint) except Exception as ex: validation_errors = [{"message": f"Unexpected exception: {ex}"}] if validation_errors: exit_with_instantiation_errors( 1, validation_errors, message="Integration validation failed." ) else: print_successful_validation_result() def validate_entrypoint(entrypoint): module_name = ".".join(entrypoint.split(".")[:-1]) function_name = entrypoint.split(".")[-1] if module_name == "" or function_name == "": return [{"message": "Bad entrypoint format. `Expected filename.functionname`"}] # Try to import the world and find the entrypoint try: util.import_workdir() except ImportError as ex: return [{"message": f"Can't import {module_name}: {ex}"}] method = action.all().get(entrypoint) if method is None: return [ { "message": f"No matching action found. Is your action annotated with @action?" } ] errors = [] # Read the arguments and environment variables we expect and make sure they've all been defined args, _, _ = inspect.getargs(method.__code__) for arg in args: if not MappedColumnEnvVar(arg).valid(): errors.append({"message": "Column mapping must be defined", "inputId": arg}) env_vars = list(ConnectionParam.all()) + list(ActionParam.for_action(entrypoint)) for var in env_vars: if not var.valid(): errors.append( {"message": "Environment variable must be defined", "inputId": var.name} ) return errors if __name__ == "__main__": main() PK!} lhub_integ/util.pyfrom __future__ import print_function import importlib import inspect import json import sys import fileinput from pathlib import Path from typing import Dict, Callable, Any, List, Tuple LHUB_ID = "lhub_id" def read_all_data(): """ If an argument is provided, fileinput.input() will iterate over the lines in that file. Otherwise, it will read from stdin. See https://docs.python.org/3/library/fileinput.html :return: """ return fileinput.input() def exit_with_instantiation_errors( code, errors, message="Integration validation failed." ): error_wrapper = {"errors": errors, "message": message} print(f"[result] {json.dumps(error_wrapper)}", file=sys.stderr) sys.exit(code) def print_error(message: str, data=None): error = {"has_error": True, "error": message, "data": data} print_result(json.dumps(error)) def print_result(msg, original_lhub_id=None): if original_lhub_id is not None: print_correlated_result(msg, original_lhub_id) else: print(f"[result] {msg}") def print_correlated_result(msg, original_lhub_id): meta_data_dict = {"original_lhub_id": original_lhub_id} meta_json = json.dumps(meta_data_dict) print(f"[result][meta]{meta_json}[/meta] {msg}") def print_successful_validation_result(): print_result("{}") def print_each_result_in_list(results, original_lhub_id=None): if not results: return print_result( json.dumps({"noResults": "no results returned"}), original_lhub_id=original_lhub_id, ) for result in results: print_result(json.dumps(result), original_lhub_id=original_lhub_id) def import_workdir() -> Tuple[List[Exception], List[str]]: """ Attempt to import all the files in the working directory :return: A list of errors """ errors = [] docstrings = [] for file in Path(".").iterdir(): if file.suffix == ".py": as_module = file.name[: -len(".py")] try: module = importlib.import_module(as_module) doc = module.__doc__ if doc: docstrings.append(doc) except Exception as ex: errors.append(ex) return errors, docstrings def get_entrypoint_fn(entrypoint): module_name = ".".join(entrypoint.split(".")[:-1]) function_name = entrypoint.split(".")[-1] module = importlib.import_module(module_name) return getattr(module, function_name) def get_module_docstring(entrypoint): module_name = ".".join(entrypoint.split(".")[:-1]) module = importlib.import_module(module_name) if module.__doc__: return module.__doc__.strip() return None CONVERTIBLE_TYPES = [int, str, float, bool] def get_input_converter(entrypoint_fn) -> Dict[str, Callable[[str], Any]]: """ Build the input_conversion map to allow promotion from String to to int, float, and bool :param entrypoint_fn: :return: Dict from the name of the function arguments to a converter function. """ sig = inspect.signature(entrypoint_fn) converter = {} for param in sig.parameters: annot = sig.parameters[param].annotation # The annotation is the Python class -- in these simple cases we can just call # the class constructor if annot in CONVERTIBLE_TYPES: converter[param] = lambda inp: annot(inp) elif annot == inspect.Parameter.empty: converter[param] = lambda x: x else: exit_with_instantiation_errors( 1, [ f"Unsupported type annotation: {annot}. Valid annotations are: {CONVERTIBLE_TYPES}" ], ) return converter def deser_output(output): prefix = "[result]" if output.startswith(prefix): output = output[len(prefix) :] try: return json.loads(output) except json.decoder.JSONDecodeError: raise Exception(f"Could not parse JSON: {output}") PK!H?K+lhub_integ-0.1.1.dist-info/entry_points.txtN+I/N.,()J*KI-J-,,JM+)(M+IM׃H#K[&fqqPK!H STT lhub_integ-0.1.1.dist-info/WHEEL 1 0 нR. \I$ơ7.ZON `h6oi14m,b4>4ɛpK>X;baP>PK!H ym#lhub_integ-0.1.1.dist-info/METADATAVn7}W "Qʗ '!; l^KiYqI+ۢ!YrΙ ['಼ i+mXBy&ϟCQWh9<` R|;^PU|K; WO6YSRxj[SpK sr5X Q!+dO ,w}UKX =c+oo,s<>jlZU-G)~;|L uyq#bϹf[LXm tNo3rDF&/AvgבLsO#y;+c0iҟ:t1n&MP7}ܨŕ9P 8iF;%;%h+RMKvb(d»EiAi]pqRwұks)BlcLxG{ \X&iHǰ8̢b /K:_rFKe&ɚੑ`g񣎦񝎯&,ϙ=ܳCӵ%VbD)<DWKEथv"oysOLSJ4q!AbyJ~o,+!IUpC @PK!H} !lhub_integ-0.1.1.dist-info/RECORD}˒0}? Y "(wP@E ͧg5XSՕEvdq 8g92M7iu'ipVmuMkPFG[2ָjfvHn"\T,L`[WttlpI@QEѢzɬb9I)9i.H^k Ť@V7lH݇ B`Z6No"L+vsL=QoεѬ$]f*GzkA^Ӷ|÷kVH ?2XV Kr糑t< `x)I5vGMuem:a9*Iy%xb8\GtHg,<˷;;U976Ã[Ɩ&ل{\i0UkJⱸ=p)X~ɽJ:KHXM-Śy>o/% :=PbCG9X)pUK,%7D=}-UiO^BeDk'y%o ~ "nYX;4ex^~?PK!RG))lhub_integ/__init__.pyPK!á!]lhub_integ/bundle_requirements.pyPK!g4EElhub_integ/decorators.pyPK!oʷ(lhub_integ/env.pyPK!_L=lhub_integ/generate_metadata.pyPK!^{ #lhub_integ/params.pyPK!շ].lhub_integ/shim_exec.pyPK!N 5lhub_integ/shim_validate.pyPK!} ?lhub_integ/util.pyPK!H?K+zOlhub_integ-0.1.1.dist-info/entry_points.txtPK!H STT Plhub_integ-0.1.1.dist-info/WHEELPK!H ym#Plhub_integ-0.1.1.dist-info/METADATAPK!H} !BUlhub_integ-0.1.1.dist-info/RECORDPK W