PK!iŲpicard/__init__.py"""Picard combines the idea of Ansible with the execution of Make.""" # I'm only slightly worried about the rebindings of ``file`` and ``rule``... from picard.api import make, sync from picard.context import Context from picard.file import ( FileRecipePostConditionError, FileTarget, file, file_target ) from picard.pattern import pattern from picard.rule import rule from picard.shell import sh from picard.typing import Target PK!HLXpicard/afunctor.py"""Recursive asynchronous fmap for Python.""" import asyncio import itertools import typing as t import typing_extensions as tex @tex.runtime class AsyncFunctor(tex.Protocol): """A user-defined functor with an asynchronous fmap. The sad state of async programming in Python dictates that we have two implementations of everything. """ async def _afmap_(self, function) -> 'AsyncFunctor': # pylint: disable=no-self-use,unused-argument,pointless-statement ... async def _afmap_str(function, string): ss = await asyncio.gather(*map(function, string)) return ''.join(ss) async def _afmap_generator(function, xs): # These type specializations are awaited unconditionally in :func:`afmap`. # If this function itself were a generator, it could not be awaited. # Instead, it must return an awaitable that returns an async generator. # Gross. async def generator(): for x in xs: yield await afmap(function, x) return generator() async def _afmap_iterable(function, iterable): ys = await asyncio.gather(*(afmap(function, x) for x in iterable)) return type(iterable)(ys) async def _afmap_mapping(function, mapping): async def curried(functor): return await afmap(function, functor) values = await asyncio.gather(*map(curried, mapping.values())) kvs = itertools.zip_longest(mapping.keys(), values) return type(mapping)(kvs) # Only ``str`` and ``range`` need to be here because they do not fit nicely # into the protocol branches, but we include the rest of the built-in types # here anyways as an optimization (because known types are checked first). _IMPLEMENTATIONS = { bytes: _afmap_iterable, dict: _afmap_mapping, frozenset: _afmap_iterable, list: _afmap_iterable, range: _afmap_generator, set: _afmap_iterable, str: _afmap_str, tuple: _afmap_iterable, } async def afmap(function: t.Callable, functor): """Recursively apply an asynchronous function within a functor. By "recursive", we mean if a functor is nested, even with different types, e.g. a list of dicts, then :func:`afmap` will recurse into each level and apply :param:`function` at the "leaf" values. Calls of the function will be evaluated concurrently, thus there is no guaranteed order of execution (you want a monad for that). Parameters ---------- function : A function to apply to the values within :param:`functor`. It must be able to handle any type found within :param:`functor`. functor : A functor. Known functors include lists, dicts, and other collections. User-defined functors with an ``_afmap_`` method will be recognized. Returns ------- AsyncFunctor A copy of :param:`functor` with the results of applying :param:`function` to all of its leaf values. """ # pylint: disable=too-many-return-statements # Currying async functions is inconvenient and slow. We don't do it. # 1. If the functor is a known type: impl = _IMPLEMENTATIONS.get(type(functor), None) if impl is not None: return await impl(function, functor) # 2. If it is an AsyncFunctor: if isinstance(functor, AsyncFunctor): return await functor._afmap_(function) # pylint: disable=protected-access # 3. If it is a Mapping: if isinstance(functor, t.Mapping): return await _afmap_mapping(function, functor) # 4. If it is an Iterator: if isinstance(functor, t.Iterator): return _afmap_generator(function, functor) # 5. If it is an Iterable: if isinstance(functor, t.Iterable): return await _afmap_iterable(function, functor) # Assume it is the identity functor. return await function(functor) PK! fM picard/api.py"""The rest of the public API.""" import asyncio import inspect import logging import os import typing as t from picard.afunctor import afmap from picard.argparse import parse_args from picard.context import Context from picard.typing import Target # Targets = Traversable[Target] Targets = t.Any async def sync(target: Targets, context: Context = None): """Swiss-army function to synchronize one or more targets. Parameters ---------- targets : One or more targets. This function will recurse into functors like sequences and mappings. (Remember that mappings are functors over their values, not their keys.) context : An optional context. If ``None`` is passed (the default), an empty one will be created for you. Returns ------- Traversable[Any] The value(s) of the targets in the same functor. """ if context is None: context = Context() async def _sync(value): if isinstance(value, Target): return await value.recipe(context) return value return await afmap(_sync, target) def make( target: Targets, config: t.Mapping[str, t.Any] = None, rules: t.Mapping[str, Target] = None, ): """Parse targets and configuration from the command line.""" logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) if rules is None: stack = inspect.stack() module = inspect.getmodule(stack[0].frame) if module is None: raise NotImplementedError( 'cannot get module of caller; ' 'you must pass the "rules" argument') rules = vars(module) targets, overrides = parse_args() targets_ = ( [target] if not targets else [rules[t] for t in targets] ) config = {} if config is None else dict(config) config.update(os.environ) config.update(overrides) context = Context(config=config) return _run(sync(targets_, context)) def _run(awaitable): """A shim for :func:`asyncio.run` from 3.7+.""" loop = asyncio.get_event_loop() return loop.run_until_complete(awaitable) PK! >> parse_args([]) ((), {}) >>> parse_args(['a']) (('a',), {}) >>> parse_args(['a', 'b']) (('a', 'b'), {}) >>> parse_args(['--name', 'value']) ((), {'name': 'value'}) >>> parse_args(['name=value']) ((), {'name': 'value'}) >>> parse_args(['a', 'b', '--name', 'value']) (('a', 'b'), {'name': 'value'}) >>> parse_args(['a', '--name', 'value', 'b']) (('a', 'b'), {'name': 'value'}) >>> parse_args(['--name', 'value', 'a', 'b']) (('a', 'b'), {'name': 'value'}) """ if argv is None: argv = sys.argv[1:] args: t.List[str] = [] kwargs: t.MutableMapping[str, t.Any] = {} key = None for arg in argv: if arg.startswith('--'): if arg == '--help': print(USAGE) raise SystemExit if key is not None: kwargs[key] = True key = arg[2:] continue match = re.match('^(\\w+)=(.*)$', arg) if match: if key is not None: kwargs[key] = True key = None kwargs[match.group(1)] = match.group(2) continue if key is not None: kwargs[key] = arg key = None continue args.append(arg) if key is not None: kwargs[key] = True return (tuple(args), kwargs) USAGE = f'{sys.argv[0]} [options] [targets]' PK!* picard/aws.py"""Patterns for Amazon Web Services resources. The patterns in this module correspond to AWS resources, e.g. an EC2 instance or an S3 bucket. That is, their post-condition asserts that the resource exists with the given parameters. Each target requires *at least* the parameters necessary to create the resource. These parameters should be enough to identify the resource in a search. """ import boto3 # type: ignore from picard.context import Context from picard.pattern import pattern @pattern() async def security_group(self, context: Context, description: str = ''): """An AWS security group.""" name = self.name # Search for the security group by name. client = boto3.client('ec2') response = client.describe_security_groups(GroupNames=[name]) groups = response['SecurityGroups'] # More than one? Ambiguous. if len(groups) > 1: # TODO: Set the description. raise Exception(f'ambiguous security group name: {name}') # None? We must create it. ec2 = boto3.resource('ec2') if not groups: response = client.create_security_group( GroupName=name, Description=description) return ec2.SecurityGroup(response['GroupId']) # Exactly one? Check for differences, then return it. assert len(groups) == 1 group = groups[0] gid = group['GroupId'] actual = group['Description'] if actual != description: context.log.warning( f'description for security group {name} ' f'(#{gid}) does not match:\n' f'expected: {description}\n' f'actual: {actual}' ) return ec2.SecurityGroup(gid) @pattern() async def key_pair(self, context: Context): """An AWS key pair.""" # pylint: disable=unused-argument name = self.name # Search for the key pair by name. client = boto3.client('ec2') response = client.describe_key_pairs(KeyNames=[name]) key_pairs = response['KeyPairs'] # More than one? Ambiguous. if len(key_pairs) > 1: raise Exception(f'ambiguous key pair name: {name}') # None? We must create it. ec2 = boto3.resource('ec2') if not key_pairs: response = client.create_key_pair(KeyName=name) return ec2.KeyPair(name) # Can we get the key material *after* the key has been created? # Exactly one? Return it. assert len(key_pairs) == 1 return ec2.KeyPair(name) @pattern() async def instance(self, context: Context): # pylint: disable=unused-argument """An AWS instance.""" # Search for instance by name (kept in a tag). # Careful to filter for non-terminated instance. # Use the tag "Name". It shows up in the dashboard on amazon.com. # If more than zero instances with name, count how many have the right # configuration. If too many, stop the excess and the non-matching # instances. If too few, change the non-matching to match, then stop # the rest if we have enough, or start more if not. # If zero instances with name, start them, with tags. PK!4picard/clang.py"""Patterns for compiling C and C++ objects and executables.""" import re import typing as t from picard.file import file, file_target, FileLike, FileTargetLike from picard.shell import sh from picard.typing import Target def object_(source: FileTargetLike) -> Target: """Compile an object file from a source file. Parameters ---------- source : A filename, path, or file target. Returns ------- Target A :ref:`file target ` for the object file compiled from ``source``. """ # TODO: Implement :func:`find_headers`. # headers = [file_target(h) for h in find_headers(filename)] headers: t.Iterable[FileTargetLike] = [] source = file_target(source) @file(re.sub('\\.c$', '.o', source.name), source, *headers) async def target(self, context, source, *headers): # pylint: disable=unused-argument cc = context.config.get('CC', 'cc') cpp_flags = context.config.get('CPPFLAGS', None) c_flags = context.config.get('CFLAGS', None) await sh(cc, cpp_flags, c_flags, '-c', source) return target def objects(*sources): """Return a set of object files mapped from a set of source files.""" return [object_(s) for s in sources] def executable(filename: FileLike, *objects: FileTargetLike) -> Target: """Link an executable from object files. Parameters ---------- filename : The filename or path to where the executable should be built. *objects : A set of filenames, paths, or targets for the object files. Returns ------- Target A :ref:`file target ` for the executable linked from ``objects``. """ @file(filename, *objects) async def target(self, context, *objects): # pylint: disable=unused-argument cc = context.config.get('CC', 'cc') ld_flags = context.config.get('LDFLAGS', None) ld_libs = context.config.get('LDLIBS', None) await sh(cc, ld_flags, '-o', self.name, *objects, ld_libs) return target PK!a)picard/context.py"""Context shared through the dependency graph.""" import logging import typing as t class Context: """A configuration mapping and a logger.""" def __init__( self, config: t.Mapping[str, t.Any] = None, log: logging.Logger = logging.getLogger(), ) -> None: self.config = {} if config is None else config self.log = log PK!-kpicard/file.py"""Make-style file (and directory) targets.""" import os from pathlib import Path import typing as t from picard.context import Context from picard.typing import Target FileLike = t.Union[str, os.PathLike] FileTargetLike = t.Union[Target, FileLike] # Need a way to type *args and **kwargs without ignoring the known parameters. Recipe = t.Callable[ [Target, Context, t.Any], t.Awaitable[t.Union[None, Path]], ] def is_file_like(value): # For now, ``isinstance`` does not play well with ``typing.Union``. # https://stackoverflow.com/a/45959000/618906 return isinstance(value, (str, os.PathLike)) class FileRecipePostConditionError(Exception): """Raised when a file recipe fails to update its target.""" class FileTarget(Target): """A file that must be newer than its prerequisite files.""" def __init__( self, path: Path, recipe: Recipe, *prereqs: FileTargetLike, ) -> None: self.path = path self.prereqs = [file_target(p) for p in prereqs] self._recipe = recipe @property def name(self) -> str: # type: ignore return str(self.path) async def recipe(self, context: Context) -> Path: """Conditionally rebuild this file. The conditions are (1) if this file does not exist or (2) if its prerequisites have changed since it was last touched. """ # TODO: Memoize value. # There is no way around evaluating all of the prerequisites. Either # (1) some have changed but we must feed them all to the recipe or (2) # we must check all of them to make sure none have changed. from picard.api import sync # pylint: disable=cyclic-import prereqs = await sync(self.prereqs) if not await self._is_up_to_date(context, prereqs): context.log.info(f'start: {self.name}') value = await self._recipe(self, context, *prereqs) if value is not None and value != self.path: context.log.warning( f'discarding value returned by {self._recipe}: {value}') if not await self._is_up_to_date(context, prereqs): raise FileRecipePostConditionError(self.name) context.log.info(f'finish: {self.name}') return self.path async def _is_up_to_date( self, context: Context, prereqs: t.Iterable[t.Any] ) -> bool: try: mtime = os.stat(self.name).st_mtime except FileNotFoundError: return False for prereq in prereqs: if not is_file_like(prereq): # Not a filename. context.log.warn( f'skipping non-filename dependency: {prereq}') continue if os.stat(prereq).st_mtime > mtime: # Prerequisite has been modified after target. return False return True def file_target(value: FileTargetLike) -> Target: """Canonicalize a value to a :class:`Target`. If the value is already a :class:`Target`, it is returned as-is. If it is a :class:`str`, it is returned as a :class:`FileTarget`. Parameters ---------- value : A value convertible to :class:`Target`. Returns ------- Target A target. Raises ------ Exception If :param:`value` is not convertible to a target. """ if isinstance(value, Target): return value if is_file_like(value): # Treat ``value`` as a filename. return file(value)() raise Exception(f'not a target: {value}') async def _noop(*args, **kwargs) -> None: # pylint: disable=unused-argument """A function that does nothing.""" def file(target: FileLike, *prereqs: FileTargetLike): """A file that is newer than its prerequisite files.""" # pylint: disable=unused-argument def decorator(recipe: Recipe = _noop): return FileTarget(Path(target), recipe, *prereqs) return decorator PK!j. picard/functor.py"""Recursive synchronous fmap for Python.""" import itertools import typing as t import typing_extensions as tex @tex.runtime class Functor(tex.Protocol): """A user-defined functor with a synchronous fmap.""" def _fmap_(self, function) -> 'Functor': # pylint: disable=no-self-use,unused-argument,pointless-statement ... def _fmap_str(function, string): return ''.join(map(function, string)) def _fmap_generator(function, xs): for x in xs: yield fmap(function, x) def _fmap_iterable(function, iterable): return type(iterable)(fmap(function, x) for x in iterable) def _fmap_mapping(function, mapping): values = _fmap_generator(function, mapping.values()) kvs = itertools.zip_longest(mapping.keys(), values) return type(mapping)(kvs) # Only ``str`` and ``range`` need to be here because they do not fit nicely # into the protocol branches, but we include the rest of the built-in types # here anyways as an optimization (because known types are checked first). _IMPLEMENTATIONS = { bytes: _fmap_iterable, dict: _fmap_mapping, frozenset: _fmap_iterable, list: _fmap_iterable, range: _fmap_generator, set: _fmap_iterable, str: _fmap_str, tuple: _fmap_iterable, } _NO_ARGUMENT = object() def fmap(function: t.Callable, functor=_NO_ARGUMENT): """Recursively apply a function within a functor. By "recursive", we mean if a functor is nested, even with different types, e.g. a list of dicts, then :func:`fmap` will recurse into each level and apply :param:`function` at the "leaf" values. Parameters ---------- function : A function to apply to the values within :param:`functor`. It must be able to handle any type found within :param:`functor`. functor : A functor. Known functors include lists, dicts, and other collections. User-defined functors with an ``_fmap_`` method will be recognized. Returns ------- Functor A copy of :param:`functor` with the results of applying :param:`function` to all of its leaf values. """ # pylint: disable=too-many-return-statements # Currying: if functor is _NO_ARGUMENT: return lambda functor: fmap(function, functor) # 1. If the functor is a known type: impl = _IMPLEMENTATIONS.get(type(functor), None) if impl is not None: return impl(function, functor) # 2. If it is a Functor: if isinstance(functor, Functor): return functor._fmap_(function) # pylint: disable=protected-access # 3. If it is a Mapping: if isinstance(functor, t.Mapping): return _fmap_mapping(function, functor) # 4. If it is an Iterator: if isinstance(functor, t.Iterator): return _fmap_generator(function, functor) # 5. If it is an Iterable: if isinstance(functor, t.Iterable): return _fmap_iterable(function, functor) # Assume it is the identity functor. return function(functor) PK!?VVpicard/pattern.py"""Templates for rules, a la Make pattern rules.""" import functools import typing as t from picard.context import Context from picard.typing import Target Recipe = t.Any class PatternTarget(Target): """A template for rules, a.k.a. :class:`Target`s.""" def __init__(self, name: str, recipe: Recipe, *args, **kwargs) -> None: self.name = name self.prereqs = (args, kwargs) self._recipe = recipe if recipe.__doc__: self.__doc__ = recipe.__doc__ async def recipe(self, context: Context) -> t.Any: # TODO: Memoize value. from picard.api import sync # pylint: disable=cyclic-import args, kwargs = await sync(self.prereqs) context.log.info(f'start: {self.name}') value = await self._recipe(self, context, *args, **kwargs) context.log.info(f'finish: {self.name}') return value def pattern() -> t.Callable[[Recipe], t.Callable[..., Target]]: """Turn a recipe function into a target constructor. The constructor's parameters are the prerequisites, which will be passed, evaluated, to the recipe. Example ------- .. code-block:: python import picard @picard.pattern() async def object_file(target, context, source): await picard.sh('gcc', '-c', source, '-o', target.name) hello_o = object_file('hello.o', 'hello.c') example_o = object_file('example.o', source='example.c') """ def decorator(recipe): @_wraps(recipe) def constructor(*args, **kwargs): return PatternTarget(recipe.__name__, recipe, *args, **kwargs) return constructor return decorator def _wraps(wrapped): """Like :func:`functools.wraps`, but do not set ``__wrapped__``. Because we change the type of the function, we do not want to set the ``__wrapped__`` attribute, which would give Sphinx autodoc the wrong impression. """ def decorator(wrapper): for attr in functools.WRAPPER_ASSIGNMENTS: setattr(wrapper, attr, getattr(wrapped, attr)) return wrapper return decorator PK!u\>kkpicard/rule.py"""Turn a named function into a :class:`Target`.""" import typing as t from picard.context import Context from picard.pattern import PatternTarget from picard.typing import Target # Need a way to type *args and **kwargs without ignoring the known parameters. Recipe = t.Callable[[Context, t.Any], t.Awaitable[t.Any]] def rule(*args, **kwargs) -> t.Callable[[Recipe], Target]: """Turn a recipe function into a target. The parameters are the prerequisites, which will be passed, evaluated, to the recipe. Example ------- .. code-block:: python from pathlib import Path import picard @picard.rule() async def gitdir(context): path = Path('.git') if not path.is_dir(): picard.sh('git', 'init', '.') return path """ # pylint: disable=unused-argument def decorator(recipe: Recipe): async def selfless_recipe(self, context, *args, **kwargs): return await recipe(context, *args, **kwargs) return PatternTarget(recipe.__name__, selfless_recipe, *args, **kwargs) return decorator PK!Tpicard/shell.py"""A shortcut for subprocesses.""" import asyncio async def sh(*args, **kwargs): """Echo and execute a command. Parameters ---------- *args : Command line arguments. ``None``s will be removed and the rest passed through ``str`` before execution. """ args = tuple(str(a) for a in args if a is not None) print(' '.join(args)) p = await asyncio.create_subprocess_exec(*args, **kwargs) await p.wait() PK!ivpicard/typing.py"""Types for mypy.""" import typing as t import typing_extensions as tex from picard.context import Context # mypy does not yet support nested types, so we must use :class:`t.Any`. # Prerequisites = t.Union['Target', t.Iterable['Prerequisites']] Prerequisites = t.Any """An arbitrary traversable structure for target prerequisites. The structure can be a single target, a (possibly empty) collection of targets, or arbitrarily nested collections of targets. We just need a way to walk the structure and pull out the targets buried within, which we have with :func:`picard.functor.fmap`. """ @tex.runtime class Target(tex.Protocol): """A protocol_ (i.e. structural type) for targets. .. protocol_: https://www.python.org/dev/peps/pep-0544/ """ name: str prereqs: Prerequisites async def recipe(self, context: Context) -> t.Any: # pylint: disable=unused-argument,pointless-statement ... PK!HnHTUpicard.py-0.1.0.dist-info/WHEEL A н#Z;/"d&F[xzw@Zpy3Fv]\fi4WZ^EgM_-]#0(q7PK!Hi"picard.py-0.1.0.dist-info/METADATA[O0 +=1i "[@#2$)]:9sl)`[r^JP&"CC j+d݊X&2׍1Zs|"j,.˻LtN#6aesĈkV݃SdnF;Ѣe/瘦#9L9KMwl]0êl8(?m㍉]pޓP l#ؚL2unHS1O_Kk^C_PK!H picard.py-0.1.0.dist-info/RECORD}ɎH}} dA0E,zfp fNK]8WgQ@n?}?+?jKs2xC+{Pi8-ix|ϺuKO Wuy.=K+h6kmN8! $OL7&HN)NhM[MeV_a̞%fT۽Zs{4RHWS{ 7}Hzo=G[N&|/+L7e yPuȭ&? s|T͐L$`cVTe<&'hӤJtTwjӬ㸝kIF*xODX bN=]w+uẟhl-J:Lp 2#c[WPIޞB#'!$.wP"uP֬,=I.e ~Kuu1)$aټl**JH(dYPHzk_H^U+a)v"' Xi:b׃|6rKśC+:T $[xqϺ"򱉙f~a(^L7oӇQ`;.)~ʶDp^.*8_|-k;:+k=LȨw6MH(d"nxD0g{z8TS=c6O i4OȘ&<-"jCC: W7lR(ǿPK!iŲpicard/__init__.pyPK!HLXpicard/afunctor.pyPK! fM picard/api.pyPK! kk'[picard/rule.pyPK!T_picard/shell.pyPK!ivapicard/typing.pyPK!HnHTUepicard.py-0.1.0.dist-info/WHEELPK!Hi"fpicard.py-0.1.0.dist-info/METADATAPK!H Xgpicard.py-0.1.0.dist-info/RECORDPKoj