PK!Copype/__init__.pyfrom . import app PK!$pype/__main__.pyfrom . import cli cli.cli() PK!-pype/_version.py__version__ = "0.0.61" PK!~A pype/app.py#!/usr/bin/env python """Command line pipes in python.""" from __future__ import generator_stop import collections import importlib import io import os import sys import textwrap import token import tokenize import itertools import re import contextlib import typing import types import functools import pathlib from typing import Callable from typing import Awaitable from typing import AsyncIterable from typing import AsyncIterator from typing import Optional from typing import List import attr import parso import click import click_default_group import toolz import trio import async_generator import async_exit_stack try: import asks except ImportError: pass else: asks.init("trio") from . import _version from . import config from . import utils from . import asynch from . import interpret async def program_runner(pairs, items, max_concurrent): async with async_exit_stack.AsyncExitStack() as stack: for how, function in pairs: if how == "map": items = await stack.enter_async_context( asynch.async_map(function, items, max_concurrent) ) elif how == "filter": items = await stack.enter_async_context( asynch.async_filter(function, items, max_concurrent) ) elif how == "apply": items = asynch.AsyncIterableWrapper( [await function([x async for x in items])] ) elif how == "eval": items = asynch.AsyncIterableWrapper([await function(None)]) elif how == "stack": items = asynch.AsyncIterableWrapper( [await function("".join([x + "\n" async for x in items]))] ) else: raise NotImplementedError(how) return stack.pop_all(), items async def async_main( pairs, max_concurrent=config.DEFAULTS["max_concurrent"], exec_before=config.DEFAULTS["exec_before"], autocall=config.DEFAULTS["autocall"], ): stream = trio._unix_pipes.PipeReceiveStream(os.dup(0)) receiver = asynch.TerminatedFrameReceiver(stream, b"\n") result = (item.decode() async for item in receiver) global_namespace = interpret.build_global_namespace(exec_before) pairs = [ (how, interpret.build_function(what, global_namespace, autocall)) for how, what in pairs ] stack, items = await program_runner(pairs, result, max_concurrent) async with stack: async for item in items: print(item) def main(pairs, **kwargs): trio.run(functools.partial(async_main, pairs, **kwargs)) PK!~M pype/asynch.py#!/usr/bin/env python """Command line pipes in python.""" from __future__ import generator_stop import collections import importlib import io import os import sys import textwrap import token import tokenize import itertools import re import contextlib import typing import types import functools import pathlib from typing import Callable from typing import Awaitable from typing import AsyncIterable from typing import AsyncIterator from typing import Optional from typing import List import attr import parso import click import click_default_group import toolz import trio import async_generator import async_exit_stack try: import asks except ImportError: pass else: asks.init("trio") from . import _version from . import config from . import utils T = typing.TypeVar("T") U = typing.TypeVar("U") _PYPE_VALUE = "_PYPE_VALUE" BUFSIZE = 2 ** 14 counter = itertools.count() _RECEIVE_SIZE = 4096 # pretty arbitrary class TerminatedFrameReceiver: """Parse frames out of a Trio stream, where each frame is terminated by a fixed byte sequence. For example, you can parse newline-terminated lines by setting the terminator to b"\n". This uses some tricks to protect against denial of service attacks: - It puts a limit on the maximum frame size, to avoid memory overflow; you might want to adjust the limit for your situation. - It uses some algorithmic trickiness to avoid "slow loris" attacks. All algorithms are amortized O(n) in the length of the input. """ def __init__( self, stream: trio.abc.ReceiveStream, terminator: bytes, max_frame_length: int = 16384, ) -> None: self.stream = stream self.terminator = terminator self.max_frame_length = max_frame_length self._buf = bytearray() self._next_find_idx = 0 async def receive(self) -> bytearray: while True: terminator_idx = self._buf.find(self.terminator, self._next_find_idx) if terminator_idx < 0: # no terminator found if len(self._buf) > self.max_frame_length: raise ValueError("frame too long") # next time, start the search where this one left off self._next_find_idx = max(0, len(self._buf) - len(self.terminator) + 1) # add some more data, then loop around more_data = await self.stream.receive_some(_RECEIVE_SIZE) if more_data == b"": if self._buf: raise ValueError("incomplete frame") raise trio.EndOfChannel self._buf += more_data else: # terminator found in buf, so extract the frame frame = self._buf[:terminator_idx] # Update the buffer in place, to take advantage of bytearray's # optimized delete-from-beginning feature. del self._buf[: terminator_idx + len(self.terminator)] # next time, start the search from the beginning self._next_find_idx = 0 return frame def __aiter__(self) -> "TerminatedFrameReceiver": return self async def __anext__(self) -> bytearray: try: return await self.receive() except trio.EndOfChannel: raise StopAsyncIteration class AsyncIterableWrapper: def __init__(self, iterable): self.iterable = iter(iterable) def __aiter__(self): return self async def __anext__(self): try: return next(self.iterable) except StopIteration: raise StopAsyncIteration @async_generator.asynccontextmanager async def async_map( function: Callable[[T], Awaitable[U]], iterable: AsyncIterable[T], max_concurrent ) -> AsyncIterator[AsyncIterable[U]]: send_result, receive_result = trio.open_memory_channel[U](0) limiter = trio.CapacityLimiter(max_concurrent) async def wrapper(prev_done: trio.Event, self_done: trio.Event, item: T) -> None: maybe_coroutine_result = function(item) if isinstance(maybe_coroutine_result, types.CoroutineType): async with limiter: result = await maybe_coroutine_result else: result = maybe_coroutine_result await prev_done.wait() await send_result.send(result) self_done.set() async def consume_input(nursery) -> None: prev_done = trio.Event() prev_done.set() async for item in iterable: self_done = trio.Event() nursery.start_soon(wrapper, prev_done, self_done, item, name=function) prev_done = self_done await prev_done.wait() await send_result.aclose() async with trio.open_nursery() as nursery: nursery.start_soon(consume_input, nursery) yield receive_result nursery.cancel_scope.cancel() @async_generator.asynccontextmanager async def async_filter( function: Callable[[T], Awaitable[T]], iterable: AsyncIterable[T], max_concurrent ) -> AsyncIterator[AsyncIterable[T]]: send_result, receive_result = trio.open_memory_channel[T](0) limiter = trio.CapacityLimiter(max_concurrent) async def wrapper(prev_done: trio.Event, self_done: trio.Event, item: T) -> None: maybe_coroutine_result = function(item) if isinstance(maybe_coroutine_result, types.CoroutineType): async with limiter: result = await maybe_coroutine_result else: result = maybe_coroutine_result await prev_done.wait() if result: await send_result.send(item) self_done.set() async def consume_input(nursery) -> None: prev_done = trio.Event() prev_done.set() async for item in iterable: self_done = trio.Event() nursery.start_soon(wrapper, prev_done, self_done, item, name=function) prev_done = self_done await prev_done.wait() await send_result.aclose() async with trio.open_nursery() as nursery: nursery.start_soon(consume_input, nursery) yield receive_result nursery.cancel_scope.cancel() PK!@ g pype/cli.pyimport os import click from . import config from . import utils from . import _version from . import app config.DEFAULTS.update( config.load_config( dir_path=os.environ.get(f"{utils.NAME}_CONFIG_DIR".upper(), None) ) ) CONTEXT_SETTINGS = {"default_map": config.DEFAULTS} @click.group(chain=True, context_settings=CONTEXT_SETTINGS) @click.option("--max-concurrent", type=int, default=config.DEFAULTS["max_concurrent"]) @click.option( "--exec-before", help="Python source code to be executed before any stage.", default=config.DEFAULTS["exec_before"], ) @click.option( "--autocall/--no-autocall", is_flag=True, default=config.DEFAULTS["autocall"] ) @click.version_option(_version.__version__, prog_name="pype") def cli(**kwargs): pass def make_subcommand(name): @click.command(name) @click.argument("command") def _subcommand(command): return (name, command) return _subcommand subcommand_names = ["map", "apply", "filter", "eval", "stack"] subcommands = [make_subcommand(name) for name in subcommand_names] for subcommand in subcommands: cli.add_command(subcommand) @cli.resultcallback() def cli_main(pairs, **kwargs): app.main(pairs, **kwargs) PK!Z]pype/config.pyimport pathlib import appdirs import toml from . import utils DEFAULTS = {"max_concurrent": 5, "exec_before": None, "autocall": False} def load_config(dir_path=None): if dir_path is None: config_dir = pathlib.Path(appdirs.user_config_dir(utils.NAME)) else: config_dir = pathlib.Path(dir_path) config_path = config_dir / "config.toml" try: with open(config_path) as f: return toml.load(f) except OSError: return {} PK!.?  pype/interpret.py#!/usr/bin/env python """Command line pipes in python.""" from __future__ import generator_stop import collections import importlib import io import os import sys import textwrap import token import tokenize import itertools import re import contextlib import typing import types import functools import pathlib from typing import Callable from typing import Awaitable from typing import AsyncIterable from typing import AsyncIterator from typing import Optional from typing import List import attr import parso import click import click_default_group import toolz import trio import async_generator import async_exit_stack from . import _version from . import config from . import utils def _get_named_module(name): builtins = sys.modules["builtins"] if hasattr(builtins, name): return builtins try: return __import__(name, {}, {}) except ImportError as e: pass raise LookupError(f"Could not find {name}") def _get_autoimport_module(fullname): name_parts = fullname.split(".") try_names = [] for idx in range(len(name_parts)): try_names.insert(0, ".".join(name_parts[: idx + 1])) for name in try_names: try: module = _get_named_module(name) except LookupError: pass else: if module is sys.modules["builtins"]: return {} return {name: module} return {} def find_maybe_module_names(text): # TODO: Use a real parser. return re.findall(r"\b[^\d\W]\w*(?:\.[^\d\W]\w*)+\b", text) def split_pipestring(s, sep="!"): segments = [] tree = parso.parse(s) current_nodes = [] for c in tree.children: if isinstance(c, parso.python.tree.PythonErrorLeaf) and c.value == sep: segments.append(current_nodes) current_nodes = [] else: current_nodes.append(c) segments.append(current_nodes) return ["".join(node.get_code() for node in seg) for seg in segments] def make_autocall(expression): if expression.endswith(")"): return expression return expression + "(x)" def build_source(components, autocall): components = [c.strip() for c in components] if autocall: components = [make_autocall(c) for c in components] indent = " " lines = "".join([f"{indent}x = {c}\n" for c in components]) source = textwrap.dedent( f"""\ async def _pype_runner(x): {lines} return x """ ) return source def build_name_to_module(command): name_to_module = {} components = split_pipestring(command) module_names = {name for c in components for name in find_maybe_module_names(c)} for name in module_names: name_to_module.update(_get_autoimport_module(name)) return name_to_module def build_function(command, global_namespace, autocall): name_to_module = build_name_to_module(command) global_namespace = {**name_to_module, **global_namespace} source = build_source(split_pipestring(command), autocall) local_namespace = {} exec(source, global_namespace, local_namespace) function = local_namespace["_pype_runner"] return function def build_global_namespace(source): if source is None: return {} namespace = {} exec(source, {}, namespace) return namespace PK!; pype/sync.pyimport itertools def run(program, items): for how, what in program: if how == "map": items = map(what, items) elif how == "apply": items = what(items) elif how == "filter": items = filter(what, items) else: raise ValueError return items def main(program, items): result = run(program, items) try: for item in result: print(item) except TypeError: print(result) if __name__ == "__main__": main( program=[ ("map", str.upper), ("map", len), ("apply", lambda it: itertools.chain.from_iterable(itertools.tee(it))), ("map", lambda x: x + 1), ("filter", lambda x: x > 4), ("map", lambda x: x * 10), ], items=["a", "bb", "ccc", "dddd", "eeeee", "ffffff"], ) PK!w pype/utils.pyNAME = "pype" PK!Huxt#%-python_pype-0.0.61.dist-info/entry_points.txtN+I/N.,()*,HV9z@PK!HڽTU"python_pype-0.0.61.dist-info/WHEEL A н#Z;/"d&F[xzw@Zpy3Fv]\fi4WZ^EgM_-]#0(q7PK!Hm%python_pype-0.0.61.dist-info/METADATAWmo6< &4eq4hu鶴KӢ/+lsDb{؏s丶a8X7d_W&8]; #7R܎q8Pdt ]V7o?D*/-(.,WMʤ4&?rJ.OOѳn܎.^ 3C'LCq-a3mq;}|Gk~4U7RNNiswEf;~KZWR.64͡3]yL׌wm)$:ej^y%zOϋ ˆjdal8'$5I`iH"t$T$&*EBHhȻ!)2i4.%w^r c.GҠS!A+J$Y"MC,kh,T=)?>iS7s ))C5"JsACۍZgtE7VP*~i]/ `;*8ؿ?羾TDrɡ hhMjE~dhI}nq@ )4+@NKiHzm紻Td.X]4vz\ *W!=*,پL(8W(Ip&댡m17c$, <ue7!_w$2x}RU*FDt Hw7| 鎖 rR; sC;+ljr zԞbSKϾAbi)q0o0| amPVj&59M繩{@_z_频)wC?z#lܥ^xm&8"Dcq{R7)E *@܍'Y8o71N[J15Ɔ#kgǘUm47v|=w?k@9ְ̌@E $[L]9 'w}gv-lΖlK [ Q*t^^"Y\X0eDrLCZK>dDfF ,_;yO2h!>LI x_̤XE^K[7E2|7;ūxҽEw=,iJi-/0'=#^e }1 E (S>=eq4>:HRb>VEysģ[Fc/tb,\:rGvFPK!H7֍#python_pype-0.0.61.dist-info/RECORDɖ@} )a2 "4 JTB'69f}罇fc@k4}ńY) ԁ| YsgLAfbDb613%w;\\ژ1R0XV4[Ve̘w=lfmIMzINsvizk҉Mr@}1 B/B?w{uTLIƌtIvܔGF?7Y HND#9JDkF/ݥ|1Y_ ǓSu0) D@q?G`$h NmL^ efx8XyX*{X {*y@0 HA$JUD]CŸL8 E1Osx; W~{ZGqanEics8/c_p-W-lͪ#Qým^쵛0ZQ=mRzfxK-op7Ǩٸ_al9Etu0\롌Qc&oxpjbǰ̪D|SE?QlGAŸaJctynyg׸v0m&Y7tWDo?PK!Copype/__init__.pyPK!$@pype/__main__.pyPK!-pype/_version.pyPK!~A pype/app.pyPK!~M  pype/asynch.pyPK!@ g 5$pype/cli.pyPK!Z]')pype/config.pyPK!.?  8+pype/interpret.pyPK!; 8pype/sync.pyPK!w ,<pype/utils.pyPK!Huxt#%-e<python_pype-0.0.61.dist-info/entry_points.txtPK!HڽTU"<python_pype-0.0.61.dist-info/WHEELPK!Hm%g=python_pype-0.0.61.dist-info/METADATAPK!H7֍#tCpython_pype-0.0.61.dist-info/RECORDPKDF