PK!u)pillars/__init__.pyfrom . import ( # noQa: F401 engines, exceptions, middlewares, request, sites, transports, utils, ) from .app import Application, SubApp # noQa: F401 from .request import Response # noQa: F401 PK!?Fpillars/app.pyimport asyncio import collections import logging import signal from dataclasses import dataclass from typing import Any, Callable, List, Optional from aiohttp import signals from aiohttp.web_runner import BaseRunner, BaseSite import setproctitle from . import exceptions LOG = logging.getLogger(__name__) @dataclass class SubApp: name: str app: Any sites: List[Callable[[BaseRunner], BaseSite]] runner: BaseRunner async def status(self) -> bool: result = list() if hasattr(self.app, "status"): result.append(await self.app.status()) for site in self.sites: if hasattr(site, "status"): result.append(await site.status()) # type: ignore return all(result) class Application(collections.MutableMapping): def __init__( self, name: str, state: Optional[collections.MutableMapping] = None ) -> None: if name: setproctitle.setproctitle(name) self._state = state or dict() self._frozen = False self._subapps: dict = dict() self._on_startup: signals.Signal = signals.Signal(self) self._on_cleanup: signals.Signal = signals.Signal(self) self._on_shutdown: signals.Signal = signals.Signal(self) self["name"] = name @property def name(self) -> str: return self["name"] @property def subapps(self) -> dict: return self._subapps # Start Application def run(self) -> None: LOG.debug("Starting Application") loop = asyncio.get_event_loop() try: loop.add_signal_handler(signal.SIGINT, self._raise_exit) loop.add_signal_handler(signal.SIGTERM, self._raise_exit) except NotImplementedError: LOG.debug("Loop signal not supported") loop.run_until_complete(self.start()) try: loop.run_forever() except (KeyboardInterrupt, exceptions.GracefulExit): pass except Exception: raise finally: loop.run_until_complete(self.stop()) def listen( self, *, app: Any, sites: List[Callable[[BaseRunner], BaseSite]], runner: BaseRunner, name: Optional[str] = None, ): if not name: name = f"{app.__module__}.{app.__class__.__qualname__}" subapp = SubApp(name=name, app=app, sites=sites, runner=runner) if self._frozen: raise RuntimeError("Cannot add subapp to frozen application") elif not isinstance(subapp, SubApp): raise TypeError(f"SubApp {subapp} is not of type {type(SubApp)}") self._subapps[subapp.name] = subapp def _freeze(self) -> None: if self._frozen: return self.on_startup.freeze() self.on_shutdown.freeze() self.on_cleanup.freeze() self._frozen = True async def start(self) -> None: self._freeze() await self.on_startup.send(self) await asyncio.gather( *(subapp.runner.setup() for subapp in self._subapps.values()) ) for subapp in self._subapps.values(): subapp.sites = [site(subapp.runner) for site in subapp.sites] subapp.app._container = self if isinstance(subapp.app, collections.MutableMapping): subapp.app.state = collections.ChainMap( # type: ignore subapp.app, self._state ) else: subapp.app.state = collections.ChainMap({}, self._state) await asyncio.gather( *( site.start() for subapp in self._subapps.values() for site in subapp.sites ) ) async def stop(self) -> None: LOG.debug("Stopping application") coros = (subapp.runner.shutdown() for subapp in self._subapps.values()) await asyncio.gather(*coros) await self.on_shutdown.send(self) coros = (subapp.runner.cleanup() for subapp in self._subapps.values()) await asyncio.gather(*coros) await self.on_cleanup.send(self) async def status(self) -> dict: result = dict() for subapp in self._subapps.values(): if hasattr(subapp, "status"): result[subapp.name] = await subapp.status() return result ########### # Signals # ########### @property def on_startup(self) -> signals.Signal: return self._on_startup @property def on_shutdown(self) -> signals.Signal: return self._on_shutdown @property def on_cleanup(self) -> signals.Signal: return self._on_cleanup def _raise_exit(self) -> None: raise exceptions.GracefulExit() ###################### # MutableMapping API # ###################### def __eq__(self, other): return self is other def __getitem__(self, key): return self._state[key] def __setitem__(self, key, value): self._state[key] = value def __delitem__(self, key): del self._state[key] def __len__(self): return len(self._state) def __iter__(self): return iter(self._state) PK!+ Tsspillars/base.pyimport asyncio import signal import socket from abc import ABC, abstractmethod from yarl import URL class BaseRunner(ABC): __slots__ = ("_handle_signals", "_kwargs", "_server", "_sites") def __init__(self, *, handle_signals=False, **kwargs): self._handle_signals = handle_signals self._kwargs = kwargs self._server = None self._sites = [] @property def server(self): return self._server @property def addresses(self): return [ sock.getsockname() for site in self._sites for sock in site._server.sockets ] @property def sites(self): return set(self._sites) async def setup(self): self._server = await self._make_server() @abstractmethod async def shutdown(self): pass # pragma: no cover async def cleanup(self): loop = asyncio.get_event_loop() if self._server is None: # no started yet, do nothing return # The loop over sites is intentional, an exception on gather() # leaves self._sites in unpredictable state. # The loop guaranties that a site is either deleted on success or # still present on failure for site in list(self._sites): await site.stop() await self._cleanup_server() self._server = None if self._handle_signals: try: loop.remove_signal_handler(signal.SIGINT) loop.remove_signal_handler(signal.SIGTERM) except NotImplementedError: # pragma: no cover # remove_signal_handler is not implemented on Windows pass @abstractmethod async def _make_server(self): pass # pragma: no cover @abstractmethod async def _cleanup_server(self): pass # pragma: no cover def _reg_site(self, site): if site in self._sites: raise RuntimeError( "Site {} is already registered in runner {}".format(site, self) ) self._sites.append(site) def _check_site(self, site): if site not in self._sites: raise RuntimeError( "Site {} is not registered in runner {}".format(site, self) ) def _unreg_site(self, site): if site not in self._sites: raise RuntimeError( "Site {} is not registered in runner {}".format(site, self) ) self._sites.remove(site) class BaseSite(ABC): __slots__ = ("_runner", "_shutdown_timeout", "_ssl_context", "_backlog", "_server") def __init__(self, runner, *, shutdown_timeout=60.0, ssl_context=None, backlog=128): if runner.server is None: raise RuntimeError("Call runner.setup() before making a site") self._runner = runner self._shutdown_timeout = shutdown_timeout self._ssl_context = ssl_context self._backlog = backlog self._server = None @property @abstractmethod def name(self): pass # pragma: no cover @abstractmethod async def start(self): self._runner._reg_site(self) async def stop(self): self._runner._check_site(self) if self._server is None: self._runner._unreg_site(self) return # not started yet self._server.close() await self._server.wait_closed() await self._runner.shutdown() await self._runner.server.shutdown(self._shutdown_timeout) self._runner._unreg_site(self) class TCPSite(BaseSite): __slots__ = ("_host", "_port", "_reuse_address", "_reuse_port") def __init__( self, runner, host=None, port=None, *, shutdown_timeout=60.0, ssl_context=None, backlog=128, reuse_address=None, reuse_port=None ): super().__init__( runner, shutdown_timeout=shutdown_timeout, ssl_context=ssl_context, backlog=backlog, ) if host is None: host = "0.0.0.0" self._host = host if port is None: port = 8443 if self._ssl_context else 8080 self._port = port self._reuse_address = reuse_address self._reuse_port = reuse_port @property def name(self): scheme = "https" if self._ssl_context else "http" return str(URL.build(scheme=scheme, host=self._host, port=self._port)) async def start(self): await super().start() loop = asyncio.get_event_loop() self._server = await loop.create_server( self._runner.server, self._host, self._port, ssl=self._ssl_context, backlog=self._backlog, reuse_address=self._reuse_address, reuse_port=self._reuse_port, ) class UnixSite(BaseSite): __slots__ = ("_path",) def __init__( self, runner, path, *, shutdown_timeout=60.0, ssl_context=None, backlog=128 ): super().__init__( runner, shutdown_timeout=shutdown_timeout, ssl_context=ssl_context, backlog=backlog, ) self._path = path @property def name(self): scheme = "https" if self._ssl_context else "http" return "{}://unix:{}:".format(scheme, self._path) async def start(self): await super().start() loop = asyncio.get_event_loop() self._server = await loop.create_unix_server( self._runner.server, self._path, ssl=self._ssl_context, backlog=self._backlog, ) class SockSite(BaseSite): __slots__ = ("_sock", "_name") def __init__( self, runner, sock, *, shutdown_timeout=60.0, ssl_context=None, backlog=128 ): super().__init__( runner, shutdown_timeout=shutdown_timeout, ssl_context=ssl_context, backlog=backlog, ) self._sock = sock scheme = "https" if self._ssl_context else "http" if hasattr(socket, "AF_UNIX") and sock.family == socket.AF_UNIX: name = "{}://unix:{}:".format(scheme, sock.getsockname()) else: host, port = sock.getsockname()[:2] name = str(URL.build(scheme=scheme, host=host, port=port)) self._name = name @property def name(self): return self._name async def start(self): await super().start() loop = asyncio.get_event_loop() self._server = await loop.create_server( self._runner.server, sock=self._sock, ssl=self._ssl_context, backlog=self._backlog, ) PK!A#44pillars/engines/__init__.pyfrom . import ari, pg, redis, systemd # noQa: F401 PK!I|w pillars/engines/ari.pyimport logging import time from dataclasses import dataclass, field from typing import Optional import aiohttp import ujson from ..app import Application LOG = logging.getLogger(__name__) @dataclass class ChannelCounter: time: int = field(default_factory=lambda: int(time.time())) counter: int = field(default=0) def new(self): current_time = int(time.time()) if current_time == self.time: self.counter += 1 else: self.time = current_time self.counter = 1 return f"{current_time}.{self.counter}" class AriClient: def __init__(self, app: Application, url: str, auth: aiohttp.BasicAuth) -> None: self._name = app["name"] self._base_url = url self._auth = auth self._channel_counter = ChannelCounter() self._client: Optional[aiohttp.ClientSession] = None app.on_startup.append(self._startup) app.on_cleanup.append(self._cleanup) async def _startup(self, app: Application) -> None: LOG.debug("Starting ARI client engine") self._client = aiohttp.ClientSession( auth=self._auth, json_serialize=ujson.dumps ) async def _cleanup(self, app: Application) -> None: LOG.debug("Cleaning up ARI client engine") if self._client: await self._client.close() async def status(self) -> bool: try: await self.request("GET", f"applications/{self._name}") except Exception: LOG.exception("ARI Client failed status") return False else: return True async def request( self, method: str, url: str, data: Optional[dict] = None, params: Optional[dict] = None, ) -> dict: LOG.log(4, "ARI request %s to %s with %s %s", method, url, params, data) url = self._base_url + url response = await self._request(method, url, data, params) return response async def _request( self, method: str, url: str, data: Optional[dict] = None, params: Optional[dict] = None, ) -> dict: if not self._client: raise RuntimeError("Engine not started") response = await self._client.request(method, url, json=data, params=params) response.raise_for_status() response_data = await response.text() if response_data: return ujson.loads(response_data) else: return {} ########### # HELPERS # ########### def generate_channel_id(self, channel_prefix: str = None) -> str: if channel_prefix: return f"{channel_prefix}.{self._channel_counter.new()}" else: return self._channel_counter.new() PK!ߓhpillars/engines/pg.pyimport asyncio import logging from contextlib import asynccontextmanager from typing import Optional import async_timeout import asyncpg import ujson from ..app import Application LOG = logging.getLogger(__name__) class PG: def __init__( self, app: Application, *args, reconnection_timeoff: int = 10, shutdown_timeout: int = 5, **kwargs ) -> None: self._loop = asyncio.get_event_loop() self._task: Optional[asyncio.Task] = None self._result: asyncio.Future = asyncio.Future() self._connection_info = (args, kwargs) self._shutdown_timeout = shutdown_timeout self._reconnection_timeoff = reconnection_timeoff app.on_startup.append(self._startup) app.on_shutdown.append(self._shutdown) app.on_cleanup.append(self._cleanup) async def _connect(self) -> None: try: pool = await asyncpg.create_pool( *self._connection_info[0], **self._connection_info[1] ) except ConnectionError: LOG.exception("PostgreSQL connection error") await asyncio.sleep(self._reconnection_timeoff) self._task = self._loop.create_task(self._connect()) except asyncio.CancelledError: pass except Exception as e: LOG.exception("PostgreSQL connection error") self._result.set_exception(e) else: LOG.info("PostgreSQL connection pool created") self._result.set_result(pool) @asynccontextmanager async def connection(self, timeout: int = 5) -> asyncpg.Connection: async with async_timeout.timeout(timeout): pool = await asyncio.shield(self._result) try: connection = await pool.acquire() except ConnectionError: LOG.debug("Connection error while acquiring connection") self._result = asyncio.Future() self._task = self._loop.create_task(self._connect()) pool = await asyncio.shield(self._result) connection = await pool.acquire() try: yield connection finally: await pool.release(connection) async def status(self, timeout: int = 2) -> bool: try: async with self.connection(timeout=timeout) as con: await con.fetchval("SELECT 1") except asyncio.TimeoutError: return False except Exception: LOG.exception("PostgreSQL failed status") return False else: LOG.log(4, "PostgreSQL status OK") return True async def _startup(self, app: Application) -> None: LOG.debug("Starting PostgreSQL engine") self._task = self._loop.create_task(self._connect()) async def _shutdown(self, app: Application) -> None: LOG.debug("Shutting down PostgreSQL engine") if self._task and not self._task.done(): self._task.cancel() if not self._result.done(): self._result.cancel() async def _cleanup(self, app: Application) -> None: LOG.debug("Cleaning up PostgreSQL engine") try: pool = await self._result except asyncio.CancelledError: pass else: await asyncio.wait_for(pool.close(), timeout=self._shutdown_timeout) async def register_json_codec(con: asyncpg.Connection) -> None: await con.set_type_codec( "json", encoder=ujson.dumps, decoder=ujson.loads, schema="pg_catalog" ) await con.set_type_codec( "jsonb", encoder=jsonb_encoder, decoder=jsonb_decoder, schema="pg_catalog", format="binary", ) def jsonb_encoder(value: str) -> bytes: try: return b"\x01" + ujson.dumps(value).encode("utf-8") except Exception: LOG.error("""Unable to encode to JSONB: %s""", value) raise def jsonb_decoder(value: bytes) -> dict: return ujson.loads(value[1:].decode("utf-8")) PK! pillars/engines/redis.pyimport asyncio import logging from contextlib import asynccontextmanager from typing import Optional import aioredis import async_timeout from ..app import Application LOG = logging.getLogger(__name__) class Redis: def __init__( self, app: Application, *args, reconnection_timeoff: int = 10, shutdown_timeout: int = 5, **kwargs ) -> None: self._loop = asyncio.get_event_loop() self._task: Optional[asyncio.Task] = None self._result: asyncio.Future = asyncio.Future() self._connection_info = (args, kwargs) self._shutdown_timeout = shutdown_timeout self._reconnection_timeoff = reconnection_timeoff app.on_startup.append(self._startup) app.on_shutdown.append(self._shutdown) app.on_cleanup.append(self._cleanup) async def _connect(self) -> None: try: pool = await aioredis.create_pool( *self._connection_info[0], **self._connection_info[1] ) except ConnectionError: LOG.exception("Redis connection error") await asyncio.sleep(self._reconnection_timeoff) self._task = self._loop.create_task(self._connect()) except Exception as e: LOG.exception("Redis connection error") self._result.set_exception(e) else: LOG.info("Redis connection pool created") self._result.set_result(pool) @asynccontextmanager async def connection(self, timeout: int = 5) -> aioredis.RedisConnection: async with async_timeout.timeout(timeout): pool = await asyncio.shield(self._result) try: connection = await pool.acquire() except ConnectionError: LOG.debug("Connection error while acquiring connection") self._result = asyncio.Future() self._task = self._loop.create_task(self._connect()) pool = await asyncio.shield(self._result) connection = await pool.acquire() try: yield connection finally: pool.release(connection) async def status(self, timeout: int = 2) -> bool: try: async with self.connection(timeout=timeout) as con: await con.execute("SET", "xxx_STATUS", 1) await con.execute("DEL", "xxx_STATUS", 1) except asyncio.TimeoutError: return False except Exception: LOG.exception("Redis failed status") return False else: LOG.log(4, "Redis status OK") return True async def _startup(self, app: Application) -> None: LOG.debug("Starting Redis engine") self._task = self._loop.create_task(self._connect()) self._result = asyncio.Future() async def _shutdown(self, app: Application) -> None: LOG.debug("Shutting down Redis engine") if self._task and not self._task.done(): self._task.cancel() if self._result.done(): pool = await self._result pool.close() else: self._result.cancel() async def _cleanup(self, app: Application) -> None: LOG.debug("Cleaning up Redis engine") try: pool = await self._result except asyncio.CancelledError: pass else: await asyncio.wait_for(pool.wait_closed(), timeout=self._shutdown_timeout) PK!)p pillars/engines/systemd.pyimport asyncio import logging import os import socket from typing import Awaitable, Callable, Optional from ..app import Application LOG = logging.getLogger(__name__) class Watchdog: def __init__( self, app: Application, path: Optional[str] = None, interval: Optional[int] = None, healthcheck: Optional[Callable[["Watchdog"], Awaitable[None]]] = None, ) -> None: if interval is None: usec = int(os.environ["WATCHDOG_USEC"]) interval = int(usec / 2000000) self._loop = asyncio.get_event_loop() self._path = path or os.environ["NOTIFY_SOCKET"] self._running = False self._interval = interval self._protocol: Optional[asyncio.DatagramProtocol] = None self._transport: Optional[asyncio.DatagramTransport] = None self._healthcheck = healthcheck or self.ping app.on_startup.append(self._startup) # Shutdown the watchdog first to skip healthcheck during teardown app.on_shutdown.insert(0, self._shutdown) LOG.debug("Systemd watchdog ping interval: %s seconds", self._interval) async def _start(self) -> None: self._transport, self._protocol = await self._loop.create_datagram_endpoint( # type: ignore asyncio.DatagramProtocol, family=socket.AF_UNIX, remote_addr=self._path, # type: ignore ) self.send(b"READY=1\nSTATUS=STARTING") await asyncio.sleep(1) while self._running: try: await self._healthcheck(self) except Exception: LOG.exception("Unhandle Error during healthcheck") await asyncio.sleep(self._interval) async def _startup(self, app: Application) -> None: LOG.debug("Starting Systemd engine") self._running = True self._task = self._loop.create_task(self._start()) async def _shutdown(self, app: Application) -> None: LOG.debug("Shutting down Systemd engine") self._running = False if self._transport: self.send(b"STOPPING=1") self._transport.close() self._transport = None self._protocol = None def send(self, data: bytes) -> None: if self._transport: self._transport.sendto(data) else: raise RuntimeError("No transport") async def ping(self, _) -> None: LOG.debug("Sending watchdog ping") self.send(b"WATCHDOG=1") PK!yp66pillars/exceptions.pyimport logging LOG = logging.getLogger(__name__) class GracefulExit(SystemExit): code = 0 class DataValidationError(Exception): def __init__(self, errors: dict) -> None: self.errors = errors class NotFound(Exception): def __init__(self, item: dict) -> None: self.item = item PK!KBBpillars/middlewares/__init__.pyfrom . import http # noQa: F401 from .pg import pg # noQa: F401 PK![YdUUpillars/middlewares/http.pyimport logging from typing import Awaitable, Callable import aiohttp.web from .. import exceptions from ..request import BaseRequest LOG = logging.getLogger(__name__) @aiohttp.web.middleware async def exception_handler( request: BaseRequest, handler: Callable[[BaseRequest], Awaitable[aiohttp.web.Response]], ) -> aiohttp.web.Response: try: response = await handler(request) except exceptions.DataValidationError as e: return aiohttp.web.json_response(status=400, data={"errors": e.errors}) except exceptions.NotFound as e: return aiohttp.web.json_response(status=404, data={"item": e.item}) except Exception: LOG.exception("Error handling request: %s", request.path) return aiohttp.web.json_response(status=500, data={"errors": ["Unknown error"]}) else: return response PK!;vpillars/middlewares/pg.pyimport contextlib from typing import Any, Awaitable, Callable from ..request import BaseRequest async def pg(request: BaseRequest, handler: Callable[[BaseRequest], Awaitable[Any]]): async with contextlib.AsyncExitStack() as stack: if "pg" in request.config: request["pg_connection"] = await stack.enter_async_context( request["pg"].connection() ) elif "pg_transaction" in request.config: request["pg_connection"] = await stack.enter_async_context( request["pg"].connection() ) await stack.enter_async_context(request["pg_connection"].transaction()) return await handler(request) PK!pillars/py.typedPK!ǧpillars/request.pyimport collections import json import logging import uuid from dataclasses import dataclass, field from typing import Any, Optional, Type LOG = logging.getLogger(__name__) class BaseRequest: def __init__(self, app_state: collections.ChainMap) -> None: self.id = uuid.uuid4() self._state = app_state.new_child() # MutableMapping API def __eq__(self, other): return self is other def __getitem__(self, key): return self._state[key] def __setitem__(self, key, value): self._state[key] = value def __delitem__(self, key): del self._state[key] def __len__(self): return len(self._state) def __iter__(self): return iter(self._state) async def data(self) -> dict: raise NotImplementedError() @property def initial(self) -> Any: raise NotImplementedError() @property def config(self) -> dict: raise NotImplementedError() @property def method(self) -> Optional[str]: raise NotImplementedError() @property def path(self) -> str: raise NotImplementedError() @dataclass class Response: status: int data: dict = field(default_factory=dict) json_encoder: Type[json.JSONEncoder] = field(default=json.JSONEncoder) PK!pillars/sites/__init__.pyfrom .datagram import DatagramSockSite, DatagramUnixSite, UDPSite # noQa: F401 from .protocol import ProtocolType, SockSite, TCPSite, UnixSite # noQa: F401 from .websocket import WSClientSite # noQa: F401 PK!D66pillars/sites/datagram.pyimport asyncio import logging import os import socket import stat from typing import Optional from aiohttp.web_runner import ( # noQa: F401 BaseRunner, BaseSite, SockSite, TCPSite, UnixSite, ) from .protocol import ProtocolType LOG = logging.getLogger(__name__) class DatagramServer: """ Shim to present a unified server interface. """ def __init__(self, transport: asyncio.DatagramTransport) -> None: self.transport = transport def close(self) -> None: self.transport.close() async def wait_closed(self) -> None: pass class UDPSite(BaseSite): def __init__( self, runner: BaseRunner, host: str = None, port: int = None, *, shutdown_timeout: float = 60.0, reuse_address: Optional[bool] = None, reuse_port=Optional[bool], ) -> None: super().__init__(runner, shutdown_timeout=shutdown_timeout) if host is None: host = "0.0.0.0" self._host = host if port is None: port = 8443 if self._ssl_context else 8080 self._port = port self._reuse_address = reuse_address self._reuse_port = reuse_port self._protocol_type = ProtocolType.DATAGRAM @property def name(self) -> str: return f"UDP://{self._host}:{self._port}" async def start(self) -> None: await super().start() loop = asyncio.get_event_loop() if self._runner.server: transport, protocol = await loop.create_datagram_endpoint( self._runner.server, local_addr=(self._host, self._port), reuse_address=self._reuse_address, reuse_port=self._reuse_port, ) # type: ignore self._server = DatagramServer(transport) # type: ignore class DatagramUnixSite(BaseSite): def __init__( self, runner: BaseRunner, path: str, *, shutdown_timeout: float = 60.0 ) -> None: super().__init__(runner, shutdown_timeout=shutdown_timeout) self._path = path self._protocol_type = ProtocolType.DATAGRAM @property def name(self) -> str: return f"UDP://unix:{self._path}" async def start(self) -> None: await super().start() await self._clean_stale_unix_socket(self._path) loop = asyncio.get_event_loop() transport, protocol = await loop.create_datagram_endpoint( # type: ignore self._runner.server, family=socket.AF_UNIX, local_addr=self._path ) self._server = DatagramServer(transport) # type: ignore @staticmethod async def _clean_stale_unix_socket(path: str) -> None: if path[0] not in (0, "\x00"): try: if stat.S_ISSOCK(os.stat(path).st_mode): os.remove(path) except FileNotFoundError: pass except OSError as err: # Directory may have permissions only to create socket. LOG.error( "Unable to check or remove stale UNIX socket %r: %r", path, err ) class DatagramSockSite(BaseSite): def __init__( self, runner: BaseRunner, sock: socket.socket, *, shutdown_timeout: float = 60.0 ) -> None: super().__init__(runner, shutdown_timeout=shutdown_timeout) self._sock = sock if hasattr(socket, "AF_UNIX") and sock.family == socket.AF_UNIX: name = f"UDP://unix:{sock.getsockname()}" else: host, port = sock.getsockname()[:2] name = f"UDP://{host}:{port}" self._name = name self._protocol_type = ProtocolType.DATAGRAM @property def name(self) -> str: return self._name async def start(self) -> None: await super().start() loop = asyncio.get_event_loop() if self._runner.server: transport, protocol = await loop.create_datagram_endpoint( self._runner.server, sock=self._sock ) # type: ignore self._server = DatagramServer(transport) # type: ignore PK!Q Npillars/sites/protocol.pyimport enum import logging from aiohttp.web_runner import SockSite, TCPSite, UnixSite LOG = logging.getLogger(__name__) @enum.unique class ProtocolType(enum.Enum): STREAM = 1 DATAGRAM = 2 WS = 3 TCPSite._protocol_type = ProtocolType.STREAM # type: ignore UnixSite._protocol_type = ProtocolType.STREAM # type: ignore SockSite._protocol_type = ProtocolType.STREAM # type: ignore PK!g(NNpillars/sites/websocket.pyimport asyncio import logging from typing import Optional, Union import aiohttp import aiohttp.http_websocket from ..base import BaseRunner, BaseSite from .protocol import ProtocolType LOG = logging.getLogger(__name__) class WSTransport(asyncio.BaseTransport): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) self._ws: Optional[aiohttp.ClientWebSocketResponse] = None self._closing = False self._closed: Optional[asyncio.Task] = None def close(self) -> None: self._closing = True if self._ws: self._closed = asyncio.create_task(self._close()) async def _close(self): await self._ws.close() self._ws = None async def closed(self): if self._closed: await self._closed async def status(self) -> bool: if self._ws: try: await self._ws.ping() except Exception: LOG.exception("Failed status for WS transport: %s", self._ws) return False return True else: return False def is_closing(self): """Return True if the transport is closing or closed.""" return self._closing def set_protocol(self, protocol): """Set a new protocol.""" raise NotImplementedError def get_protocol(self): """Return the current protocol.""" raise NotImplementedError class WSServer: """ Shim to present a unified server interface. """ def __init__(self, transport: WSTransport) -> None: self.transport = transport def close(self) -> None: self.transport.close() async def wait_closed(self) -> None: await self.transport.closed() class WSProtocol(asyncio.BaseProtocol): def message_received( self, message_type: aiohttp.http_websocket.WSMsgType, data: Union[str, bytes, aiohttp.http_websocket.WSCloseCode], extra: str, ): raise NotImplementedError() class WSClientSite(BaseSite): def __init__( self, runner: BaseRunner, url: str, *, shutdown_timeout: float = 60.0, session: aiohttp.ClientSession = None, ) -> None: super().__init__(runner, shutdown_timeout=shutdown_timeout) self._url = url self._name = f"WS://{url}" self._server = None self._session = session or aiohttp.ClientSession() self._protocol: Optional[WSProtocol] = None self._transport: Optional[WSTransport] = None self._closing = False self._protocol_type = ProtocolType.WS @property def name(self) -> str: return self._name async def start(self) -> None: await super().start() self._protocol: asyncio.Protocol = self._runner.server() self._transport: WSTransport = WSTransport() asyncio.create_task(self._ws_connection()) # type: ignore self._server = WSServer(transport=self._transport) async def stop(self) -> None: self._closing = True await super().stop() async def _ws_connection(self) -> None: if not self._transport or not self._protocol: raise TypeError("Missing transport and protocol") try: async with self._session.ws_connect(self._url) as ws: self._transport._ws = ws self._protocol.connection_made(self._transport) async for message in ws: LOG.log(2, "Data received: %s", message) self._protocol.message_received( message.type, message.data, message.extra ) # WSMsgType.CLOSE should call connection_lost # TODO: mypy #5537 09/2018 self._protocol.connection_lost(None) # type: ignore except aiohttp.client_exceptions.ClientError as e: LOG.debug("Failed to connect to %: %s", self._url, e) await asyncio.sleep(0.1) except Exception as e: self._protocol.connection_lost(e) if self._closing: await self._session.close() else: asyncio.create_task(self._ws_connection()) # type: ignore async def status(self) -> bool: if self._transport: return await self._transport.status() else: return False PK!٧==pillars/transports/__init__.pyfrom . import ari, fast_agi, http, sip, syslog # noQa: F401 PK!>o==pillars/transports/ari.pyimport asyncio import collections import functools import logging from typing import Any, Awaitable, Callable, Iterable, List, Optional, Tuple, Union import aiohttp.http_websocket import async_timeout import ujson from ..app import Application as MainApplication from ..base import BaseRunner from ..request import BaseRequest from ..sites.websocket import WSProtocol LOG = logging.getLogger(__name__) class AppRunner(BaseRunner): def __init__(self, app: "Application") -> None: super().__init__() self._app = app async def shutdown(self) -> None: await self._app.shutdown() async def _make_server(self) -> "AriServer": return AriServer(self._app._handler) async def _cleanup_server(self) -> None: await self._app.cleanup() class Event: def __init__(self, app, config, data): self.app = app self.data = data self.config = config @property def type(self): return self.data["type"].lower() class Application(collections.MutableMapping): def __init__( self, app: MainApplication, middlewares: Optional[Iterable] = None ) -> None: if middlewares: middlewares = list(middlewares) middlewares.insert(0, middleware) else: middlewares = (middleware,) self.router = Router() self._state = collections.ChainMap({}, app) self._middlewares = middlewares async def shutdown(self) -> None: pass async def cleanup(self) -> None: pass async def _handler(self, data: dict) -> None: route, config = self.router.resolve(data["type"].lower()) if route: event = Event(app=self, config=config, data=data) await self._call_route(route, event) return LOG.debug("No route for event: %s", data["type"]) async def _call_route( self, route: Callable[[Event], Awaitable[None]], event: Event ) -> None: LOG.log(4, "Handling event: %s", event.type) for middleware in reversed(self._middlewares): route = functools.partial(middleware, handler=route) try: await route(event) except Exception: LOG.exception("Exception while handling event: %s ", event) # MutableMapping API def __eq__(self, other): return self is other def __getitem__(self, key): return self._state[key] def __setitem__(self, key, value): self._state[key] = value def __delitem__(self, key): del self._state[key] def __len__(self): return len(self._state) def __iter__(self): return iter(self._state) class AriServer: def __init__(self, handler: Callable[[dict], Awaitable[None]]) -> None: self._handler = handler self._connections: List["AriProtocol"] = list() def __call__(self) -> "AriProtocol": proto = AriProtocol(handler=self._handler) self._connections.append(proto) return proto async def shutdown(self, timeout: int) -> None: async with async_timeout.timeout(timeout): await asyncio.gather(*(proto.shutdown() for proto in self._connections)) class AriProtocol(WSProtocol): def __init__(self, handler: Callable[[dict], Awaitable[None]]) -> None: self._handler = handler self._tasks: List[asyncio.Task] = list() def message_received( self, message_type: aiohttp.http_websocket.WSMsgType, data: Union[str, bytes, aiohttp.http_websocket.WSCloseCode], extra: str, ): LOG.log(2, "Message received: %s %s", message_type, data) if isinstance(data, (str, bytes)): # TODO mypy #1533 payload = ujson.loads(data) # type: ignore task = asyncio.create_task(self._handler(payload)) self._tasks.append(task) task.add_done_callback(self._task_completed) else: LOG.debug("Unhandled websocket message: %s", message_type) def connection_lost(self, error: Optional[Exception]) -> None: if error: LOG.error(error) def _task_completed(self, task): self._tasks.remove(task) async def shutdown(self): await asyncio.gather(*(task for task in self._tasks)) class AriRequest(BaseRequest): def __init__(self, event: Event) -> None: super().__init__(event.app.state) self._event = event async def data(self) -> dict: return self._event.data @property def initial(self) -> Event: return self._event @property def config(self) -> dict: return self._event.config @property def method(self) -> None: return None @property def path(self) -> str: return self._event.type async def middleware(event: Event, handler: Callable[[BaseRequest], Awaitable[None]]): request = AriRequest(event) await handler(request) class Router: def __init__(self) -> None: self._routes: dict = dict() def add( self, event: str, handler: Callable[..., Awaitable[None]], config: Any = None ): self._routes[event.lower()] = (handler, config) def resolve( self, event: str ) -> Union[Tuple[None, None], Tuple[Callable[..., Awaitable[None]], Any]]: return self._routes.get(event.lower(), self._routes.get("*", (None, None))) PK!ppillars/transports/fast_agi.pyimport asyncio import collections import functools import logging from typing import Awaitable, Callable, Iterable, Optional import panoramisk from ..base import BaseRunner from ..request import BaseRequest LOG = logging.getLogger(__name__) async def middleware( request: "Request", handler: Callable[["FastAGIRequest"], Awaitable[None]] ) -> None: common_request = FastAGIRequest(request) await handler(common_request) class Application(collections.MutableMapping): def __init__(self, middlewares: Optional[Iterable] = None) -> None: self.routes: dict = dict() self._state: dict = dict() if middlewares: middlewares = list(middlewares) else: middlewares = list() self._middlewares = middlewares async def shutdown(self) -> None: pass async def cleanup(self) -> None: pass async def _handler(self, request: "Request") -> None: request.app = self agi_network_script = request.get("agi_network_script") LOG.info( 'Received FastAGI request from %r for "%s" route', request._transport.get_extra_info("peername"), agi_network_script, ) if agi_network_script is not None: route = self.routes.get(agi_network_script) if route is not None: for m in reversed(self._middlewares): route = functools.partial(m, handler=route) try: await route(request) except Exception as e: LOG.exception(e) else: LOG.error('No route for the request "%s"', agi_network_script) else: LOG.error("No agi_network_script header for the request") request.close() LOG.debug("Client socket closed") # MutableMapping API def __eq__(self, other): return self is other def __getitem__(self, key): return self._state[key] def __setitem__(self, key, value): self._state[key] = value def __delitem__(self, key): del self._state[key] def __len__(self): return len(self._state) def __iter__(self): return iter(self._state) class Request(collections.MutableMapping): def __init__(self, *, transport: asyncio.Transport, **kwargs) -> None: self.app: Optional[Application] = None self.hangup: bool = False self._state = kwargs self._futures: list = list() self._transport = transport async def send_command(self, command: str) -> dict: if not command.endswith("\n"): command += "\n" f: asyncio.Future = asyncio.Future() self._futures.append(f) self._transport.write(command.encode()) return await f def _response(self, data: str) -> None: agi_result = panoramisk.utils.parse_agi_result(data) f = self._futures.pop() if "error" in agi_result: f.set_exception(FastAGIEException(agi_result)) else: f.set_result(agi_result) def close(self) -> None: self._transport.close() def __repr__(self) -> str: return f"" # MutableMapping API def __eq__(self, other): return self is other def __getitem__(self, key): return self._state[key] def __setitem__(self, key, value): self._state[key] = value def __delitem__(self, key): del self._state[key] def __len__(self): return len(self._state) def __iter__(self): return iter(self._state) class FastAGIRequest(BaseRequest): def __init__(self, request): super().__init__(request.app.state) self._request = request async def data(self) -> dict: return self._request._state @property def initial(self) -> Request: return self._request @property def config(self) -> dict: return dict() @property def method(self) -> None: return None @property def path(self) -> str: return self._request.get("agi_network_script", "") class FastAGIEException(Exception): def __init__(self, data: dict) -> None: self.data = data class AppRunner(BaseRunner): def __init__(self, app: Application) -> None: super().__init__() self._app = app async def shutdown(self) -> None: await self._app.shutdown() async def _make_server(self) -> "FastAGIServer": return FastAGIServer(self._app._handler) async def _cleanup_server(self) -> None: await self._app.cleanup() class FastAGIServer: def __init__(self, handler: Callable[["Request"], Awaitable[None]]) -> None: self._handler = handler def __call__(self): return FastAGIProtocol(handler=self._handler) async def shutdown(self, timeout): pass class FastAGIProtocol(asyncio.Protocol): def __init__(self, handler: Callable[["Request"], Awaitable[None]]) -> None: self._buffer = b"" self._request: Optional[Request] = None self._handler = handler self._transport: Optional[asyncio.BaseTransport] = None def connection_made(self, transport: asyncio.BaseTransport) -> None: self._transport = transport LOG.log(4, "connection made") def connection_lost(self, exc: Optional[Exception]) -> None: LOG.log(4, "connection lost") def data_received(self, raw_data: bytes) -> None: LOG.log(2, raw_data) if not self._request and b"\n\n" not in raw_data: self._buffer += raw_data return if not self._request: raw_data, self._buffer = self._buffer + raw_data, b"" lines = raw_data.decode().split("\n") data: dict = dict( line.split(": ", 1) for line in lines if line # type: ignore ) LOG.log(4, data) self._request = Request(transport=self._transport, **data) # type: ignore asyncio.ensure_future(self._handler(self._request)) elif raw_data == b"HANGUP\n": self._request.hangup = True else: self._request._response(raw_data.decode()) PK!pillars/transports/http.pyimport functools import json import logging from typing import Awaitable, Callable, Optional import aiohttp.web import cerberus from aiohttp.abc import AbstractMatchInfo import ujson from ..exceptions import DataValidationError from ..request import BaseRequest, Response LOG = logging.getLogger(__name__) @aiohttp.web.middleware async def middleware( request: aiohttp.web.Request, handler: Callable[["HttpRequest"], Awaitable[aiohttp.web.Response]], ): common_request = HttpRequest(request) response = await handler(common_request) if isinstance(response, Response): return aiohttp.web.json_response( status=response.status, data=response.data, dumps=functools.partial(json.dumps, cls=response.json_encoder), ) else: return response class HttpRequest(BaseRequest): def __init__(self, request: aiohttp.web.Request) -> None: super().__init__(request.app.state) # type: ignore self._request = request self._data: Optional[dict] = None self["validator"] = self._request["validator"] async def data(self, validate: bool = None) -> dict: if self._data is None: if "json" in self.config: self._data = await self._request.json(loads=ujson.loads) elif self._request.method == "GET": self._data = dict(self._request.query) else: self._data = {"text": await self._request.text()} return self._data # TODO mypy (self._data can not be None) self._data.update(self._request.match_info) # type: ignore if validate: if self["validator"].validate(self._data): self._data = self["validator"].document else: raise DataValidationError(self["validator"].errors) return self._data or dict() @property def initial(self) -> aiohttp.web.Request: return self._request @property def config(self) -> dict: return self._request["config"] @property def method(self) -> str: return self._request.method @property def path(self) -> str: return self._request.path class Application(aiohttp.web.Application): def __init__(self, **kwargs) -> None: if "router" not in kwargs: kwargs["router"] = Router() if "middlewares" not in kwargs: kwargs["middlewares"] = (middleware,) else: kwargs["middlewares"] = list(kwargs["middlewares"]) kwargs["middlewares"].insert(0, middleware) super().__init__(**kwargs) class Router(aiohttp.web.UrlDispatcher): def __init__(self) -> None: super().__init__() self.config: dict = dict() self.validators: dict = dict() def add_route( self, *args, config: Optional[dict] = None, data_schema: Optional[dict] = None, **kwargs ): route = super().add_route(*args, **kwargs) if data_schema: self.validators[route] = cerberus.Validator(data_schema) if config: self.config[route] = set(config) else: self.config[route] = set() return route async def resolve(self, request: aiohttp.web.Request) -> AbstractMatchInfo: match_info = await super().resolve(request) request["config"] = self.config.get(match_info.handler, ()) request["validator"] = self.validators.get(match_info.handler) return match_info PK!FFpillars/transports/sip.pyimport asyncio import logging from typing import Union import aiosip from ..base import BaseRunner from ..sites import ProtocolType LOG = logging.getLogger(__name__) class Application(aiosip.Application): async def shutdown(self) -> None: await self.close() async def cleanup(self) -> None: pass class AppRunner(BaseRunner): def __init__(self, app: Application) -> None: super().__init__() self._app = app async def shutdown(self) -> None: await self._app.shutdown() async def _make_server(self) -> "SipServer": return SipServer(app=self._app) async def _cleanup_server(self) -> None: await self._app.cleanup() def _reg_site(self, site) -> None: super()._reg_site(site) if self._server._protocol_type is None: self._server._protocol_type = site._protocol_type elif self._server._protocol_type != site._protocol_type: raise TypeError("All sites must use the same protocol_type") class SipServer: def __init__(self, app: Application) -> None: self._app = app self._protocol_type = None def __call__(self) -> Union[aiosip.WS, aiosip.TCP, aiosip.UDP]: if self._protocol_type == ProtocolType.STREAM: protocol = aiosip.TCP elif self._protocol_type == ProtocolType.DATAGRAM: protocol = aiosip.UDP else: raise RuntimeError("Unknown protocol type") return protocol(self._app, loop=asyncio.get_event_loop()) async def shutdown(self, timeout: int) -> None: pass PK!!u u pillars/transports/syslog.pyimport asyncio import collections import logging from typing import Awaitable, Callable, Optional, Tuple, Union from ..base import BaseRunner LOG = logging.getLogger(__name__) class Application(collections.MutableMapping): async def shutdown(self) -> None: pass async def cleanup(self) -> None: pass async def _handler(self, data: Union[str, bytes], addr: Tuple[str, int]) -> None: LOG.debug(data, addr) def __init__(self) -> None: self._state: dict = dict() # MutableMapping API def __eq__(self, other): return self is other def __getitem__(self, key): return self._state[key] def __setitem__(self, key, value): self._state[key] = value def __delitem__(self, key): del self._state[key] def __len__(self): return len(self._state) def __iter__(self): return iter(self._state) class AppRunner(BaseRunner): def __init__(self, app: Application) -> None: super().__init__() self._app = app async def shutdown(self) -> None: await self._app.shutdown() async def _make_server(self) -> "SyslogServer": return SyslogServer(self._app._handler) async def _cleanup_server(self) -> None: await self._app.cleanup() class SyslogServer: def __init__( self, handler: Callable[[Union[str, bytes], Tuple[str, int]], Awaitable[None]] ) -> None: self._handler = handler def __call__(self) -> "SyslogProtocol": return SyslogProtocol(handler=self._handler) async def shutdown(self, timeout) -> None: pass class SyslogProtocol(asyncio.Protocol, asyncio.DatagramProtocol): def __init__( self, handler: Callable[[Union[str, bytes], Tuple[str, int]], Awaitable[None]] ) -> None: self._handler = handler self.transport: Optional[asyncio.BaseTransport] = None def connection_made(self, transport: asyncio.BaseTransport) -> None: self.transport = transport def data_received(self, data: Union[str, bytes]) -> None: if self.transport: addr = self.transport.get_extra_info("peername") else: addr = ("", 0) asyncio.ensure_future(self._handler(data, addr)) def datagram_received(self, data: Union[str, bytes], addr: Tuple[str, int]) -> None: asyncio.ensure_future(self._handler(data, addr)) PK!Q:hhpillars/utils.pyimport json import logging import uuid class LoggingSTDOutFilter(logging.Filter): def filter(self, record: logging.LogRecord) -> bool: return record.levelno < logging.WARNING class JSONUUIDEncoder(json.JSONEncoder): def default(self, o): if isinstance(o, uuid.UUID): return o.hex return super().default(self, o) PK!HڽTUpillars-0.2.5.dist-info/WHEEL A н#Z;/"d&F[xzw@Zpy3Fv]\fi4WZ^EgM_-]#0(q7PK!Hԛ pillars-0.2.5.dist-info/METADATAVr6 }Wp<};fzldiusicZen(RKRv<{AJq6%ϸџ`jDOy#Z )dcvDnf=ZJHnQ= %\:̄)k M3VO9_J, K. 9BISDPp!Go?咭ܜJd@${ qZZ1ot4o4z _zVL}btnxQஸ+ ^{xGRX )KgI¹cl<17 YQ9Q\,s9Ss9d?邛 ˁG TM\qu ;\\iB>Yy!؂&pꋭRj]6 3/Fբ (ݭKO.~uL'uS_8.M 3pQdv]|ɠRInGkD[A Awqx3Y@t+kUd.ܢT1"1υjJոrM ,]n„G %P<֫JdƸqsZ[٣s hts7™Bv"㥋 PK!H^pillars-0.2.5.dist-info/RECORDuǚF< j D  >r @Oo-nfnwyUM>yGAc87U\ 71SIxu2DX]1: ؐ(K]SĩIClMa6ZC(4mn 80{B`; )2N}(9^V5D>I&sQ{Tңqϕ"Y.W5}M|WĪ|``@śAS(%)B)uN;FL|rveO=0vu|3@oGe[aU#Ҕ` v^Cd,$ p8u#}{;:X|;tí`*b o=7)\øH`3kDzZK~Į:k:P.?:*^BsYH9eA??O¾y xaLӚ15TOGD qc4q$J¨EU4صc/ɌUζW#+L͆.np@[)^9lԕųhDfp5l #̻D)o$#[90By ȴ[SL{UM;v#|/jP#8d34DUa` s@^ !.6VUC ]u'XV HONTRP~hD,WzkB g&¤w^˼0s:Me '>]zV/}ţV>[1\~]wzp&2&n. 54o I|SrxI8*J"wXPK!u)pillars/__init__.pyPK!?Fpillars/app.pyPK!+ Tsspillars/base.pyPK!A#44h0pillars/engines/__init__.pyPK!I|w 0pillars/engines/ari.pyPK!ߓh;pillars/engines/pg.pyPK! 1Lpillars/engines/redis.pyPK!)p .Zpillars/engines/systemd.pyPK!yp66o==Lpillars/transports/ari.pyPK!ppillars/transports/fast_agi.pyPK!pillars/transports/http.pyPK!FF pillars/transports/sip.pyPK!!u u pillars/transports/syslog.pyPK!Q:hh6pillars/utils.pyPK!HڽTUpillars-0.2.5.dist-info/WHEELPK!Hԛ [pillars-0.2.5.dist-info/METADATAPK!H^0pillars-0.2.5.dist-info/RECORDPK]