PK! V{xpymortar/__init__.py__version__ = '0.1.0' from pymortar import mortar_pb2 from pymortar import mortar_pb2_grpc from pymortar.result import Result from pymortar.mortar_pb2 import GetAPIKeyRequest, FetchRequest, QualifyRequest, Stream, TimeParams, Timeseries, Collection, Selection from pymortar.mortar_pb2 import AGG_FUNC_RAW as RAW from pymortar.mortar_pb2 import AGG_FUNC_MEAN as MEAN from pymortar.mortar_pb2 import AGG_FUNC_MIN as MIN from pymortar.mortar_pb2 import AGG_FUNC_MAX as MAX from pymortar.mortar_pb2 import AGG_FUNC_COUNT as COUNT from pymortar.mortar_pb2 import AGG_FUNC_SUM as SUM import pandas as pd import os import json import grpc import logging logging.basicConfig(level=logging.DEBUG) class Client: """ Mortar client Parameters ---------- cfg: dict Contains the configuration information for connecting to the Mortar API. Expects the following keys: mortar_address: address + port to connect to, e.g. "localhost:9001". Defaults to $MORTAR_API_ADDRESS from the environment. Currently expects a TLS-secured endpoint username: your Mortar API username. Defaults to MORTAR_API_USERNAME env var password: your Mortar API password. Defaults to MORTAR_API_PASSWORD env var Returns ------- client: Client An instance of the Mortar Client. """ def __init__(self, cfg=None): if cfg is not None: self._cfg = cfg else: self._cfg = {} # get username/password from environment or config file if 'username' not in self._cfg or not self._cfg['username']: self._cfg['username'] = os.environ.get('MORTAR_API_USERNAME') if 'password' not in self._cfg or not self._cfg['password']: self._cfg['password'] = os.environ.get('MORTAR_API_PASSWORD') if self._cfg.get('mortar_address') is None: self._mortar_address = os.environ.get('MORTAR_API_ADDRESS','mortardata.org:9001') else: self._mortar_address = self._cfg.get('mortar_address') # setup GRPC client: gzip + tls if self._cfg.get('abandon_all_tls') == "yes i'm sure": print('insecure') self._channel = grpc.insecure_channel(self._mortar_address, options=[ ('grpc.default_compression_algorithm', 2) # 2 is GZIP ]) else: credentials = grpc.ssl_channel_credentials() self._channel = grpc.secure_channel(self._mortar_address, credentials, options=[ ('grpc.default_compression_algorithm', 2) # 2 is GZIP ]) self._client = mortar_pb2_grpc.MortarStub(self._channel) # TODO: check if a .pymortartoken.json file exists. If it does, then use the token # and don't do the username/password login if os.path.exists(".pymortartoken.json"): self._token = json.load(open(".pymortartoken.json", "r")) #print("loaded token: {0}".format(self._token)) else: self._token = None # TODO: handle the refresh token recycling automatically for the user # TODO: break this out into a method that can be called when we notice that the token is expired if self._token is None: self._refresh() def _refresh(self): response = self._client.GetAPIKey(mortar_pb2.GetAPIKeyRequest(username=self._cfg["username"],password=self._cfg["password"])) #print(response) self._token = response.token json.dump(self._token, open(".pymortartoken.json", "w")) def fetch(self, request): """ Calls the Mortar API Fetch command Parameters ---------- req: mortar_pb2.FetchRequest TODO: need to document the fetch request parameters sites: list of strings Each string is a site name. These can be found through the qualify() API call streams: list of Streams Streams are how Mortar refers to collections of timeseries data. time: TimeParams Defines the temporal parameters for the data query Returns ------- resp: pandas.DataFrame The column names are the UUIDs either explicitly annotated in the request or found through the Stream definitions TODO: figure out how to add in the metadata component """ try: resp = self._client.Fetch(request, metadata=[('token', self._token)]) except Exception as e: if e.details() == 'parse jwt token err: Token is expired': self._refresh() return self.fetch(request) else: raise e res = Result() for x in resp: if x.error != "": logging.error(x.error) break #res.add(x) res.add2(x) res.build() return res def qualify(self, required_queries): """ Calls the Mortar API Qualify command Parameters ---------- required_queries: list of str list of queries we want to use to filter sites Returns ------- sites: list of str List of site names to be used in a subsequent fetch command """ try: resp = self._client.Qualify(QualifyRequest(required=required_queries), metadata=[('token', self._token)]) if resp.error: raise Exception(resp.error) return resp except Exception as e: if hasattr(e, 'details') and e.details() == 'parse jwt token err: Token is expired': self._refresh() return self.qualify(required_queries) else: raise e PK!tnJ pymortar/mortar_pb2.py# Generated by the protocol buffer compiler. DO NOT EDIT! # source: mortar.proto import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf.internal import enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='mortar.proto', package='mortar', syntax='proto3', serialized_options=_b('Z\010mortarpb'), serialized_pb=_b('\n\x0cmortar.proto\x12\x06mortar\"L\n\x10GetAPIKeyRequest\x12\x10\n\x08username\x18\x01 \x01(\t\x12\x10\n\x08password\x18\x02 \x01(\t\x12\x14\n\x0crefreshtoken\x18\x03 \x01(\t\"5\n\x0e\x41PIKeyResponse\x12\r\n\x05token\x18\x01 \x01(\t\x12\x14\n\x0crefreshtoken\x18\x02 \x01(\t\"4\n\x0eQualifyRequest\x12\x10\n\x08required\x18\x01 \x03(\t\x12\x10\n\x08optional\x18\x02 \x03(\t\"/\n\x0fQualifyResponse\x12\r\n\x05\x65rror\x18\x01 \x01(\t\x12\r\n\x05sites\x18\x02 \x03(\t\"\xb0\x01\n\x0c\x46\x65tchRequest\x12\r\n\x05sites\x18\x01 \x03(\t\x12\x1f\n\x07streams\x18\x02 \x03(\x0b\x32\x0e.mortar.Stream\x12 \n\x04time\x18\x03 \x01(\x0b\x32\x12.mortar.TimeParams\x12\'\n\x0b\x63ollections\x18\x04 \x03(\x0b\x32\x12.mortar.Collection\x12%\n\nselections\x18\x05 \x03(\x0b\x32\x11.mortar.Selection\"\x80\x01\n\x06Stream\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\ndefinition\x18\x02 \x01(\t\x12\x10\n\x08\x64\x61taVars\x18\x06 \x03(\t\x12\r\n\x05uuids\x18\x03 \x03(\t\x12$\n\x0b\x61ggregation\x18\x04 \x01(\x0e\x32\x0f.mortar.AggFunc\x12\r\n\x05units\x18\x05 \x01(\t\"\xc6\x01\n\rFetchResponse\x12\r\n\x05\x65rror\x18\x01 \x01(\t\x12\x0c\n\x04site\x18\x02 \x01(\t\x12\x12\n\ncollection\x18\t \x01(\t\x12\x11\n\tselection\x18\n \x01(\t\x12\x10\n\x08variable\x18\x03 \x01(\t\x12\x12\n\nidentifier\x18\x04 \x01(\t\x12\r\n\x05times\x18\x05 \x03(\x03\x12\x0e\n\x06values\x18\x06 \x03(\x01\x12\x11\n\tvariables\x18\x07 \x03(\t\x12\x19\n\x04rows\x18\x08 \x03(\x0b\x32\x0b.mortar.Row\"\"\n\x03Row\x12\x1b\n\x06values\x18\x01 \x03(\x0b\x32\x0b.mortar.URI\"\'\n\x03URI\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"I\n\nTimeParams\x12\r\n\x05start\x18\x01 \x01(\t\x12\x0b\n\x03\x65nd\x18\x02 \x01(\t\x12\x0e\n\x06window\x18\x03 \x01(\t\x12\x0f\n\x07\x61ligned\x18\x04 \x01(\x08\"=\n\nCollection\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05sites\x18\x02 \x03(\t\x12\x12\n\ndefinition\x18\x03 \x01(\t\"\x94\x01\n\tSelection\x12\x0c\n\x04name\x18\x01 \x01(\t\x12$\n\x0b\x61ggregation\x18\x02 \x01(\x0e\x32\x0f.mortar.AggFunc\x12\x0e\n\x06window\x18\x03 \x01(\t\x12\x0c\n\x04unit\x18\x04 \x01(\t\x12&\n\ntimeseries\x18\x05 \x03(\x0b\x32\x12.mortar.Timeseries\x12\r\n\x05uuids\x18\x06 \x03(\t\"2\n\nTimeseries\x12\x12\n\ncollection\x18\x01 \x01(\t\x12\x10\n\x08\x64\x61taVars\x18\x02 \x03(\t*\x8e\x01\n\x07\x41ggFunc\x12\x14\n\x10\x41GG_FUNC_INVALID\x10\x00\x12\x10\n\x0c\x41GG_FUNC_RAW\x10\x01\x12\x11\n\rAGG_FUNC_MEAN\x10\x02\x12\x10\n\x0c\x41GG_FUNC_MIN\x10\x03\x12\x10\n\x0c\x41GG_FUNC_MAX\x10\x04\x12\x12\n\x0e\x41GG_FUNC_COUNT\x10\x05\x12\x10\n\x0c\x41GG_FUNC_SUM\x10\x06\x32\xbb\x01\n\x06Mortar\x12=\n\tGetAPIKey\x12\x18.mortar.GetAPIKeyRequest\x1a\x16.mortar.APIKeyResponse\x12:\n\x07Qualify\x12\x16.mortar.QualifyRequest\x1a\x17.mortar.QualifyResponse\x12\x36\n\x05\x46\x65tch\x12\x14.mortar.FetchRequest\x1a\x15.mortar.FetchResponse0\x01\x42\nZ\x08mortarpbb\x06proto3') ) _AGGFUNC = _descriptor.EnumDescriptor( name='AggFunc', full_name='mortar.AggFunc', filename=None, file=DESCRIPTOR, values=[ _descriptor.EnumValueDescriptor( name='AGG_FUNC_INVALID', index=0, number=0, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='AGG_FUNC_RAW', index=1, number=1, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='AGG_FUNC_MEAN', index=2, number=2, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='AGG_FUNC_MIN', index=3, number=3, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='AGG_FUNC_MAX', index=4, number=4, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='AGG_FUNC_COUNT', index=5, number=5, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( name='AGG_FUNC_SUM', index=6, number=6, serialized_options=None, type=None), ], containing_type=None, serialized_options=None, serialized_start=1190, serialized_end=1332, ) _sym_db.RegisterEnumDescriptor(_AGGFUNC) AggFunc = enum_type_wrapper.EnumTypeWrapper(_AGGFUNC) AGG_FUNC_INVALID = 0 AGG_FUNC_RAW = 1 AGG_FUNC_MEAN = 2 AGG_FUNC_MIN = 3 AGG_FUNC_MAX = 4 AGG_FUNC_COUNT = 5 AGG_FUNC_SUM = 6 _GETAPIKEYREQUEST = _descriptor.Descriptor( name='GetAPIKeyRequest', full_name='mortar.GetAPIKeyRequest', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='username', full_name='mortar.GetAPIKeyRequest.username', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='password', full_name='mortar.GetAPIKeyRequest.password', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='refreshtoken', full_name='mortar.GetAPIKeyRequest.refreshtoken', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=24, serialized_end=100, ) _APIKEYRESPONSE = _descriptor.Descriptor( name='APIKeyResponse', full_name='mortar.APIKeyResponse', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='token', full_name='mortar.APIKeyResponse.token', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='refreshtoken', full_name='mortar.APIKeyResponse.refreshtoken', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=102, serialized_end=155, ) _QUALIFYREQUEST = _descriptor.Descriptor( name='QualifyRequest', full_name='mortar.QualifyRequest', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='required', full_name='mortar.QualifyRequest.required', index=0, number=1, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='optional', full_name='mortar.QualifyRequest.optional', index=1, number=2, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=157, serialized_end=209, ) _QUALIFYRESPONSE = _descriptor.Descriptor( name='QualifyResponse', full_name='mortar.QualifyResponse', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='error', full_name='mortar.QualifyResponse.error', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='sites', full_name='mortar.QualifyResponse.sites', index=1, number=2, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=211, serialized_end=258, ) _FETCHREQUEST = _descriptor.Descriptor( name='FetchRequest', full_name='mortar.FetchRequest', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='sites', full_name='mortar.FetchRequest.sites', index=0, number=1, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='streams', full_name='mortar.FetchRequest.streams', index=1, number=2, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='time', full_name='mortar.FetchRequest.time', index=2, number=3, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='collections', full_name='mortar.FetchRequest.collections', index=3, number=4, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='selections', full_name='mortar.FetchRequest.selections', index=4, number=5, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=261, serialized_end=437, ) _STREAM = _descriptor.Descriptor( name='Stream', full_name='mortar.Stream', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='name', full_name='mortar.Stream.name', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='definition', full_name='mortar.Stream.definition', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='dataVars', full_name='mortar.Stream.dataVars', index=2, number=6, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='uuids', full_name='mortar.Stream.uuids', index=3, number=3, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='aggregation', full_name='mortar.Stream.aggregation', index=4, number=4, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='units', full_name='mortar.Stream.units', index=5, number=5, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=440, serialized_end=568, ) _FETCHRESPONSE = _descriptor.Descriptor( name='FetchResponse', full_name='mortar.FetchResponse', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='error', full_name='mortar.FetchResponse.error', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='site', full_name='mortar.FetchResponse.site', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='collection', full_name='mortar.FetchResponse.collection', index=2, number=9, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='selection', full_name='mortar.FetchResponse.selection', index=3, number=10, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='variable', full_name='mortar.FetchResponse.variable', index=4, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='identifier', full_name='mortar.FetchResponse.identifier', index=5, number=4, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='times', full_name='mortar.FetchResponse.times', index=6, number=5, type=3, cpp_type=2, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='values', full_name='mortar.FetchResponse.values', index=7, number=6, type=1, cpp_type=5, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='variables', full_name='mortar.FetchResponse.variables', index=8, number=7, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='rows', full_name='mortar.FetchResponse.rows', index=9, number=8, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=571, serialized_end=769, ) _ROW = _descriptor.Descriptor( name='Row', full_name='mortar.Row', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='values', full_name='mortar.Row.values', index=0, number=1, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=771, serialized_end=805, ) _URI = _descriptor.Descriptor( name='URI', full_name='mortar.URI', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='namespace', full_name='mortar.URI.namespace', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='value', full_name='mortar.URI.value', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=807, serialized_end=846, ) _TIMEPARAMS = _descriptor.Descriptor( name='TimeParams', full_name='mortar.TimeParams', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='start', full_name='mortar.TimeParams.start', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='end', full_name='mortar.TimeParams.end', index=1, number=2, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='window', full_name='mortar.TimeParams.window', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='aligned', full_name='mortar.TimeParams.aligned', index=3, number=4, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=848, serialized_end=921, ) _COLLECTION = _descriptor.Descriptor( name='Collection', full_name='mortar.Collection', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='name', full_name='mortar.Collection.name', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='sites', full_name='mortar.Collection.sites', index=1, number=2, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='definition', full_name='mortar.Collection.definition', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=923, serialized_end=984, ) _SELECTION = _descriptor.Descriptor( name='Selection', full_name='mortar.Selection', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='name', full_name='mortar.Selection.name', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='aggregation', full_name='mortar.Selection.aggregation', index=1, number=2, type=14, cpp_type=8, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='window', full_name='mortar.Selection.window', index=2, number=3, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='unit', full_name='mortar.Selection.unit', index=3, number=4, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='timeseries', full_name='mortar.Selection.timeseries', index=4, number=5, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='uuids', full_name='mortar.Selection.uuids', index=5, number=6, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=987, serialized_end=1135, ) _TIMESERIES = _descriptor.Descriptor( name='Timeseries', full_name='mortar.Timeseries', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( name='collection', full_name='mortar.Timeseries.collection', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='dataVars', full_name='mortar.Timeseries.dataVars', index=1, number=2, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], serialized_options=None, is_extendable=False, syntax='proto3', extension_ranges=[], oneofs=[ ], serialized_start=1137, serialized_end=1187, ) _FETCHREQUEST.fields_by_name['streams'].message_type = _STREAM _FETCHREQUEST.fields_by_name['time'].message_type = _TIMEPARAMS _FETCHREQUEST.fields_by_name['collections'].message_type = _COLLECTION _FETCHREQUEST.fields_by_name['selections'].message_type = _SELECTION _STREAM.fields_by_name['aggregation'].enum_type = _AGGFUNC _FETCHRESPONSE.fields_by_name['rows'].message_type = _ROW _ROW.fields_by_name['values'].message_type = _URI _SELECTION.fields_by_name['aggregation'].enum_type = _AGGFUNC _SELECTION.fields_by_name['timeseries'].message_type = _TIMESERIES DESCRIPTOR.message_types_by_name['GetAPIKeyRequest'] = _GETAPIKEYREQUEST DESCRIPTOR.message_types_by_name['APIKeyResponse'] = _APIKEYRESPONSE DESCRIPTOR.message_types_by_name['QualifyRequest'] = _QUALIFYREQUEST DESCRIPTOR.message_types_by_name['QualifyResponse'] = _QUALIFYRESPONSE DESCRIPTOR.message_types_by_name['FetchRequest'] = _FETCHREQUEST DESCRIPTOR.message_types_by_name['Stream'] = _STREAM DESCRIPTOR.message_types_by_name['FetchResponse'] = _FETCHRESPONSE DESCRIPTOR.message_types_by_name['Row'] = _ROW DESCRIPTOR.message_types_by_name['URI'] = _URI DESCRIPTOR.message_types_by_name['TimeParams'] = _TIMEPARAMS DESCRIPTOR.message_types_by_name['Collection'] = _COLLECTION DESCRIPTOR.message_types_by_name['Selection'] = _SELECTION DESCRIPTOR.message_types_by_name['Timeseries'] = _TIMESERIES DESCRIPTOR.enum_types_by_name['AggFunc'] = _AGGFUNC _sym_db.RegisterFileDescriptor(DESCRIPTOR) GetAPIKeyRequest = _reflection.GeneratedProtocolMessageType('GetAPIKeyRequest', (_message.Message,), dict( DESCRIPTOR = _GETAPIKEYREQUEST, __module__ = 'mortar_pb2' # @@protoc_insertion_point(class_scope:mortar.GetAPIKeyRequest) )) _sym_db.RegisterMessage(GetAPIKeyRequest) APIKeyResponse = _reflection.GeneratedProtocolMessageType('APIKeyResponse', (_message.Message,), dict( DESCRIPTOR = _APIKEYRESPONSE, __module__ = 'mortar_pb2' # @@protoc_insertion_point(class_scope:mortar.APIKeyResponse) )) _sym_db.RegisterMessage(APIKeyResponse) QualifyRequest = _reflection.GeneratedProtocolMessageType('QualifyRequest', (_message.Message,), dict( DESCRIPTOR = _QUALIFYREQUEST, __module__ = 'mortar_pb2' # @@protoc_insertion_point(class_scope:mortar.QualifyRequest) )) _sym_db.RegisterMessage(QualifyRequest) QualifyResponse = _reflection.GeneratedProtocolMessageType('QualifyResponse', (_message.Message,), dict( DESCRIPTOR = _QUALIFYRESPONSE, __module__ = 'mortar_pb2' # @@protoc_insertion_point(class_scope:mortar.QualifyResponse) )) _sym_db.RegisterMessage(QualifyResponse) FetchRequest = _reflection.GeneratedProtocolMessageType('FetchRequest', (_message.Message,), dict( DESCRIPTOR = _FETCHREQUEST, __module__ = 'mortar_pb2' # @@protoc_insertion_point(class_scope:mortar.FetchRequest) )) _sym_db.RegisterMessage(FetchRequest) Stream = _reflection.GeneratedProtocolMessageType('Stream', (_message.Message,), dict( DESCRIPTOR = _STREAM, __module__ = 'mortar_pb2' # @@protoc_insertion_point(class_scope:mortar.Stream) )) _sym_db.RegisterMessage(Stream) FetchResponse = _reflection.GeneratedProtocolMessageType('FetchResponse', (_message.Message,), dict( DESCRIPTOR = _FETCHRESPONSE, __module__ = 'mortar_pb2' # @@protoc_insertion_point(class_scope:mortar.FetchResponse) )) _sym_db.RegisterMessage(FetchResponse) Row = _reflection.GeneratedProtocolMessageType('Row', (_message.Message,), dict( DESCRIPTOR = _ROW, __module__ = 'mortar_pb2' # @@protoc_insertion_point(class_scope:mortar.Row) )) _sym_db.RegisterMessage(Row) URI = _reflection.GeneratedProtocolMessageType('URI', (_message.Message,), dict( DESCRIPTOR = _URI, __module__ = 'mortar_pb2' # @@protoc_insertion_point(class_scope:mortar.URI) )) _sym_db.RegisterMessage(URI) TimeParams = _reflection.GeneratedProtocolMessageType('TimeParams', (_message.Message,), dict( DESCRIPTOR = _TIMEPARAMS, __module__ = 'mortar_pb2' # @@protoc_insertion_point(class_scope:mortar.TimeParams) )) _sym_db.RegisterMessage(TimeParams) Collection = _reflection.GeneratedProtocolMessageType('Collection', (_message.Message,), dict( DESCRIPTOR = _COLLECTION, __module__ = 'mortar_pb2' # @@protoc_insertion_point(class_scope:mortar.Collection) )) _sym_db.RegisterMessage(Collection) Selection = _reflection.GeneratedProtocolMessageType('Selection', (_message.Message,), dict( DESCRIPTOR = _SELECTION, __module__ = 'mortar_pb2' # @@protoc_insertion_point(class_scope:mortar.Selection) )) _sym_db.RegisterMessage(Selection) Timeseries = _reflection.GeneratedProtocolMessageType('Timeseries', (_message.Message,), dict( DESCRIPTOR = _TIMESERIES, __module__ = 'mortar_pb2' # @@protoc_insertion_point(class_scope:mortar.Timeseries) )) _sym_db.RegisterMessage(Timeseries) DESCRIPTOR._options = None _MORTAR = _descriptor.ServiceDescriptor( name='Mortar', full_name='mortar.Mortar', file=DESCRIPTOR, index=0, serialized_options=None, serialized_start=1335, serialized_end=1522, methods=[ _descriptor.MethodDescriptor( name='GetAPIKey', full_name='mortar.Mortar.GetAPIKey', index=0, containing_service=None, input_type=_GETAPIKEYREQUEST, output_type=_APIKEYRESPONSE, serialized_options=None, ), _descriptor.MethodDescriptor( name='Qualify', full_name='mortar.Mortar.Qualify', index=1, containing_service=None, input_type=_QUALIFYREQUEST, output_type=_QUALIFYRESPONSE, serialized_options=None, ), _descriptor.MethodDescriptor( name='Fetch', full_name='mortar.Mortar.Fetch', index=2, containing_service=None, input_type=_FETCHREQUEST, output_type=_FETCHRESPONSE, serialized_options=None, ), ]) _sym_db.RegisterServiceDescriptor(_MORTAR) DESCRIPTOR.services_by_name['Mortar'] = _MORTAR # @@protoc_insertion_point(module_scope) PK!.T  pymortar/mortar_pb2_grpc.py# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! import grpc from . import mortar_pb2 as mortar__pb2 class MortarStub(object): # missing associated documentation comment in .proto file pass def __init__(self, channel): """Constructor. Args: channel: A grpc.Channel. """ self.GetAPIKey = channel.unary_unary( '/mortar.Mortar/GetAPIKey', request_serializer=mortar__pb2.GetAPIKeyRequest.SerializeToString, response_deserializer=mortar__pb2.APIKeyResponse.FromString, ) self.Qualify = channel.unary_unary( '/mortar.Mortar/Qualify', request_serializer=mortar__pb2.QualifyRequest.SerializeToString, response_deserializer=mortar__pb2.QualifyResponse.FromString, ) self.Fetch = channel.unary_stream( '/mortar.Mortar/Fetch', request_serializer=mortar__pb2.FetchRequest.SerializeToString, response_deserializer=mortar__pb2.FetchResponse.FromString, ) class MortarServicer(object): # missing associated documentation comment in .proto file pass def GetAPIKey(self, request, context): # missing associated documentation comment in .proto file pass context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def Qualify(self, request, context): """identify which sites meet the requirements of the queries """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def Fetch(self, request, context): """pull data from Mortar """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def add_MortarServicer_to_server(servicer, server): rpc_method_handlers = { 'GetAPIKey': grpc.unary_unary_rpc_method_handler( servicer.GetAPIKey, request_deserializer=mortar__pb2.GetAPIKeyRequest.FromString, response_serializer=mortar__pb2.APIKeyResponse.SerializeToString, ), 'Qualify': grpc.unary_unary_rpc_method_handler( servicer.Qualify, request_deserializer=mortar__pb2.QualifyRequest.FromString, response_serializer=mortar__pb2.QualifyResponse.SerializeToString, ), 'Fetch': grpc.unary_stream_rpc_method_handler( servicer.Fetch, request_deserializer=mortar__pb2.FetchRequest.FromString, response_serializer=mortar__pb2.FetchResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( 'mortar.Mortar', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) PK!S00pymortar/result.pyimport sqlite3 import time import pandas as pd def format_uri(uri): if uri.namespace: return uri.namespace+"#"+uri.value else: return uri.value.strip('"') def make_table(_conn, tablename, varnames): c = _conn.cursor() colnames = [] for varname in varnames: varname = varname.lstrip('?') colnames.append( "{0} text".format(varname) ) c.execute("CREATE TABLE {0} ({1}, site text)".format(tablename, ", ".join(colnames))) return _conn """ The result object helps pymortar build from streaming responses to a query, and provides an interface to look at both metadata and timeseries data that is the output of a call to Fetch(...) """ class Result: def __init__(self): """ Returns ------- o: Result A Result object """ # result object has its own sqlite3 in-memory database self.conn = sqlite3.connect(':memory:') self._series = {} self._selections = {} self._df = None self._dfs = {} self._tables = {} def __repr__(self): numtables = len(self._tables) if self._tables else "n/a" selections = self._selections.values() numcols = sum(map(lambda x: len(x.columns), self._dfs.values())) numvals = sum(map(lambda x: x.size, self._dfs.values())) values = [ "collections:{0}".format(numtables), "selections:{0}".format(len(selections)), "timeseries:{0}".format(numcols), "vals:{0}".format(numvals) ] return "".format(" ".join(values)) def describe_table(self, tablename): """ Prints out a description of the table with the provided name Parameters ---------- tablename: string Table name. This will be from the pymortar.Stream object 'name' field. List can be retrieved using Result.tables() Returns ------- n/a (prints out result) """ s = "Columns: {0}".format(' '.join(self._tables.get(tablename, []))) s += "\nCount: {0}".format(self.query("SELECT COUNT(*) FROM {0}".format(tablename))[0][0]) print(s) def add2(self, resp): """ Adds the next FetchResponse object from the streaming call into the current Result object Parameters ---------- resp: FetchResponse This parameter is a FetchResponse object obtained from calling the Mortar Fetch() call. """ if resp.collection not in self._tables and len(resp.variables) > 0: make_table(self.conn, resp.collection, resp.variables) self._tables[resp.collection] = list(map(lambda x: x.lstrip("?"), resp.variables)) self._tables[resp.collection].append("site") if resp.collection in self._tables: c = self.conn.cursor() for row in resp.rows: values = ['"{0}"'.format(format_uri(u)) for u in row.values] values.append('"{0}"'.format(resp.site)) c.execute("INSERT INTO {0} values ({1})".format(resp.collection, ", ".join(values))) if resp.identifier and resp.selection: if resp.selection not in self._selections: self._selections[resp.selection] = {} if resp.identifier not in self._selections[resp.selection]: self._selections[resp.selection][resp.identifier] = [] self._selections[resp.selection][resp.identifier].append( pd.Series(resp.values, index=pd.to_datetime(resp.times), name=resp.identifier) ) def add(self, resp): """ Adds the next FetchResponse object from the streaming call into the current Result object Parameters ---------- resp: FetchResponse This parameter is a FetchResponse object obtained from calling the Mortar Fetch() call. """ if resp.error != "": raise Exception(resp.error) if resp.variable not in self._tables and len(resp.variables) > 0: make_table(self.conn, resp.variable, resp.variables) self._tables[resp.variable] = list(map(lambda x: x.lstrip("?"), resp.variables)) self._tables[resp.variable].append("site") if resp.variable in self._tables: c = self.conn.cursor() for row in resp.rows: values = ['"{0}"'.format(format_uri(u)) for u in row.values] values.append('"{0}"'.format(resp.site)) c.execute("INSERT INTO {0} values ({1})".format(resp.variable, ", ".join(values))) # SELECT * FROM sqlite_master; if resp.identifier: if resp.identifier not in self._series: self._series[resp.identifier] = [] self._series[resp.identifier].append( pd.Series(resp.values, index=pd.to_datetime(resp.times), name=resp.identifier) ) def build(self): if len(self._selections) == 0: self._df = pd.DataFrame() return t = time.time() for selection, timeseries in self._selections.items(): timeseries = self._selections[selection] for uuidname, contents in timeseries.items(): ser = pd.concat(contents) ser = ser[~ser.index.duplicated()] self._selections[selection][uuidname] = ser self._dfs[selection] = pd.concat(self._selections[selection].values(), axis=1, copy=False) t2 = time.time() #print("Building DF took {0}".format(t2-t)) def __getitem__(self, key): if key not in self._selections: return None if key not in self._dfs: self.build() return self._dfs[key] def __contains__(self, key): return key in self._selections def get(self, key, default=None): if key not in self._selections: return default return self[key] @property def collections(self): """ Returns the list of collections in this result. Access collections as SQL tables using Result.query("select * from {collection name}") Returns ------- l: list of str collection names """ return self.tables @property def selections(self): """ Returns the list of selections in this result. Access selections using Result['selection name'] or Result.get('selection name') Returns ------- l: list of str selection names """ return list(self._selections.keys()) @property def tables(self): """ Returns a list of the table names, containing the retrieved metadata Returns ------- l: list of string Each string is a table name """ return list(self._tables.keys()) def vars(self, table): """ Returns a lsit of the column names for the given table Parameters ---------- tablename: string Name of the table Returns ------- l: list of string Column names of the table """ return self._tables.get(table, []) def query(self, q): c = self.conn.cursor() return list(c.execute(q)) PK!HڽTUpymortar-0.3.2.dist-info/WHEEL A н#Z;/"d&F[xzw@Zpy3Fv]\fi4WZ^EgM_-]#0(q7PK!H-{$!pymortar-0.3.2.dist-info/METADATAJ0Ra ];n( nU=&&El)]%wȒP󚭂L ).KoeƠ< -`!+{6$:l||/?UG8w"d VArEZ$}L_vNC|騘4n{)FKǍCcm`}rztFbnZN{Q1lE8l,FE&%Uw#6L*\NFE<'PK!Ht"W pymortar-0.3.2.dist-info/RECORD}̽r0g : K(Ÿ]֡m_g9C6"$5!â%xbGl5Er1ۓMo dK^;ݘ?(88C0%,!;E5 h:F䷏Yq|J,TnΨĮSb5]iέ@ NUMZ?U-sJ\ajzم. =7\^U%Yi&oQwC4S\\|hiGrc؊|zL*㘆jE|l{V[$wjh*x::a.ijt9 U$r[Xb[ PK! V{xpymortar/__init__.pyPK!tnJ pymortar/mortar_pb2.pyPK!.T  6pymortar/mortar_pb2_grpc.pyPK!S00~pymortar/result.pyPK!HڽTUpymortar-0.3.2.dist-info/WHEELPK!H-{$!npymortar-0.3.2.dist-info/METADATAPK!Ht"W pymortar-0.3.2.dist-info/RECORDPKR