PK!@@caroline/__init__.pyimport logging from caroline.config import config from caroline.engine import Base as Prototype from caroline.errors import * name = "caroline" __all__ = [ "Prototype", "CarolineConfigurationError", "CarolineConnectionError", "config", ] logging.getLogger(__name__).addHandler(logging.NullHandler()) PK!::caroline/config.pyimport os from addict import Dict config = Dict() config.default_db = os.environ.get("CAROLINE_DEFAULT_DB", "elasticsearch") config.connection_timeout = os.environ.get("CAROLINE_CONNECTION_TIMEOUT", 0.01) config.redis_default_url = "redis://localhost:6379/0" # TODO: accept additional config vars for elasticsearch besides just URL config.elasticsearch_default_url = "localhost:9200" redis_env_addr = os.environ.get("CAROLINE_REDIS_URL", config.redis_default_url) es_env_addr = os.environ.get( "CAROLINE_ELASTICSEARCH_URL", config.elasticsearch_default_url ) PK!caroline/databases/__init__.pyPK!S #caroline/databases/elasticsearch.pyimport logging import elasticsearch from elasticsearch import Elasticsearch from caroline.config import config from caroline.config import es_env_addr from caroline.errors import CarolineConnectionError log = logging.getLogger(__name__) # While we're at it, Elasticsearch doesn't use connection pools the same way # that Redis does, so we'll set up the client here and have all the active ES # models use it. We won't check to see if it's actually valid unless it's called. # The timeout is also set extremely low because otherwise the library will hang # if elasticsearch is not running; for a system that uses Redis only, then we want # it to fail quickly if a connection is not immediately detected. This can be # adjusted by changing the environment variable `CAROLINE_CONNECTION_TIMEOUT`. es_caroline_connector = Elasticsearch( es_env_addr, timeout=config.connection_timeout, max_retries=0 ) class elasticsearch_db(object): def __init__(self, es_conn=None): try: if es_conn: self.es = es_conn else: # we didn't get anything, so we'll piggyback off of the connection # that caroline creates self.es = es_caroline_connector log.debug("Trying to connect to Elasticsearch...") self.es.info() log.debug("Connection complete!") except elasticsearch.exceptions.ConnectionError: # Elasticsearch client raises _a lot_ of nested errors in this instance. # Nuke them all with extreme prejudice. raise CarolineConnectionError( "Cannot reach Elasticsearch! Is it running?" ) from None def load(self, scope, key): try: result = self.es.get( index="caroline", doc_type=scope.db_key_unformatted, id=key ) # we don't need the whole response from Elasticsearch; we only need the # body that we set. result = result.get("_source") except elasticsearch.exceptions.NotFoundError: log.debug("ES key {} not found, returning None.".format(key)) result = None return result def save(self, scope): self.es.index( index="caroline", doc_type=scope.db_key_unformatted, id=scope.record_id, body=scope.to_dict(), ) def all_keys(self, scope): raise NotImplementedError( "This is only valid for Redis databases at the moment." ) PK!pj]caroline/databases/redis.pyimport json import logging import redis from caroline.config import redis_env_addr from caroline.errors import CarolineConfigurationError from caroline.errors import CarolineConnectionError log = logging.getLogger(__name__) # we don't actually connect to redis until we use the object; putting it here # means that we have the connection pool available if we need it. # Let's see if we have a Redis environment variable set! pool = redis.ConnectionPool.from_url(redis_env_addr) class redis_db(object): def __init__(self, redis_conn=None): try: if redis_conn: # We have something -- we'll run it through the same testing code to # make sure that it works. self.r = redis_conn else: self.r = redis.Redis(connection_pool=pool) self.r.ping() except redis.exceptions.ConnectionError: raise CarolineConnectionError("Unable to reach Redis.") except Exception as e: raise CarolineConfigurationError( "Caught {} -- please pass in an instantiated Redis " "connection.".format(e) ) def load(self, scope, key): """ :return: Dict or None; the loaded information from Redis. """ result = self.r.get(scope.db_key.format(key)) if not result: log.debug("Redis key {} not found, returning None.".format(key)) return None return json.loads(result.decode()) def save(self, scope): self.r.set(scope.db_key.format(scope.record_id), json.dumps(scope.data)) def all_keys(self, scope): return self.r.scan_iter("{}*".format(scope.db_key)) PK!oQcaroline/engine.pyimport logging from jsonschema import validate from caroline.config import config from caroline.databases.elasticsearch import elasticsearch_db from caroline.databases.redis import redis_db from caroline.errors import CarolineConfigurationError log = logging.getLogger(__name__) def validate_config(): if config.default_db not in ["redis", "elasticsearch"]: raise CarolineConfigurationError( "Default DB has been changed to invalid option; use either " '"elasticsearch" or "redis"' ) class Base(object): """ Welcome to the weirdness that is Caroline. Caroline is a ODM that specializes in all things json, because frankly I really don't like working with ORMs and the way that they're normally handled in Python (thanks, Django!). It's very simple: you create a class using a minimum of two objects: * A dict of what you want your data to look like * a dict of valid jsonschema that will be used to validate your data on save That's it. No models, no craziness, minor setup time, and hopefully pretty simple to use. That's the goal, anyways. Why Caroline? Because we can. Also, here's a song list of tracks you may or may not already be familiar with, arranged by release year! * Caroline - Steep Canyon Rangers (2017) * Caroline I See You - James Taylor (2002) * Caroline - Fleetwood Mac (1987) * Oh Caroline - Cheap Trick (1977) * Sweet Caroline - Neil Diamond (1965) """ def __init__(self, record_id): """ Everything that we should need is passed in by the user and found under the `self` object. Here's what we should be seeing: class User(Prototype): default = {valid dict} # optional flags schema = {valid jsonschema} db_key = "user-obj" redis_conn = r OR elasticsearch_conn = e The schema is technically optional, but we want people to use it. Why else use a library like Caroline? Because the user creates the class with those variables defined, we can structure the parent around them. Fun! """ # First things first! What DB are we using? Gonna do this the long # way for legibility purposes and because the hasattr call is not # expensive. db_map = {"elasticsearch": elasticsearch_db, "redis": redis_db} if hasattr(self, "redis_conn") and hasattr(self, "elasticsearch_conn"): raise CarolineConfigurationError( "Received both a Redis connection and an Elasticsearch connection. " "You need to use one or the other -- Caroline does not support " "handling both at the same time." ) if hasattr(self, "redis_conn"): self.db = redis_db(self.redis_conn) if hasattr(self, "elasticsearch_conn"): self.db = elasticsearch_db(self.elasticsearch_conn) if not hasattr(self, "db"): try: self.db = db_map[config.default_db]() except KeyError: raise CarolineConfigurationError( "Did not receive db connection in model and environment variable " "points towards an invalid location. " "Valid connections are: {}".format(", ".join([x for x in db_map])) ) if isinstance(self.db, str): if self.db in db_map: log.debug(f"Overriding defaults with requested db base {self.db}") self.db = db_map[self.db]() else: raise CarolineConfigurationError( "The requested db {} is not available as an option. Usable " "options are: {}".format(self.db, ", ".join([x for x in db_map])) ) if not hasattr(self, "default"): log.warning( "Did not receive a default dict; no default attributes will be " "applied to the model {}. Please define a `default` attribute " "on your model in order for values to be assigned appropriately " "on creation.".format(self.__class__.__name__) ) self.default = {} if not isinstance(self.default, dict): raise CarolineConfigurationError("default must be a dict!") if not hasattr(self, "db_key"): # if we don't have a db_key passed in, then we use the name of the # class that the developer defined as the key. self.db_key = self.__class__.__name__.lower() log.debug('No db_key passed; defaulting to key "{}"'.format(self.db_key)) else: self.db_key = str(self.db_key) # back up the original key in case we're using elasticsearch so that we can # use it for the object type. self.db_key_unformatted = self.db_key # note: this is a way of allowing us to only format the first field. # It'll render out as "::thing::{}" which we can then format again. self.db_key = "::{}::{{}}".format(self.db_key) if not hasattr(self, "schema"): self.schema = None self.record_id = record_id result = self._load(self.record_id) if result: self.data = result else: self.data = self.default def __repr__(self): return repr(self.data) def __len__(self): return len(self.data) def __getitem__(self, item): return self.data[item] def __setitem__(self, key, value): self.data[key] = value def __delitem__(self, key): del self.data[key] def get(self, key, default_return=None): return self.data.get(key, default_return) def _load(self, requested_key): """ :return: Dict or None; the loaded information from the db. """ return self.db.load(scope=self, key=requested_key) def save(self): if self.validate(): self.db.save(scope=self) def update(self, key, value): self.data[key] = value def to_dict(self): return self.data def validate(self): # validate will return None if it succeeds or throw an exception, so if # we get to the return statement then we're good. # Alternatively, they can just not give us a schema -- in which case, # just return True and don't sweat it. if self.schema: validate(self.data, self.schema) return True def upgrade(self): """ Use this when you've got existing keys in your db and you need to change the defaults and the schema. This will merge the new fields with the default values set in the model every time a key is loaded. Upgrade-as-you-go, if you will. :return: None """ # first we add in all the new fields that may have been set in the default # dict for key in self.default: if key in self.to_dict(): continue self.update(key, self.default[key]) # now we nuke any additional fields that may have been removed from the # default dict. keys_to_remove = list() for key in self.to_dict(): if key not in self.default: keys_to_remove.append(key) if len(keys_to_remove) > 0: for k in keys_to_remove: del self[k] def all_keys(self): # returns a generator that the developer can handle how they wish -- this # will return all of the keys matching this model type currently stored in # the db. Currently only works when using the Redis connection. return self.db.all_keys(self) PK!,Ocaroline/errors.pyclass Error(Exception): pass class CarolineConfigurationError(Error): pass class CarolineConnectionError(Error): pass PK!'ݼcaroline/test_engine.pyfrom unittest.mock import MagicMock from unittest.mock import patch # noinspection PyUnresolvedReferences import elasticsearch import pytest # noinspection PyUnresolvedReferences import redis from caroline.engine import Base from caroline.errors import CarolineConfigurationError @patch("caroline.engine.Base._load", return_value={"hello": "world"}) def test_generic_launch(a): class x(Base): redis_conn = MagicMock() default = {} y = x("asdf") assert y.to_dict() == {"hello": "world"} @patch("caroline.engine.Base._load", return_value=None) def test_default_loading(a): class x(Base): redis_conn = MagicMock() default = {"yo": "world"} y = x("asdf") # asdf didn't come back with anything because _load failed, so it should # be the default structure. assert y.to_dict() == {"yo": "world"} @patch("caroline.engine.Base._load", return_value=None) def test_redis_key_var(a): class x(Base): redis_conn = MagicMock() default = {"yo": "world"} db_key = "snarfleblat" y = x("asdf") assert y.db_key == "::snarfleblat::{}" assert y.db_key_unformatted == "snarfleblat" @patch("caroline.engine.Base._load", return_value=None) def test_update_methods(a): class x(Base): redis_conn = MagicMock() default = {"yo": "world"} y = x("asdf") assert y.to_dict() == {"yo": "world"} y.update("yo", "hello there") assert y.to_dict() == {"yo": "hello there"} y["yo"] = "general kenobi" assert y.to_dict() == {"yo": "general kenobi"} @patch("caroline.engine.Base._load", return_value=None) def test_optional_default(a): class x(Base): redis_conn = MagicMock() y = x("asdf") assert y.default == {} def test_multiple_dbs_configured(): with pytest.raises(CarolineConfigurationError): class x(Base): redis_conn = MagicMock() elasticsearch_conn = MagicMock() y = x("asdf") PK!?$Ɩ44 caroline-0.4.2.dist-info/LICENSEMIT License Copyright (c) 2018 Grafeas Group, Ltd. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PK!H+dUTcaroline-0.4.2.dist-info/WHEEL HM K-*ϳR03rOK-J,/R(O-)T0343 /, (-JLR()*M IL*4KM̫PK!HjR"l !caroline-0.4.2.dist-info/METADATAXas6_8sgkF;uNα=7n;7$x(Y "_rtR .v]gd**M1X$2.TԾۋUZűUݥj5JRwbBHTr)Q@e5r J;]V3%mlytUXl:ZLڱTPe:X%d14inaѫM] HRgc[KZWʎ>޽<:?<cL+h>UK2WWNڊX<#N)p8SD  {{չ8.,!:fϛ=yż;@Oߺ/Ň,}Oڔ:ax}<2qSDUɨ7;֍LS8h?p oIL*(YOB(,V2IhMar-)Ȗ޶iŀJˋTڙj=lKj⫧qbݐsݶOM*]:PҺD9XMͪ~Zgi(?v`RQvC!;1i%dqXU I~@p4kFZ:qfKvT>{zx?Iz0{dz<=ߛ{Yz8Bl&rQ[YT]~cM]%h_e^79VGYxq Rϋ#}o8 `!8@x;JXΐL&{u>B6k/{,1bQ*vԊ̧]'~NO}/`_9 "8F% Z;ֿUmL4 WicB:zh2: OVN¸J3TA75X;Aab ѽJO)0?,3+`jRZ8FB0aaY-1jj܉RPl o"9 :j4 a>^rϘ:-:%0Pܲ+ڸ‚̘[҂Ca Wۑ\\2cɉT% 78^W.d` ٿf]`glj'z' A~kV"5B)*b 6X=rYe6K H8-XO" /)A}"ʎh2ڣ/"f.Ƒ_fΜ8oۚVpdk,EtVx/}CqL N#FgDк l6|x󩪾$xD'o>m#3q؞/XߖhUaX( =D[W4@| Ji[ 0JC6V`Mi- [luOmaBk$jv(_l X;S*3F Z1cE GDw@Ho,%D<6+ԇ볱tL^pA.wώęDglorl>`SERCfƟ6cx%b=lHwFSSE+d Af`Lj(I&y[fOֳhH} cC=B n YtXS hU5͖ ,ey&_8wvsz^iٶzI%+nŠA J.X=s ~qaˢ"iUWRX9q~g^8IH`dLRؙ\balQ W~LCPF:=,Bzwdw4ťْRb<;*b&b&ʍ7`8O'NnE]6Hf ,_ $9 F7XNju9{],'#e&IH]zH`NSn3PT(,|;K1Da`-<-; YJyb"  rk4iӦ_P ӓ\; 6eXf/Nߞ_ w,NϮ/9>L_\Q Seg͢2?|r=lsm֝CnS$bmU#MB ]fɷ7zH0Є<wXJvRZ Mi⣙%h]NgV 8?=lj<}ؙ&3jw)ZP5s`PK!HF?<caroline-0.4.2.dist-info/RECORDuKs@}~ 6 YKD@PZA@IVwթsnmp1WL e${а0}UFhlg*8Or ·u+4|lҽ܉Hu\TP6G]W`A}/S:jOy{U2( iSn>hLfҘc_2onLSm,Jn|&PL *<iif8VzU6wc')|w_;|lZrr'eu4m?4PS/=Ԁb\b${{m/l)1H׿o䍿r۫{(CS})k GIgȨtsIeWLoPK!@@caroline/__init__.pyPK!::rcaroline/config.pyPK!caroline/databases/__init__.pyPK!S #caroline/databases/elasticsearch.pyPK!pj]Pcaroline/databases/redis.pyPK!oQ@caroline/engine.pyPK!,O4caroline/errors.pyPK!'ݼ4caroline/test_engine.pyPK!?$Ɩ44 <caroline-0.4.2.dist-info/LICENSEPK!H+dUT8Acaroline-0.4.2.dist-info/WHEELPK!HjR"l !Acaroline-0.4.2.dist-info/METADATAPK!HF?<Mcaroline-0.4.2.dist-info/RECORDPK cO