PKWMk55tormor/__init__.py"""Database migration helper""" __version__ = "1.2" PKNVMrK} tormor/cmdline.pyimport csv from tormor.connection import execute_sql_file, Connection, SchemaNotPresent from tormor.schema import enablemodules, migrate from psycopg2 import ProgrammingError import os import sys from getopt import gnu_getopt SHORTOPTS = "?d:h:p:U:P:" PGOPTMAP = { "-d": ("dbname", "PGDATABASE"), "-h": ("host", "PGHOST"), "-p": ("port", "PGPORT"), "-U": ("user", "PGUSER"), "-P": ("password", "PGPASSWORD") } HELPTEXT = """Tormor -- Migration management tormor [opts] command [args] opts are one or more of: -? Print this text -h hostname Postgres host -d database Database name -U username Database username (role) -p port Database port -P password Database password (Not recommended, use environment variable PGPASSWORD instead) Command is one of: enable-modules mod1 [mod2 [mod3 ...]] Enable the modules. help Show this text include filename Find commands (one per line) in the specified file and run them migrate Run all migrations. sql filename Load the filename and present the SQL in it to the database for execution. This is useful for choosing migrations scripts to run. """ def makedsn(opts, args): dsnargs = {} for arg, (opt, env) in PGOPTMAP.items(): if arg in opts: dsnargs[opt] = opts[arg] elif os.getenv(env): dsnargs[opt] = os.getenv(env) return " ".join(["%s='%s'" % (n, v) for n, v in dsnargs.items()]) def include(cnx, filename): with open(filename, newline="") as f: lines = csv.reader(f, delimiter=" ") for line in lines: if len(line) and not line[0].startswith("#"): command(cnx, *[p for p in line if p]) COMMANDS = { "enable-modules": enablemodules, "migrate": migrate, "include": include, "sql": execute_sql_file, } class UnknownCommand(Exception): pass def command(cnx, cmd, *args): if cmd in COMMANDS: COMMANDS[cmd](cnx, *args) else: raise UnknownCommand(cmd) def script(): optlist, args = gnu_getopt(sys.argv, SHORTOPTS) opts = dict(optlist) if len(args) < 2: print("No command given. Type 'tormor help' for help") exit(3) elif '-?' in opts or args[1] == 'help': print(HELPTEXT) else: try: dsn = makedsn(opts, args) cnx = Connection(dsn) command(cnx, *(args[1:])) cnx.commit() except UnknownCommand as e: print("Unknown command '{}'. Type 'tormor help' for help".format(e)) exit(1) except ProgrammingError as e: print("** Postgres error", e.pgcode) print() print(e) exit(2) except SchemaNotPresent: print("This database does not have the Tormor schemas installed") print("Use `tormor enable-modules` to bootstrap it") exit(3) PKWM tormor/connection.pyimport base64 import os from datetime import datetime, timezone from getpass import getpass import psycopg2 from psycopg2.extensions import ISOLATION_LEVEL_SERIALIZABLE EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) class SchemaNotPresent(Exception): """Thrown if we can't load the list of modules because the `modules` table doesn't exist. """ pass class ModuleNotPresent(Exception): """ Thrown when a required module has not been installed. """ pass class Connection(object): def __init__(self, dsn): try: self.pg = psycopg2.connect(dsn) except psycopg2.OperationalError as e: if "fe_sendauth" in str(e): pwd = getpass() dsn = dsn + " password='%s'" % pwd self.pg = psycopg2.connect(dsn) else: raise e self.pg.set_session(isolation_level=ISOLATION_LEVEL_SERIALIZABLE) self.cursor = self.pg.cursor() self._modules = set() def load_modules(self): if not self._modules: try: self.cursor.execute("SELECT name FROM module") self._modules = set([m[0] for m in self.cursor.fetchall()]) except psycopg2.ProgrammingError as e: if e.pgcode == "42P01": raise SchemaNotPresent() else: raise return self._modules def assert_module(self, module): """ Stops execution with an error if a required module is not installed in the system. """ if not self.has_module(module): raise ModuleNotPresent(module) def has_module(self, module): """ Returns true if the module has been installed. """ return module in self.load_modules() def execute(self, cmd, *args, **kwargs): """ Execute the SQL command and return the data rows as tuples """ self.cursor.execute(cmd, *args, **kwargs) def select(self, cmd, *args, **kwargs): """ Execute the SQL command and return the data rows as tuples """ self.cursor.execute(cmd, *args, **kwargs) return self.cursor.fetchall() def commit(self): self.pg.commit() def execute_sql_file(cnx, filename): try: with open(filename) as f: cmds = f.read() cnx.cursor.execute(cmds) cnx.load_modules() print("Executed", filename) except Exception: print("Error whilst running", filename) raise def execute_migration(cnx, module, migration, filename): try: with open(filename) as f: cmds = """ INSERT INTO module (name) VALUES('{module}') ON CONFLICT (name) DO NOTHING; INSERT INTO migration (module_name, migration) VALUES('{module}', '{migration}') ON CONFLICT (module_name, migration) DO NOTHING; {cmds} """.format( module=module, migration=migration, cmds=f.read() ) cnx.cursor.execute(cmds) cnx.load_modules() print("Executed", filename) except Exception: print("Error whilst running", filename) raise PKWM4 tormor/schema.pyimport os from tormor.connection import SchemaNotPresent, execute_migration MYDIR = os.path.dirname(os.path.abspath(__file__)) ADD_MODULE = """INSERT INTO module VALUES (%s)""" BOOTSTRAP_SQL = """ CREATE TABLE module ( name text NOT NULL, CONSTRAINT module_pk PRIMARY KEY(name) ); CREATE TABLE migration ( module_name text NOT NULL, CONSTRAINT migration_module_fkey FOREIGN KEY (module_name) REFERENCES module (name) MATCH SIMPLE ON UPDATE NO ACTION ON DELETE NO ACTION DEFERRABLE, migration text NOT NULL, CONSTRAINT migration_pk PRIMARY KEY (module_name, migration) ); """ class SchemaFilesNotFound(Exception): pass def _get_schema_files_path(): env_schema_path = os.getenv("SCHEMA_PATH") if env_schema_path: return env_schema_path.split(os.pathsep) return [os.path.join(MYDIR, "../../Schema")] DEFAULT_SCHEMA_FILES_PATH = _get_schema_files_path() def find_schema_paths(schema_files_path=DEFAULT_SCHEMA_FILES_PATH): """Searches the locations in the `SCHEMA_FILES_PATH` to try to find where the schema SQL files are located. """ paths = [] for path in schema_files_path: if os.path.isdir(path): paths.append(path) if paths: return paths raise SchemaFilesNotFound("Searched " + os.pathsep.join(schema_files_path)) def enablemodules(cnx, *modules): want = set(modules) try: got = cnx.load_modules() except SchemaNotPresent: cnx.pg.rollback() print("Tormor not loaded for this database. Bootstrapping...") cnx.execute(BOOTSTRAP_SQL) got = cnx.load_modules() for mod in want.difference(got): cnx.execute(ADD_MODULE, (mod,)) print(mod, "enabled") cnx._modules = set() def migrate(cnx): ## Find and execute all migration scripts that haven't already been run paths = find_schema_paths() migrations = set(cnx.select("SELECT module_name, migration FROM migration")) for path in paths: scripts = [] for dirname, subdirs, files in os.walk(path): relpath = os.path.relpath(dirname, path) if relpath != "." and relpath in cnx.load_modules(): scripts += [(relpath, f) for f in files if f.endswith(".sql")] scripts.sort(key=lambda m: m[1]) for (mod, migration) in scripts: if (mod, migration) not in migrations: execute_migration(cnx, mod, migration, os.path.join(path, mod, migration)) PK!H$)0%tormor-1.2.dist-info/entry_points.txtN+I/N.,()*//Pzɹ)9yVi..PK!AM 55tormor-1.2.dist-info/LICENSEMIT License Copyright (c) 2018 Proteus Technologies Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PK!H>*RQtormor-1.2.dist-info/WHEEL HM K-*ϳR03rOK-J,/RH,rzd&Y)r$[)T&UrPK!Htormor-1.2.dist-info/METADATAMSMo@)`[ $KDiQ+4UE6^e?(Ʌ Yޛ7zpϳh0q~Ŗ\a Xe,;g1[=pMMh-T-5 y߻(ZỰkk<yb`o_K6 3J"vA IEq#ae$']mEe3=jU>N{_,l&sb+X9Ppi[M/S?pjӷl#4Ly_ECU7{oy 6AfҮn@ 丑7}N(p}kf7iu;)u;dzϜOܶ9c֨FbL$5q͸>5;!8eK*ODRG< ǍP [ IjFZ/_I8%^Z]ĉ:.I}I ).9G6"/7 J@=^NJ~t3)pDX40Tp@y 2e]:4n">PK!H,Ctormor-1.2.dist-info/RECORDuI@@},FE/AE TM )C*; ^w >!D-".՗ '.yLv(&֜ʁ]:i5m(SқnG MV6qex+IP7^Y^*R<#J?n<%/%"7SX僲c?5,YR} S<'di7 7B^ b4jܪ4\ ʼn,POh3oI7+F-d$O}2FX:e(궳 ~+ fJES(Azo[;UwI46}v{6Y}Ԗch8ң(Yޣ[Ov%М>ZKag@R2**OJ~0\#;R#?$>Lu;S$K.ߣ_PKWMk55tormor/__init__.pyPKNVMrK} etormor/cmdline.pyPKWM  tormor/connection.pyPKWM4 ctormor/schema.pyPK!H$)0%`#tormor-1.2.dist-info/entry_points.txtPK!AM 55#tormor-1.2.dist-info/LICENSEPK!H>*RQ;(tormor-1.2.dist-info/WHEELPK!H(tormor-1.2.dist-info/METADATAPK!H,C+tormor-1.2.dist-info/RECORDPK x-