PKѶPO55tormor/__init__.py"""Database migration helper""" __version__ = "2.3" PK϶POctormor/commands.pyfrom tormor.exceptions import SchemaNotPresent from tormor.path_helper import get_schema_path import csv import click import os import warnings # String of queries to add module ADD_MODULE = """INSERT INTO module(name) VALUES($1);""" # String of queries to create table 'module' and 'migration' BOOTSTRAP_SQL = """ CREATE TABLE module ( name text NOT NULL, CONSTRAINT module_pk PRIMARY KEY(name) ); CREATE TABLE migration ( module_name text NOT NULL, CONSTRAINT migration_module_fkey FOREIGN KEY (module_name) REFERENCES module (name) MATCH SIMPLE ON UPDATE NO ACTION ON DELETE NO ACTION DEFERRABLE, migration text NOT NULL, CONSTRAINT migration_pk PRIMARY KEY (module_name, migration) ); """ @click.group() def subcommand(): pass @subcommand.command('migrate') @click.pass_context @click.option('--dry-run', is_flag=True) @click.argument('modules', required=False, nargs=-1) def migrate(ctx, dry_run, modules): """Run all migrations""" modules_to_be_added = set(modules) conn = ctx.obj['cnx'] paths = get_schema_path() try: migrated_modules = conn.load_modules() except SchemaNotPresent: conn.execute(BOOTSTRAP_SQL) migrated_modules = conn.load_modules() to_be_run_scripts = [] query = "" for each_path in paths: for root, dirs, files in os.walk(each_path): relpath = os.path.relpath(root, each_path) if relpath != "." and relpath in modules_to_be_added: to_be_run_scripts += [(relpath, filepath, each_path) for filepath in files if filepath.endswith(".sql")] to_be_run_scripts.sort(key=lambda m: m[1]) for (module, migration, path) in to_be_run_scripts: if (module, migration) not in migrated_modules: query += get_migrate_sql(module, migration, os.path.join(path, module, migration)) if query: if not dry_run: print("/*Migrating modules...*/") conn.execute(query) print("/*Successfully migrated modules*/") else: print(query) else: warnings.warn("migrate will be deprecated in next version, use migrate [modules...] instead", DeprecationWarning) @subcommand.command('enable-modules') @click.pass_context @click.option('--dry-run', is_flag=True) @click.argument('modules', required=True, nargs=-1) def enable_modules(ctx, dry_run, modules): """Enable modules""" ctx.invoke(migrate, dry_run = dry_run, modules = modules) warnings.warn("enable-modules will be deprecated in next version, use migrate [modules...] instead", DeprecationWarning) @subcommand.command('sql') @click.pass_context @click.argument('sqlfile', nargs=1) def execute_sql_file(ctx, sqlfile): """ Execute SQL queries in files, useful for running migration scripts """ try: conn = ctx.obj['cnx'] with open(sqlfile) as f: commands = f.read() conn.execute(commands) print("/*", sqlfile, "successfully executed*/") except Exception: print("Error whilst running", sqlfile) raise @subcommand.command() @click.pass_context @click.argument('filename', required=True, nargs=1) def include(ctx, filename): """Run all commands inside a file""" with open(filename, newline="") as f: lines = csv.reader(f, delimiter=" ") for each_line in lines: if len(each_line) and not each_line[0].startswith("#"): cmd = each_line.pop(0) if cmd == "migrate": if each_line[len(each_line)-1] == '--dry-run': each_line.pop(len(each_line)-1) ctx.invoke(migrate, dry_run = True, modules = each_line) else: ctx.invoke(migrate, dry_run = False, modules = each_line) elif cmd == "sql" and len(each_line) == 1: ctx.invoke(execute_sql_file, sqlfile = each_line[0]) else: raise click.ClickException("Unknown command or parameter") def get_migrate_sql(module, migration, filename): try: with open(filename) as f: commands = """ INSERT INTO module (name) VALUES('{module}') ON CONFLICT (name) DO NOTHING; INSERT INTO migration (module_name, migration) VALUES('{module}', '{migration}') ON CONFLICT (module_name, migration) DO NOTHING; {cmds} """.format( module=module, migration=migration, cmds=f.read() ) print("/*Read", filename, "*/") return commands except Exception: print("Error whilst running", filename) raise PK2%OaBtormor/connections.pyimport asyncpg import asyncio from tormor.exceptions import SchemaNotPresent class Connection(object): def __init__(self, dsn): try: self.loop = asyncio.get_event_loop() self.conn = self.loop.run_until_complete(asyncpg.connect(dsn)) self._modules = set() except Exception as e: raise e async def execute_in_transaction(self, sql_queries): async with self.conn.transaction(isolation='serializable'): await self.conn.execute(sql_queries) def execute(self, sql_queries): self.loop.run_until_complete(self.execute_in_transaction(sql_queries)) async def fetch_in_transaction(self, sql_queries): async with self.conn.transaction(isolation='serializable'): result = await self.conn.fetch(sql_queries) return result def fetch(self, sql_queries): return self.loop.run_until_complete(self.fetch_in_transaction(sql_queries)) def load_modules(self): try: name_records = self.fetch("SELECT name FROM module") self._modules = set(each_record['name'] for each_record in name_records) except asyncpg.UndefinedTableError: raise SchemaNotPresent return self._modules def assert_module(self, module): if module not in self.load_modules(): raise ModuleNotFoundError def close(self): self.loop.run_until_complete(self.conn.close()) PKO$p tormor/dsn.pyimport os # Dictionary of possible options PGOPTMAP = { "-d": ("dbname", "PGDATABASE", "/dbname"), "-h": ("host", "PGHOST", "@host"), "-p": ("port", "PGPORT", ":port"), "-U": ("user", "PGUSER", "user"), "-P": ("password", "PGPASSWORD", ":password") } # Create a string of "mapped_filed = value" def makeDSN(destination): template = "postgresql://user:password@host:port/dbname" for option, (field, env, placeholder) in PGOPTMAP.items(): if destination[option]: template = template.replace(field, destination[option]) elif os.getenv(env): template = template.replace(field, os.getenv(env)) else: template = template.replace(placeholder, "") return templatePKO(<tormor/exceptions.pyclass SchemaFilesNotFound(Exception): pass class SchemaNotPresent(Exception): pass class ModuleNotPresent(Exception): pass class SchemaPathNotFound(Exception): passPK϶PO8I tormor/main_script.pyimport click from tormor.dsn import makeDSN from tormor.commands import migrate, execute_sql_file, include from tormor.connections import Connection import csv @click.group() @click.option('-h', nargs=1, required=False, help="Postgres host") @click.option('-d', nargs=1, required=False, help="Database name") @click.option('-u', '-U', nargs=1, required=False, help="Database username (role)") @click.option('-p', nargs=1, required=False, help="Database port") @click.option('-password', '-P', nargs=1, required=False, help="Database password (Not recommended, use environment variable PGPASSWORD instead)") @click.pass_context def script(ctx, h, d, u, p, password): cnx_destination = { '-h': h, '-d': d, '-U': u, '-p': p, '-P': password } dsn = makeDSN(cnx_destination) ctx.obj = {'cnx': Connection(dsn)} script.add_command(migrate) script.add_command(execute_sql_file) script.add_command(include)PKO7P(VVtormor/path_helper.pyfrom tormor.exceptions import SchemaPathNotFound import os def get_schema_path(): mydir = os.getcwd() path = [] env_schema_path = os.getenv("SCHEMA_PATH") if env_schema_path: return env_schema_path.split(os.pathsep) else: for root, dirs, files in os.walk(mydir): if "schema" in root.lower() and _check_not_subfolder(path, root): path.append(root) if not path: raise SchemaPathNotFound return _check_valid_path(path) def _check_valid_path(paths): valid_path = [] for each_path in paths: if os.path.isdir(each_path): valid_path.append(each_path) return valid_path def _check_not_subfolder(main_folders, sub_folders): for any_folder in main_folders: if any_folder in sub_folders: return False return TruePKOprrtormor/tests/bootstrap.sh#!/bin/bash set -e echo "Creating database for testing..." createdb -h localhost -U postgres -O postgres tormordbPKO9; tormor/tests/cleanup.sh#!/bin/bash set -e echo "Cleaning any existing tormor test database..." dropdb -h localhost -U postgres tormordb || echo "Tormor database hasn't been created before" PK϶POY0.33tormor/tests/script_file.txt# Enable modules migrate customer product employeePK϶PO\ == tormor/tests/script_file_dry.txt# Enable modules migrate customer product employee --dry-runPK϶POfߵ tormor/tests/test_connections.pyfrom tormor.connections import Connection import asyncpg from asyncpg.exceptions import InvalidCatalogNameError, UndefinedTableError import pytest import asyncio from tormor.exceptions import ModuleNotPresent SQL_TO_CREATE_INSERT_MANY_MODULES = ''' CREATE TABLE module( name text NOT NULL, CONSTRAINT module_pk PRIMARY KEY(name)); INSERT INTO module(name) VALUES('Module1'); INSERT INTO module(name) VALUES('Module2'); INSERT INTO module(name) VALUES('Module3'); ''' def test_connection_working(): conn = Connection('postgresql://localhost/tormordb') conn.close() def test_connection_not_working(): with pytest.raises(InvalidCatalogNameError): conn = Connection('postgresql://localhost/tormordb_none') conn.close() class TestConnection: def setup(self): self.conn = Connection('postgresql://localhost/tormordb') self.conn.execute("DROP TABLE IF EXISTS module") self.conn.execute(SQL_TO_CREATE_INSERT_MANY_MODULES) def teardown(self): self.conn.execute("DROP TABLE module") self.conn.close() def test_fetch(self): result = self.conn.fetch("SELECT name FROM module") expected_result = {"Module1", "Module2", "Module3"} actual_result = set(record.get("name") for record in result) assert actual_result == expected_result def test_load_module(self): actual_result = self.conn.load_modules() expected_result = {'Module1', 'Module2', 'Module3'} assert actual_result == expected_result def test_assert_module_exist(self): self.conn.assert_module("Module1") def test_assert_module_not_exist(self): with pytest.raises(ModuleNotFoundError): self.conn.assert_module("none") def test_transaction_rollback(self): with pytest.raises(UndefinedTableError): result = self.conn.fetch("SELECT name FROM module_none")PKO22tormor/tests/test_dsn.pyfrom tormor.dsn import makeDSN def test_make_dsn(): opts = { '-h': 'localhost', '-d': 'tormordb', '-U': 'tormor', '-p': '8000', '-P': 'tormor' } expected_dsn = "postgresql://tormor:tormor@localhost:8000/tormordb" assert makeDSN(opts) == expected_dsnPKOOtormor/tests/test_path.pyfrom tormor.path_helper import get_schema_path from tormor.exceptions import SchemaPathNotFound import pytest, os def test_get_schema_path_exist(): current_path = os.path.dirname(os.path.abspath(__file__)) schema_path = os.path.join(current_path, "Schema") os.environ['SCHEMA_PATH'] = schema_path expected_result = schema_path actual_result = get_schema_path() os.environ.pop('SCHEMA_PATH') assert actual_result == [expected_result] def test_get_schema_path_exist_not_exist(): paths = get_schema_path() match = [each_path for each_path in paths if "/test/Schema2" or "/test/Schema" in each_path] assert matchPK϶POOatormor/tests/test_run_script.pyfrom tormor.main_script import script import click from click.testing import CliRunner from click.exceptions import UsageError from tormor.connections import Connection from tormor.commands import BOOTSTRAP_SQL import pytest class TestScript(): def setup(self): self.runner = CliRunner() self.conn = Connection('postgresql://localhost/tormordb') self.conn.execute("DROP TABLE IF EXISTS migration;") self.conn.execute("DROP TABLE IF EXISTS module;") self.conn.execute("DROP TABLE IF EXISTS customer;") self.conn.execute("DROP TABLE IF EXISTS employee;") self.conn.execute("DROP TABLE IF EXISTS product;") self.conn.execute(BOOTSTRAP_SQL) def teardown(self): self.conn.execute("DROP TABLE IF EXISTS migration;") self.conn.execute("DROP TABLE IF EXISTS module;") self.conn.execute("DROP TABLE IF EXISTS customer;") self.conn.execute("DROP TABLE IF EXISTS employee;") self.conn.execute("DROP TABLE IF EXISTS product;") self.conn.close() def test_script_to_invalid_command(self): result = self.runner.invoke(script, ['--xyz']) assert result.exit_code == click.UsageError.exit_code def test_script_to_migrate(self): self.runner.invoke(script, ['-h', 'localhost', '-d', 'tormordb', 'migrate', 'customer']) table_exists = self.conn.fetch(""" SELECT EXISTS (SELECT 1 FROM pg_tables WHERE schemaname = 'public' AND tablename = 'customer'); """) assert table_exists[0][0] == True customers = self.conn.fetch(""" SELECT * FROM public.customer; """) actual_result = set(record.get("name") for record in customers) expected_result = {"Customer1", "Customer2", "Customer3", "Customer5"} assert actual_result == expected_result def test_script_to_migrate_multiple_module(self): self.runner.invoke(script, ['-h', 'localhost', '-d', 'tormordb', 'migrate', 'employee', 'customer']) customer_table_exists = self.conn.fetch(""" SELECT EXISTS (SELECT 1 FROM pg_tables WHERE schemaname = 'public' AND tablename = 'customer'); """) assert customer_table_exists[0][0] == True enployee_table_exists = self.conn.fetch(""" SELECT EXISTS (SELECT 1 FROM pg_tables WHERE schemaname = 'public' AND tablename = 'employee'); """) assert enployee_table_exists[0][0] == True customers = self.conn.fetch(""" SELECT * FROM public.customer; """) actual_result = set(record.get("name") for record in customers) expected_result = {"Customer1", "Customer2", "Customer3", "Customer5"} assert actual_result == expected_result employees = self.conn.fetch(""" SELECT * FROM public.employee; """) actual_result = set(record.get("name") for record in employees) expected_result = {"Employee1", "Employee2", "Employee3"} assert actual_result == expected_result migration = self.conn.fetch(""" SELECT * FROM public.migration; """) assert migration[0][0] == 'customer' assert migration[0][1] == '01_customer.sql' assert migration[1][0] == 'employee' assert migration[1][1] == '01_employee.sql' assert migration[2][0] == 'customer' assert migration[2][1] == '03_customer.sql' def test_script_to_dry_migrate(self): self.conn.execute('''INSERT INTO module(name) VALUES ('customer')''') self.conn.execute('''INSERT INTO module(name) VALUES ('employee')''') self.conn.execute('''INSERT INTO module(name) VALUES ('product')''') self.conn.execute('''INSERT INTO module(name) VALUES ('department')''') result = self.runner.invoke(script, ['-h', 'localhost', '-d', 'tormordb', 'migrate', 'test_module', '--dry-run']) assert result.exit_code == 0 assert self.conn.fetch('''SELECT * FROM migration''') == [] def test_script_to_include(self): self.runner.invoke(script, ['-h', 'localhost', '-d', 'tormordb', 'include', 'tormor/tests/script_file.txt']) result = self.conn.fetch("SELECT name FROM module GROUP BY name ORDER BY name") actual_result = [x['name'] for x in result] assert ["customer", "employee", "product"] == actual_result def test_script_to_include_with_dry_migrate(self): self.runner.invoke(script, ['-h', 'localhost', '-d', 'tormordb', 'include', 'tormor/tests/script_file_dry.txt']) result = self.conn.fetch("SELECT name FROM module GROUP BY name ORDER BY name") actual_result = [x['name'] for x in result] assert [] == actual_result def test_script_to_include_without_file(self): result = self.runner.invoke(script, ['-h', 'localhost', '-d', 'tormordb', 'include']) assert result.exit_code == click.UsageError.exit_code def test_script_to_execute_sql(self): self.runner.invoke(script, ['-h', 'localhost', '-d', 'tormordb', 'sql', 'tormor/tests/Schema/customer/01_customer.sql']) result = self.conn.fetch("SELECT * FROM customer") actual_result = set(record.get("name") for record in result) expected_result = {"Customer1", "Customer2", "Customer3"} assert actual_result == expected_result def test_script_to_execute_sql_no_file(self): result = self.runner.invoke(script, ['-h', 'localhost', '-d', 'tormordb', 'sql']) assert result.exit_code == click.UsageError.exit_code PKO>,tormor/tests/Schema/customer/01_customer.sqlCREATE TABLE customer( name text NOT NULL, CONSTRAINT customer_pk PRIMARY KEY(name)); INSERT INTO customer(name) VALUES('Customer1'); INSERT INTO customer(name) VALUES('Customer2'); INSERT INTO customer(name) VALUES('Customer3');PKON33,tormor/tests/Schema/customer/03_customer.sqlINSERT INTO customer(name) VALUES('Customer5');PKOLq,tormor/tests/Schema/employee/01_employee.sqlCREATE TABLE employee( name text NOT NULL, CONSTRAINT employee_pk PRIMARY KEY(name)); INSERT INTO employee(name) VALUES('Employee1'); INSERT INTO employee(name) VALUES('Employee2'); INSERT INTO employee(name) VALUES('Employee3');PKOՙr*tormor/tests/Schema/product/01_product.sqlCREATE TABLE product( name text NOT NULL, CONSTRAINT product_pk PRIMARY KEY(name)); INSERT INTO product(name) VALUES('Product1'); PKOp)X33-tormor/tests/Schema2/customer/02_customer.sqlINSERT INTO customer(name) VALUES('Customer4');PKO)e1tormor/tests/Schema2/department/01_department.sqlCREATE TABLE department( name text NOT NULL, CONSTRAINT department_pk PRIMARY KEY(name)); INSERT INTO department(name) VALUES('Department1'); PK!H}q(4%tormor-2.3.dist-info/entry_points.txtN+I/N.,()*//PzyP9+PKO 55tormor-2.3.dist-info/LICENSEMIT License Copyright (c) 2018 Proteus Technologies Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PK!HMuSatormor-2.3.dist-info/WHEEL HM K-*ϳR03rOK-J,/RH,szd&Y)r$[)T&UD"PK!H Q0 tormor-2.3.dist-info/METADATAUMs6WlIuv4k;jǂ1IK~{$(1wo>,.[$c|̮x Xksn8C+njQhn)Jjv!k/(Ci2dR[Uz2Ң3Ŭ D;u: UaEƄ, OF[$ӓgv&By|T6]w rdžM+nX $ |\T)-7U<&MdH7wV>My(&m|0!MEc_=W7d?ވϽi,cpq SIlrAɊ|w3o3s~ڊkXȉjøb)7 U8ΒN E|ߍm+L}@(,,Όt2F%MPsBcO 3|!cnaӄ{+Ϲ%aMpqpb8aVG*Wl-@sΗH&k&=1؂dPA o(x1mg<hBҔݿנ`d,痧w]1h8t./߭@Q$~ d DhăXm*a=c/917.ܻ\T'fx{OBnvK ׅeHhiuv~!~<"8pUۼа%z FuV֘Z۴Hqk% 9t*շtBhEƐjeoa{Yi[au"\ 5YP*1IhJU%G#P,IԪjA:(cDCON((Mt\CYtR4Y? Os侺J3&̘j(yV;[2[Q`[q$~RCKDי;=ѭD{xYnG[h] ˶k26i[ ?"}w֙ғRPK!HYytormor-2.3.dist-info/RECORDuI:}bo"L Id&O^v=k ?构]YwAU fprA9tyĕv4> ܵb"4S{_ a0}md;S̋BdugLfi*#&pF|)VqBM;/ fىJlt|d{ hK-Wg/qǔ&h8Uvt;>AT(Vݩ2̪.k>NL v3Y+(߉(% &ꚐAuORQWWj!&~@Dfq$P(<00Q]LyGսMņ]g^6EqOL8@]@(L8k=$wt9sX5X0Hb#XIٺ F0%p'S=pZVE~G<hov3u62 Վ$%4ڋ/7x=ͧ=YKOHR|m*mio:38RD_ 8F o^NADqxTxFz}@z,>NJ~3QHci6ia89% KX-.ƍŭ#MZIpۘJ(Q] D;nG~;@]OːX:ֲM{&}2!IpsȌK[^ _&qW*,~v6+Yu͖kΠ-Zψ_!Ri,x m6E y;(N`w6lR}yS3.9FA"!v21}QvD{)tVb`YX8w\RhI" le*c(V|<74ƿ :Ơ{_nsaQv{>1SPߍ//,w ]k/4.Q8ji T?0V0ЃX)qXі r#gh헧0?شQ 7O~0$"8^00i0ZӬjjwsQ)ǿPKѶPO55tormor/__init__.pyPK϶POcetormor/commands.pyPK2%OaBtormor/connections.pyPKO$p tormor/dsn.pyPKO(<&tormor/exceptions.pyPK϶PO8I tormor/main_script.pyPKO7P(VV tormor/path_helper.pyPKOprr큇$tormor/tests/bootstrap.shPKO9; 0%tormor/tests/cleanup.shPK϶POY0.33 &tormor/tests/script_file.txtPK϶PO\ == y&tormor/tests/script_file_dry.txtPK϶POfߵ &tormor/tests/test_connections.pyPKO22.tormor/tests/test_dsn.pyPKOOO0tormor/tests/test_path.pyPK϶POOa3tormor/tests/test_run_script.pyPKO>,Itormor/tests/Schema/customer/01_customer.sqlPKON33,KJtormor/tests/Schema/customer/03_customer.sqlPKOLq,Jtormor/tests/Schema/employee/01_employee.sqlPKOՙr* Ltormor/tests/Schema/product/01_product.sqlPKOp)X33-Ltormor/tests/Schema2/customer/02_customer.sqlPKO)e1`Mtormor/tests/Schema2/department/01_department.sqlPK!H}q(4%JNtormor-2.3.dist-info/entry_points.txtPKO 55Ntormor-2.3.dist-info/LICENSEPK!HMuSa$Stormor-2.3.dist-info/WHEELPK!H Q0 Stormor-2.3.dist-info/METADATAPK!HYyWtormor-2.3.dist-info/RECORDPK\