PKVNSƝcloudsync/__init__.py""" cloudsync enables simple cloud file-level sync with a variety of cloud providers External modules: cloudsync.Event cloudsync.Provider cloudsync.Sync Example: import cloudsync prov = cloudsync.Provider('GDrive', token="237846234236784283") info = prov.upload(file, "/dest") print ("id of /dest is %s, hash of /dest is %s" % (info.id, info.hash)) Command-line example: cloudsync -p gdrive --token "236723782347823642786" -f ~/gdrive-folder --daemon """ __version__ = "0.1.1" # import modules into top level for convenience from .provider import * from .event import * from .sync import * from .exceptions import * from .eventmanager import * from .command import main if __name__ == "__main__": main() PKjlNYcloudsync/command.pydef main(): raise PKWN*00cloudsync/event.pyfrom collections import namedtuple # def update(self, side, otype, oid, path=None, hash=None, exists=True): EventBase = namedtuple('EventBase', 'side otype oid path hash exists mtime') class Event(EventBase): # TODO: test provider with all of these changed to random strings to confirm the string isn't used explicitly SOURCE_REMOTE = "remote" SOURCE_LOCAL = "local" ACTION_CREATE = "create" ACTION_RENAME = "rename" ACTION_UPDATE = "modify" ACTION_DELETE = "delete" TYPE_FILE = "file" TYPE_DIRECTORY = "directory" PKJNfOcloudsync/eventmanager.pyfrom .runnable import Runnable class EventManager(Runnable): def __init__(self, provider): super().__init__() self.provider = provider self.timeout = 1 def do(self): # One iteration of the loop for e in self.provider.events(timeout=self.timeout): if e is None: continue # update the state by calling update with the info from the event PKNϕcloudsync/exceptions.pyclass CloudFileNotFoundError(Exception): pass class CloudTemporaryError(Exception): pass class CloudFileExistsError(Exception): pass PKWNxO O cloudsync/provider.pyfrom abc import ABC, abstractmethod from collections import namedtuple from cloudsync.event import Event ProviderInfo = namedtuple('ProviderInfo', 'oid hash path') class Provider(ABC): def __init__(self, case_sensitive=True, allow_renames_over_existing=True, sep="/"): self._sep = sep # path delimiter self._case_sensitive = case_sensitive # TODO: implement support for this self._allow_renames_over_existing = allow_renames_over_existing self._events = [] self._event_cursor = 0 self.walked = False @abstractmethod def _api(self, *args, **kwargs): ... @abstractmethod def events(self, timeout): ... @abstractmethod def walk(self): ... @abstractmethod def upload(self, oid, file_like): ... @abstractmethod def create(self, path, file_like) -> 'ProviderInfo': ... @abstractmethod def download(self, oid, file_like): ... @abstractmethod def rename(self, oid, path): ... @abstractmethod def mkdir(self, path) -> str: ... @abstractmethod def delete(self, oid): ... @abstractmethod def exists_oid(self, oid): ... @abstractmethod def exists_path(self, path) -> bool: ... @staticmethod @abstractmethod def hash_data(file_like): ... @abstractmethod def remote_hash(self, oid): ... @abstractmethod def info_path(self, path) -> ProviderInfo: ... @abstractmethod def info_oid(self, oid) -> ProviderInfo: ... @abstractmethod def translate_event(self, provider_event) -> Event: ... def is_sub_path(self, folder, target, sep=None, anysep=False, strict=False): if sep is None: if anysep: sep = "/" folder = folder.replace("\\", "/") target = target.replace("\\", "/") else: sep = self._sep # Will return True for is-same-path in addition to target folder_full = str(folder) folder_full = folder_full.rstrip(sep) target_full = str(target) target_full = target_full.rstrip(sep) # .lower() instead of normcase because normcase will also mess with separators if not self._case_sensitive: folder_full = folder_full.lower() target_full = target_full.lower() # target is same as folder, or target is a subpath (ensuring separator is there for base) if folder_full == target_full: return False if strict else sep elif len(target_full) > len(folder_full) and \ target_full[len(folder_full)] == sep: if target_full.startswith(folder_full): return target_full.replace(folder_full, "", 1) else: return False return False def replace_path(self, path, from_dir, to_dir): relative = self.is_sub_path(path, from_dir) if relative: return to_dir + relative raise ValueError("replace_path used without subpath") PKPNccloudsync/runnable.pyimport time from abc import ABC, abstractmethod import threading import logging log = logging.getLogger(__name__) def time_helper(timeout, sleep=None): forever = not timeout end = forever or time.monotonic() + timeout while forever or end >= time.monotonic(): yield True if sleep is not None: time.sleep(sleep) class Runnable(ABC): def run(self, *, timeout=None, until=None, sleep=0.1): self.stopped = False for _ in time_helper(timeout, sleep=sleep): if self.stopped or (until is not None and until()): break try: self.do() except Exception: log.exception("unhandled exception in %s", self.__class__) if self.stopped: self.done() @abstractmethod def do(self): ... def stop(self): self.stopped = True def done(self): pass def test_runnable(): class Foo(Runnable): def __init__(self): self.cleaned=False self.done = 0 def do(self): self.done += 1 def done(self): self.cleaned = True foo = Foo() foo.run(timeout=0.02, sleep=0.001) assert foo.done foo.done = 0 foo.run(until=lambda: foo.done == 1) assert foo.done == 1 thread=threading.Thread(target=foo.run) thread.start() foo.stop() thread.join(timeout=1) assert foo.stopped == 1 PKVNr,,cloudsync/sync.pyimport os import time import logging import tempfile import shutil from typing import Optional __all__ = ['SyncManager', 'SyncState', 'LOCAL', 'REMOTE', 'FILE', 'DIRECTORY'] from cloudsync.exceptions import CloudFileNotFoundError from .runnable import Runnable log = logging.getLogger(__name__) # state of a single object class Reprable: def __repr__(self): return self.__class__.__name__ + str(self.__dict__) class SideState(Reprable): # pylint: disable=too-few-public-methods def __init__(self): self.exists: bool = True # exists at provider self.hash: Optional[bytes] = None # hash at provider self.path: Optional[str] = None # path at provider self.oid: Optional[str] = None # oid at provider # time of last change (we maintain this) self.changed: Optional[float] = None self.sync_hash: Optional[bytes] = None # hash at last sync self.sync_path: Optional[str] = None # path at last sync # these are not really local or remote # but it's easier to reason about using these labels LOCAL = 0 REMOTE = 1 def other_side(index): return 1-index FILE = "file" DIRECTORY = "dir" # single entry in the syncs state collection class SyncEntry(Reprable): def __init__(self, otype): self.__states = (SideState(), SideState()) self.sync_exists = None self.otype = otype self.temp_file = None def __getitem__(self, i): return self.__states[i] def update(self, providers): for i in (LOCAL, REMOTE): if self[i].changed: # get latest info from provider if self.otype == FILE: self[i].hash = providers[i].hash_oid(self[i].oid) self[i].exists = bool(self[i].hash) else: self[i].exists = providers[i].exists(self[i].oid) else: # trust local sync state self[i].exists = self.sync_exists if self[i].sync_hash: self[i].hash = self[i].sync_hash self[i].path = self[i].sync_path else: self[i].hash = None log.debug("updated state %s %s", self[LOCAL], self[REMOTE]) def hash_conflict(self): if self[0].sync_hash and self[1].sync_hash: return self[0].hash != self[0].sync_hash and self[1].hash != self[1].sync_hash return False def path_conflict(self): if self[0].sync_path and self[1].sync_path: return self[0].path != self[0].sync_path and self[1].path != self[1].sync_path return False class SyncState: def __init__(self): self._oids = ({}, {}) self._paths = ({}, {}) self._changeset = set() def _change_path(self, side, ent, path): assert type(ent) is SyncEntry assert ent[side].oid if ent[side].path: if ent[side].path in self._paths[side]: self._paths[side][ent[side].path].pop(ent[side].oid, None) if not self._paths[side][ent[side].path]: del self._paths[side][ent[side].path] if path not in self._paths[side]: self._paths[side][path] = {} self._paths[side][path][ent[side].oid] = ent ent[side].path = path def _change_oid(self, side, ent, oid): assert type(ent) is SyncEntry if ent[side].oid: self._oids[side].pop(ent[side].oid, None) self._oids[side][oid] = ent ent[side].oid = oid def lookup_oid(self, side, oid): try: return self._oids[side][oid] except KeyError: return [] def lookup_path(self, side, path): try: return self._paths[side][path].values() except KeyError: return [] def rename_dir(self, side, from_dir, to_dir, is_subpath, replace_path): """ when a directory changes, utility to rename all kids """ remove = [] for path, sub in self._paths[side].items(): if is_subpath(from_dir, sub.path): sub.path = replace_path(sub.path, from_dir, to_dir) remove.append(path) self._paths[side][sub.path] = sub for path in remove: self._paths[side].pop(path) def update(self, side, otype, oid, path=None, hash=None, exists=True): ent = self.lookup_oid(side, oid) if not ent: ent = SyncEntry(otype) if oid is not None: self._change_oid(side, ent, oid) if path is not None: self._change_path(side, ent, path) if hash is not None: ent[side].hash = hash ent[side].exists = exists ent[side].changed = time.time() self._changeset.add(ent) def changes(self): return list(self._changeset) def get_all(self): ents = set() for ent in self._oids[LOCAL].values(): ents.add(ent) for ent in self._oids[REMOTE].values(): ents.add(ent) return ents def synced(self, ent): self._changeset.remove(ent) def entry_count(self): return len(self.get_all()) class SyncManager(Runnable): def __init__(self, syncs, providers, translate): self.syncs = syncs self.providers = providers self.providers[LOCAL]._sname = "local" self.providers[REMOTE]._sname = "remote" self.translate = translate self.tempdir = tempfile.mkdtemp(suffix=".cloudsync") assert len(self.providers) == 2 def do(self): for sync in self.syncs.changes(): self.sync(sync) def done(self): log.info("cleanup %s", self.tempdir) shutil.rmtree(self.tempdir) def sync(self, sync): sync.update(self.providers) if sync.hash_conflict(): self.handle_hash_conflict(sync) if sync.path_conflict(): self.handle_path_conflict(sync) for i in (LOCAL, REMOTE): if sync[i].changed: self.embrace_change(sync, i, other_side(i)) def temp_file(self, ohash): # prefer big random name over NamedTemp which can infinite loop in odd situations! return os.path.join(self.tempdir, ohash) def finished(self, side, sync): sync[side].changed = None self.syncs.synced(sync) if sync.temp_file: try: os.unlink(sync.temp_file) except: pass sync.temp_file = None def download_changed(self, changed, sync): sync.temp_file = sync.temp_file or self.temp_file(sync[changed].hash) assert sync[changed].oid if os.path.exists(sync.temp_file): return True try: self.providers[changed].download(sync[changed].oid, open(sync.temp_file + ".tmp", "wb")) os.rename(sync.temp_file + ".tmp", sync.temp_file) return True except CloudFileNotFoundError: log.debug("download from %s failed fnf, switch to not exists", self.providers[changed]._sname) sync[changed].exists = False return False def upload_synced(self, changed, sync): synced = other_side(changed) try: info = self.providers[synced].upload(sync[synced].oid, open(sync.temp_file, "rb")) log.debug("upload to %s as path %s", self.providers[synced]._sname, sync[synced].sync_path) sync[synced].sync_hash = info.hash if info.path: sync[synced].sync_path = info.path else: sync[synced].sync_path = sync[synced].path sync[synced].oid = info.oid sync[changed].sync_hash = sync[changed].hash sync[changed].sync_path = sync[changed].path self.finished(changed, sync) except CloudFileNotFoundError: log.debug("upload to %s failed fnf, TODO fix mkdir code and stuff", self.providers[synced]._sname) raise NotImplementedError("TODO mkdir, and make syncs etc") def create_synced(self, changed, sync): synced = other_side(changed) try: translated_path = self.translate(synced, sync[changed].path) info = self.providers[synced].create(translated_path, open(sync.temp_file, "rb")) log.debug("create on %s as path %s", self.providers[synced]._sname, translated_path) sync[synced].oid = info.oid sync[synced].sync_hash = info.hash if info.path: sync[synced].sync_path = info.path else: sync[synced].sync_path = sync[synced].path sync[changed].sync_hash = sync[changed].hash sync[changed].sync_path = sync[changed].path self.finished(changed, sync) except CloudFileNotFoundError: log.debug("create on %s failed fnf, mkdir needed", self.providers[synced]._sname) raise NotImplementedError("TODO mkdir, and make syncs etc") def embrace_change(self, sync, changed, synced): log.debug("changed %s", sync) if not sync[changed].exists: # see if there are other entries for the same path, but other ids ents = list(self.syncs.lookup_path(changed, sync[changed].path)) if len(ents) == 1: assert ents[0] == sync self.providers[synced].delete(sync[synced].oid) sync[synced].exists = False self.finished(changed, sync) return if sync[changed].path != sync[changed].sync_path: if not sync[changed].sync_path: assert not sync[changed].sync_hash # looks like a new file if not self.download_changed(changed, sync): return if sync[synced].oid: upload_synced(changed, sync) return self.create_synced(changed, sync) return else: assert sync[synced].oid translated_path = self.translate(synced, sync[changed].path) log.debug("rename %s %s", sync[synced].sync_path, translated_path) self.providers[synced].rename(sync[synced].oid, translated_path) sync[synced].path = translated_path sync[synced].sync_path = translated_path sync[changed].sync_path = sync[changed].path self.finished(changed, sync) return if sync[changed].hash != sync[changed].sync_hash: # not a new file, which means we must have last sync info assert sync[synced].oid self.download_changed(changed, sync) self.upload_synced(changed, sync) return log.info("nothing changed %s, but changed is true", sync) def handle_hash_conflict(self, sync): raise NotImplementedError() def handle_path_conflict(self, sync): raise NotImplementedError() PK!H7&,*cloudsync-0.1.1.dist-info/entry_points.txtN+I/N.,()J/M)Kr3PKUNH6G((!cloudsync-0.1.1.dist-info/LICENSEMIT License Copyright (c) 2019 Atakama Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PK!HMuSacloudsync-0.1.1.dist-info/WHEEL HM K-*ϳR03rOK-J,/RH,szd&Y)r$[)T&UD"PK!H <S<"cloudsync-0.1.1.dist-info/METADATAUQR0 +4Õ4cNti`j5ҮrRhNQvY,ގ`.@"Deozt" _J@8aP&p*<UY,Ι}Ѡ]QȂ}zkjL*X}iYA+t* n{]P>q֢v6Mn<<{*9QU׬3M(َ>v>G^3iYfI adā;9m]džߨ}ȷ`ȏ3V}g˝CpV}`>4SK!$Nm.N|Cu&NX<3U5Ґ h}D_Xzmm~OgG5,3Ny+Z,ijβ8qݬbzh) 5#jI84|Ƭ -AMs J˓m;rqz3@;a/W2@eYcNtעPNQG<1dOe^[JUwVX=9υW,yG"Fex]^}5V5O`Mֺ\Y|UB?PKVNSƝcloudsync/__init__.pyPKjlNYcloudsync/command.pyPKWN*00Ncloudsync/event.pyPKJNfOcloudsync/eventmanager.pyPKNϕcloudsync/exceptions.pyPKWNxO O Scloudsync/provider.pyPKPNccloudsync/runnable.pyPKVNr,,cloudsync/sync.pyPK!H7&,* Gcloudsync-0.1.1.dist-info/entry_points.txtPKUNH6G((!wGcloudsync-0.1.1.dist-info/LICENSEPK!HMuSaKcloudsync-0.1.1.dist-info/WHEELPK!H <S<"nLcloudsync-0.1.1.dist-info/METADATAPK!Hj Ncloudsync-0.1.1.dist-info/RECORDPK P