PK!Nfile_replicator/__init__.py__version__ = "0.1.3" PK!!$m m file_replicator/cli.pyimport os.path import click from .lib import make_file_replicator, replicate_all_files, replicate_files_on_change @click.command() @click.argument("src_dir") @click.argument("dest_parent_dir") @click.argument("connection_command", nargs=-1) @click.option( "--clean-out-first", is_flag=True, default=False, help="Optionally start by cleaning out the destination directory.", ) def main(src_dir, dest_parent_dir, connection_command, clean_out_first): """Replicate files to another computer e.g. for remote development. SRC_DIR is the source directory on this machine. DEST_PARENT_DIR is the (absolute) destination parent directory on the remote machine accessed using the CONNECTION_COMMAND. The CONNECTION_COMMAND must result in a running instance of bash ready to receive commands on stdin. Example CONNECTION_COMMANDS include: ssh some.host.com bash docker exec -i my_container bash docker-compose exec -T my_container bash So a full use of the tool might look like: file-replicator my_code_dir /home/code -- docker exec -i a_container bash (the use of "--" prevents any further processing of command line arguments by file-replicator, leaving them all for docker) Initially, all files and required directories are recursively copied. Then it waits for changes before copying each modified or new file. Note that empty directories are not replicated until they contain a file. Lastly, the only time the tool deletes files or directories is if called with the optional --clean-out-first switch. """ if not connection_command: raise click.UsageError( "Please provide a connection command to access the destination server." ) if not os.path.isabs(dest_parent_dir): raise click.UsageError( "The destination parent directory must be an absolute path." ) if not os.path.exists(src_dir) or not os.path.isdir(src_dir): raise click.UsageError("The source destination must exist and be a directory.") if clean_out_first: click.secho( "Clearing out all destination files first!", fg="green", bold="true" ) with make_file_replicator( src_dir, dest_parent_dir, connection_command, clean_out_first=clean_out_first ) as copy_file: replicate_all_files(src_dir, copy_file) while replicate_files_on_change(src_dir, copy_file): click.secho( "Restarting watchers after detecting a new directory. Consider restarting!", fg="red", bold="true", ) PK!ldfile_replicator/lib.pyimport contextlib import os.path import shutil import subprocess import time import inotify.adapters __all__ = ["make_file_replicator", "replicate_all_files", "replicate_files_on_change"] # Small receiver code (written in bash for minimum dependencies) which reads a # filename from the first line, a size from the second line, and then copies that many # bytes into the given filename, creating any parent directories if necessary. # Then repeat forever until no filename is given. RECEIVER_CODE = """ set -e while true; do read filename read size [ -z "$filename" ] && break mkdir -p "$(dirname "$filename")" echo -n "Writing file $filename ($size bytes)..." if [ $size = 0 ]; then # dd will not fetch zero bytes so handle this as a special case. touch "$filename" else # Try to fetch it all in one go... dd bs=$size count=1 of="$filename" status=none # ... but we may not have it all due to non-blocking and buffering etc. So keep trying. current_size=$(stat -c "%s" "$filename") while [ $current_size != $size ]; do dd bs=$(( $size - $current_size )) count=1 status=none >>"$filename" current_size=$(stat -c "%s" "$filename") done fi echo Done done """ @contextlib.contextmanager def make_file_replicator( src_dir, dest_parent_dir, bash_connection_command, clean_out_first=False ): """Yield a copy_file() function for replicating files over a "bash connection". The must be in the given . The final path in the becomes the destination directory in the . The must be a list. """ src_dir = os.path.abspath(src_dir) dest_parent_dir = os.path.abspath(dest_parent_dir) destination_dir = os.path.join(dest_parent_dir, os.path.basename(src_dir)) p = subprocess.Popen(bash_connection_command, stdin=subprocess.PIPE) # Get the remote end up and running, waiting for commands. if clean_out_first: # Only delete the *contents* of the destination directory so that the # inode does not change (because that is irritating!). p.stdin.write(f"rm -rf {destination_dir}/*\n".encode()) p.stdin.write(RECEIVER_CODE.encode()) p.stdin.flush() def copy_file(src_filename): src_filename = os.path.abspath(src_filename) assert src_filename.startswith(src_dir), src_filename assert os.path.isfile(src_filename) size = os.path.getsize(src_filename) dest_filename = os.path.join( destination_dir, src_filename[(1 + len(src_dir)) :] ) p.stdin.write(f"{dest_filename}\n".encode()) p.stdin.write(f"{size}\n".encode()) with open(src_filename, "rb") as f: shutil.copyfileobj(f, p.stdin) p.stdin.flush() try: yield copy_file finally: p.stdin.close() p.wait() def replicate_all_files(src_dir, copy_file): """Walk src_dir to copy all files using copy_file().""" for root, dirnames, filenames in os.walk(src_dir): for filename in filenames: copy_file(os.path.join(root, filename)) def replicate_files_on_change(src_dir, copy_file, timeout=None): """Wait for changes to files in src_dir and copy with copy_file(). If provided, the timeout indicates when to return after that many seconds of no change. This is an imperfect solution because there are seemingly unavoidable race conditions when watching for file changes or additions and new directories are involved. Returns True to indicate that new directories have been added and the function should be called again. Otherwise returns None. """ please_call_me_again = False i = inotify.adapters.InotifyTree(src_dir) for event in i.event_gen(yield_nones=False, timeout_s=timeout): (_, type_names, path, filename) = event # For debugging... # print( # "PATH=[{}] FILENAME=[{}] EVENT_TYPES={}".format(path, filename, type_names) # ) if "IN_CLOSE_WRITE" in type_names: copy_file(os.path.join(path, filename)) if "IN_CREATE" in type_names and "IN_ISDIR" in type_names: # Race condition danger because a new directory was created (see warning on https://pypi.org/project/inotify). # Wait a short while for things to settle, then replicate the new directory, then start the watchers again. time.sleep(0.5) new_directory = os.path.join(path, filename) replicate_all_files(new_directory, copy_file) please_call_me_again = True break return please_call_me_again PK!HmI3<0file_replicator-0.1.3.dist-info/entry_points.txtN+I/N.,()JI-J-LN,/|L<..PK!kBړ66'file_replicator-0.1.3.dist-info/LICENSEMIT License Copyright (c) 2019 Timothy Corbett-Clark Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. PK!HnHTU%file_replicator-0.1.3.dist-info/WHEEL A н#Z;/"d&F[xzw@Zpy3Fv]\fi4WZ^EgM_-]#0(q7PK!Hz (file_replicator-0.1.3.dist-info/METADATAYSH~_1-Kp\6 ػ*e5͌ 랑l}8?Pӿ^ViL0TjU[\zc ;̎]Mu$Q| ol/XaUm@TUVO~5J[/~EAqmr؉8Zĉs}zRI{ߦh$ӨQvʖ#KcqVs*%B9jp&R'ܕ0QšT@xy@H"A~ԾdD3=x|]PP mD8K6V5jrDMɦ?2G`c۴y 8|<וNHs@T kknfIr Қr^Vކ *ߖxSJңZR@a=A%4^(~5B{h#6JH*ؑR"mH-%:&/ʝwmAyը!*^k(=5xUx&=J?`rj EH$9(' +' -ܙ  oD@BkTS-|mRRPUx ƛ,Jz| NtH+r ES@vL y տAYT#1８muYb#$.; }2ǽC dFAg bxg.\=leFUVxyG=z]SN΂"2Pprlpf]RY@膅%3yH%@"d7ro*T~ XS-ٗ+QixdJzCBFDƍ~DŽj˒:Gj!aBBJ«!F>fof7S0y^(9vCͷ  ĥ6ӱ s+jH=ySé9賗#aF nFuf6+"*U<~ZJ)!1>|%.n//n>8=^_]3LO.?~<8^6/c8eI~|[[ym5@lHH]b";_sB\W0=3 R*YP_uFAYȑRo=BMRl,w"&rjQYe7mJA1'{pYXEGM{ƃ':o2/ƌ*t[w5HlUx=fTpVr$*%&_ +r0u=Ew27fL ipOEl 4rI6mf32ҟg 3 PQ]+2rXmd!9A!/qOҽ̋w `ppvbHڢ2jptn? ڨG)LıE@%1no~nJ̸ǐ^k/lBQxVFBY# %tT3:8Mf"oP/8XfSJeG)n۬F',b7|Vk.$Qbौum@f']0 ix@}VB 4p"zW䩾ßSvi(>㮄>5У󨭥jhFmw\q|2P &dِN4ř7dlv;?TS޼:%\ {+Հx<}X|ZZPn\2Pz+hݵ iG0L-d:Ӽvyڒ.M TQ'DrX-{#8 ̯4`?l{ǩą{ EnJM ~aĤ1QSC/+@̈́V*6qسLj,bȌRŁDTn n)!hBLҶi[IO$}w7GٛH(;<1= rd:*o^V50D;< Vy *Pх@ |)mh T_T́.lW^<1ylj<z+xu.mthD޷n2Lљ2mYNЬnNߘ߄ÕdVAּ? ?fCiƪ r3ָJL-%A62mϙ,Ja~_iCnb sCM]XIqcVv}v|,TI&Fkxn̍OM’PK!H\p&file_replicator-0.1.3.dist-info/RECORDr@@} " [C@DȆБ6V?pn-q]$kSG^xJMOJ(Eh@WB~7~6?/$(K%5YLhDQliZHZT)P2 InhbHN +UOPiO\ #6fqbq[vE;nq3#CXCλ).Jfjy*i124cz?Ny0UA~E1[W W1<'çYG~.PS2Vv[RB$?VЌ.Pk3P|j;(Pwj>xEeֈV#nq yyGPK!Nfile_replicator/__init__.pyPK!!$m m Ofile_replicator/cli.pyPK!ld file_replicator/lib.pyPK!HmI3<0file_replicator-0.1.3.dist-info/entry_points.txtPK!kBړ66'2file_replicator-0.1.3.dist-info/LICENSEPK!HnHTU%"file_replicator-0.1.3.dist-info/WHEELPK!Hz (D#file_replicator-0.1.3.dist-info/METADATAPK!H\p&{.file_replicator-0.1.3.dist-info/RECORDPKJ0