{ "info": { "author": "J C Lawrence", "author_email": "claw@kanga.nu", "bugtrack_url": null, "classifiers": [], "description": "s3lncoll\n========\n\nRead files from S3 as defined by a key prefix and map them by lines to\na set of optionally gzip compressed output files in S3, with the\noutput files limited by (pre-compressed) file size. The string \"{}\"\nin the output key will be substituted with the (zero-based) index of\nthe output files.\n\n::\n\n s3lncoll: Line stream s3 files into ~uniform lumps in S3\n \n Usage: s3lncoll {{arguments}} {{options}}\n \n Arguments:\n from [text] S3 URL prefix to clump\n to [text] S3 URL for target clump ('{}' will be the count)\n \n Options:\n -h, --help Show this help message and exit\n -H, --HELP Help for all sub-commands\n -D, --debug Enable debug logging\n -d, --delete Delete source files/keys\n -j, --json Validate each line as JSONM\n -q, --quiet Be quiet, be vewy vewy quiet\n -V, --version Report installed version\n -z, --compress Ccompress (gzip) the target(s)\n -b, --blocksize [int] Maximum size of pre-compressed output files in bytes. (default: 1048576)\n\n\nArchitecture\n============\n\ns3lncoll has a pipe and filter architecture which streams a set of keys as defined by a prefix \nthrough a `LineStream`. `LineStream` reads the files under the keys and spits out a single line \nvia an iterator. `RotatingFileCtx` receives that stream of lines and aggregates them into chunked \nfiles (of a maximum size or a single line, whichever is the larger), followed by flushing the \nlines out to a provided S3 path.\n\n+----------------------------------------------------------------------------------------------------+\n| |\n| +------------+ +-------------------------+ |\n| | | | | |\n| bucket Keys | LineStream | Lines | RotatingFileCtx | S3 Files |\n| ------------> | | ------------> | | ------------> |\n| | | | | |\n| +------------+ +-------------------------+ |\n| |\n| +---------------------------+ |\n| | | |\n| | cmd.py: Scheduler | |\n| | | |\n| +---------------------------+ |\n| |\n| s3lncoll |\n+----------------------------------------------------------------------------------------------------+\n", "description_content_type": null, "docs_url": null, "download_url": "", "downloads": { "last_day": -1, "last_month": -1, "last_week": -1 }, "home_page": "https://pypi.python.org/pypi/s3lncoll", "keywords": "S3 aggregate tarball compress", "license": "LGPL v3.0", "maintainer": "", "maintainer_email": "", "name": "s3lncoll", "package_url": "https://pypi.org/project/s3lncoll/", "platform": "", "project_url": "https://pypi.org/project/s3lncoll/", "project_urls": { "Homepage": "https://pypi.python.org/pypi/s3lncoll" }, "release_url": "https://pypi.org/project/s3lncoll/0.1.post16/", "requires_dist": null, "requires_python": "", "summary": "Line stream s3 files into ~uniform lumps in S3", "version": "0.1.post16" }, "last_serial": 3264426, "releases": { "0.1.post10": [ { "comment_text": "", "digests": { "md5": "70bb49fa5c9a6c5c930ead7d4c38be7c", "sha256": "d0a1f0493fcab69149c584b0babecfe2886cd9e8809527ec289c226342b07701" }, "downloads": -1, "filename": "s3lncoll-0.1.post10.tar.gz", "has_sig": false, "md5_digest": "70bb49fa5c9a6c5c930ead7d4c38be7c", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 23786, "upload_time": "2017-09-26T01:18:37", "url": "https://files.pythonhosted.org/packages/9b/0e/5e2345754152f20e5583a6554273f0e7b50ee22d2e35a2ceb4a155b0a04f/s3lncoll-0.1.post10.tar.gz" } ], "0.1.post11": [ { "comment_text": "", "digests": { "md5": "7371359b269602969ad49350fd09281c", "sha256": "89593d71669d28e4284bacb1b06359bb7b7b2561f86a97bc5e2896aa7227c4c5" }, "downloads": -1, "filename": "s3lncoll-0.1.post11.tar.gz", "has_sig": false, "md5_digest": "7371359b269602969ad49350fd09281c", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 23935, "upload_time": "2017-09-26T21:55:41", "url": "https://files.pythonhosted.org/packages/74/22/dfdeeac756fedb125256024800440678c1e925d3c6c044628c1aeb49ee46/s3lncoll-0.1.post11.tar.gz" } ], "0.1.post13": [ { "comment_text": "", "digests": { "md5": "84f6d432cc0b24bc735c64aedef47987", "sha256": "c977a84a65c853914b5c01e6a9b34d0a87f97ea8436f8c94b0b8ef469692213e" }, "downloads": -1, "filename": "s3lncoll-0.1.post13.tar.gz", "has_sig": false, "md5_digest": "84f6d432cc0b24bc735c64aedef47987", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 23943, "upload_time": "2017-09-27T18:17:13", "url": "https://files.pythonhosted.org/packages/3a/32/1c281e8f5695cd375de355ee39f10355b2a16bc208134ef23abffae79598/s3lncoll-0.1.post13.tar.gz" } ], "0.1.post14": [ { "comment_text": "", "digests": { "md5": "77b9d32b0ee927920cedcf902367402d", "sha256": "137456625671a320bdd30ec2f7d796de231e6f1bfc7bf1e19dc2f00044ce8dcd" }, "downloads": -1, "filename": "s3lncoll-0.1.post14.tar.gz", "has_sig": false, "md5_digest": "77b9d32b0ee927920cedcf902367402d", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 23954, "upload_time": "2017-09-27T18:30:33", "url": "https://files.pythonhosted.org/packages/e0/04/93130b76b956f3cb538870f983a228cd32a6a45d05489bfad9661ec3113a/s3lncoll-0.1.post14.tar.gz" } ], "0.1.post16": [ { "comment_text": "", "digests": { "md5": "a0dbfc1d10eefc716ee0c22ffe9145ff", "sha256": "78e56edecdf62d85e8f7e84ba9d3b0b7eb71ea6067421d3b2e20b4be07e604ff" }, "downloads": -1, "filename": "s3lncoll-0.1.post16.tar.gz", "has_sig": false, "md5_digest": "a0dbfc1d10eefc716ee0c22ffe9145ff", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 24610, "upload_time": "2017-10-20T01:43:38", "url": "https://files.pythonhosted.org/packages/cc/49/1a4541f72f3f07a5e8d7808a5b8e0f6a9d9bb9728d98fded2fb6306adcba/s3lncoll-0.1.post16.tar.gz" } ], "0.1.post7": [ { "comment_text": "", "digests": { "md5": "8b2cdd2f7043fbb2759d30b907a52da1", "sha256": "8e010f55853b1f39c9ad0b121eaaafff878d3e41963dabb4f65c7cfdfe787574" }, "downloads": -1, "filename": "s3lncoll-0.1.post7.tar.gz", "has_sig": false, "md5_digest": "8b2cdd2f7043fbb2759d30b907a52da1", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 23792, "upload_time": "2017-09-25T18:43:30", "url": "https://files.pythonhosted.org/packages/2e/f3/ea1c59c3237930feb9af013479fc3db36e77dbd0b9a61cd8db5c577c21aa/s3lncoll-0.1.post7.tar.gz" } ], "0.1.post8": [ { "comment_text": "", "digests": { "md5": "f62b77faba58c499995175dba62552d0", "sha256": "d6975c12c1a28a5d5050820fe5418340df49ee795099c1882d0ef6061073fee7" }, "downloads": -1, "filename": "s3lncoll-0.1.post8.tar.gz", "has_sig": false, "md5_digest": "f62b77faba58c499995175dba62552d0", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 23784, "upload_time": "2017-09-25T19:01:06", "url": "https://files.pythonhosted.org/packages/9b/65/5700cc3d49a9242de0236fac93509de8c1bff2cbd79dd253fce5aec714ce/s3lncoll-0.1.post8.tar.gz" } ], "0.1.post9": [ { "comment_text": "", "digests": { "md5": "d611c1ac350b67f6d8bf600e069b3e1e", "sha256": "259e128e8605173d3e554697555de793da649062d32b301b0a5680bbfa12a3b6" }, "downloads": -1, "filename": "s3lncoll-0.1.post9.tar.gz", "has_sig": false, "md5_digest": "d611c1ac350b67f6d8bf600e069b3e1e", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 23923, "upload_time": "2017-09-26T01:00:06", "url": "https://files.pythonhosted.org/packages/a9/f5/2a7e1788a503428b271804b0d72d4d63f3108dade974c8436d213929f2cc/s3lncoll-0.1.post9.tar.gz" } ] }, "urls": [ { "comment_text": "", "digests": { "md5": "a0dbfc1d10eefc716ee0c22ffe9145ff", "sha256": "78e56edecdf62d85e8f7e84ba9d3b0b7eb71ea6067421d3b2e20b4be07e604ff" }, "downloads": -1, "filename": "s3lncoll-0.1.post16.tar.gz", "has_sig": false, "md5_digest": "a0dbfc1d10eefc716ee0c22ffe9145ff", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 24610, "upload_time": "2017-10-20T01:43:38", "url": "https://files.pythonhosted.org/packages/cc/49/1a4541f72f3f07a5e8d7808a5b8e0f6a9d9bb9728d98fded2fb6306adcba/s3lncoll-0.1.post16.tar.gz" } ] }