{
    "info": {
        "author": "Dmitriy Pomazunovskiy",
        "author_email": "forestwheel@gmail.com",
        "bugtrack_url": null,
        "classifiers": [
            "Development Status :: 5 - Production/Stable",
            "Intended Audience :: Developers",
            "License :: OSI Approved :: BSD License",
            "Programming Language :: Python",
            "Programming Language :: Python :: 3.7",
            "Topic :: Software Development"
        ],
        "description": "======================================\nSimple multiprocessing management tool\n======================================\n\nHandles big amounts of data using python multiprocessing\n\n=========\nConcept\n=========\n\nThe main idea is to utilize multiprocessing, by building source queue and handle items to process in workers.\n\nMultiprocess managing is implemented using `mp_manager.manager.py` with help of `mp_manager.process_helpers.py` that provides events, locks, shared values and stats, exception handler helper and `ExitStatus`\n\nWorkers for multiprocessing are built based on handlers classes.\n\n========\nHandlers\n========\n\nHandlers must inherit `mp_manager.base_handler.BaseHandler` and at least implement:\n\n1) classmethod `build_items()` that must return iterable of items to handle by workers\n\n2) `_handle_item()` which is used to handle current item\n\nThey also could implement:\n\n1) `log_item()` that is used by manager as a source for logging\n\n2) `_reset_stats()` is used internally to reset worker dict stats related to current item\n\n============\nSample usage\n============\n\n::\n\n  #!/usr/bin/env python3.7\n  # -*- coding: utf-8 -*-\n\n  import math\n  import random\n  import time\n\n  from mp_manager.base_handler import BaseHandler\n  from mp_manager.process_helpers import Stats, ns_container\n  from mp_manager.utils import build_batches\n\n\n  class SampleHandler(BaseHandler):\n\n      batch_size = 2\n\n      @classmethod\n      def build_items(cls, *args, **kwargs):\n          total = 100\n          batches_total = int(math.ceil(total / cls.batch_size))\n\n          Stats.set_stats_val('items_total', total)\n          Stats.set_stats_val('batches_total', batches_total)\n          ns_container.set_shared_value('shared_data', cls._data_to_share(),\n                                        True)\n          for i, val in enumerate(build_batches(range(total), cls.batch_size)):\n              yield i, val\n\n      @property\n      def log_item(self):\n          return f'{self.current_item[0]} / {Stats.stats.batches_total}'\n\n      @classmethod\n      def _data_to_share(cls):\n          return dict(somekey=\"some value\")\n\n      def _reset_stats(self):\n          self.stats = {\n              'items_processed': 0,\n              'items_positive': 0,\n              'items_negative': 0,\n          }\n\n      def _handle_item(self):\n          num, items = self.current_item\n          for item in items:\n              self._process_number(item)\n          Stats.inc_stats_vals(self.stats)\n          self.logger.info(Stats.get_stats())\n\n      def _process_number(self, item):\n          time.sleep(random.random())\n          value = math.factorial(item) / math.cos(item)\n          self.stats['items_processed'] += 1\n          if value > 0:\n              self.stats['items_positive'] += 1\n          else:\n              self.stats['items_negative'] += 1\n\n\n  if __name__ == '__main__':\n      import logging\n      from mp_manager.manager import ImportManager\n\n      logging.basicConfig(level=logging.DEBUG)\n      status = ImportManager(SampleHandler, num_workers=8).run()\n\n      print(status)\n      exit(status.code)\n\n\n",
        "description_content_type": "",
        "docs_url": null,
        "download_url": "",
        "downloads": {
            "last_day": -1,
            "last_month": -1,
            "last_week": -1
        },
        "home_page": "https://github.com/dmiwell/mp_manager",
        "keywords": "",
        "license": "BSD",
        "maintainer": "",
        "maintainer_email": "",
        "name": "mp-manager",
        "package_url": "https://pypi.org/project/mp-manager/",
        "platform": "",
        "project_url": "https://pypi.org/project/mp-manager/",
        "project_urls": {
            "Homepage": "https://github.com/dmiwell/mp_manager"
        },
        "release_url": "https://pypi.org/project/mp-manager/0.2/",
        "requires_dist": null,
        "requires_python": "",
        "summary": "Simple python multiprocessing manager",
        "version": "0.2"
    },
    "last_serial": 5297588,
    "releases": {
        "0.1": [
            {
                "comment_text": "",
                "digests": {
                    "md5": "0d07dce5ba54944ff695c580a16ee210",
                    "sha256": "ad0aec5b21e6071bb9f694b6144608fa32e18d4460766ad5462a524a6c47f2a3"
                },
                "downloads": -1,
                "filename": "mp_manager-0.1-py3-none-any.whl",
                "has_sig": false,
                "md5_digest": "0d07dce5ba54944ff695c580a16ee210",
                "packagetype": "bdist_wheel",
                "python_version": "py3",
                "requires_python": null,
                "size": 7218,
                "upload_time": "2019-05-21T12:04:24",
                "url": "https://files.pythonhosted.org/packages/b6/5f/1c3ab2db5c87620d3e029adeefd457d95a6a0062bf99e3e2ad4a045431ff/mp_manager-0.1-py3-none-any.whl"
            }
        ],
        "0.2": [
            {
                "comment_text": "",
                "digests": {
                    "md5": "20a0e5f50763b482e79a2f58034e32e9",
                    "sha256": "fef727e05e05541632fb029e8004a39d732c59714ecb2559a6597a1fcbf02254"
                },
                "downloads": -1,
                "filename": "mp_manager-0.2-py3-none-any.whl",
                "has_sig": false,
                "md5_digest": "20a0e5f50763b482e79a2f58034e32e9",
                "packagetype": "bdist_wheel",
                "python_version": "py3",
                "requires_python": null,
                "size": 7278,
                "upload_time": "2019-05-21T12:09:53",
                "url": "https://files.pythonhosted.org/packages/c7/b2/5668a7753640a15bcc7c2c998fa679842459bc349b0a4eff1f4b713e42c0/mp_manager-0.2-py3-none-any.whl"
            }
        ]
    },
    "urls": [
        {
            "comment_text": "",
            "digests": {
                "md5": "20a0e5f50763b482e79a2f58034e32e9",
                "sha256": "fef727e05e05541632fb029e8004a39d732c59714ecb2559a6597a1fcbf02254"
            },
            "downloads": -1,
            "filename": "mp_manager-0.2-py3-none-any.whl",
            "has_sig": false,
            "md5_digest": "20a0e5f50763b482e79a2f58034e32e9",
            "packagetype": "bdist_wheel",
            "python_version": "py3",
            "requires_python": null,
            "size": 7278,
            "upload_time": "2019-05-21T12:09:53",
            "url": "https://files.pythonhosted.org/packages/c7/b2/5668a7753640a15bcc7c2c998fa679842459bc349b0a4eff1f4b713e42c0/mp_manager-0.2-py3-none-any.whl"
        }
    ]
}