{ "info": { "author": "Rafael Herrero Solis", "author_email": "rafahsolis@hotmail.com", "bugtrack_url": null, "classifiers": [ "Development Status :: 3 - Alpha", "License :: OSI Approved :: Apache Software License", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3.6" ], "description": "scrapy\\_loaders\n===============\n\nScrapy Pipelines Loaders\n\n- Free software: MIT license\n\nInstall\n=======\n\n.. code:: bash\n\n pip install scrapy_loaders\n\nAt Your Scrapy project (Example: SpiderProject)\n-----------------------------------------------\n\nmodels.py\n^^^^^^^^^\n\n.. code:: python\n\n from sqlalchemy import (\n Column,\n String,\n Text,\n )\n from sqlalchemy.ext.declarative import declarative_base\n DeclarativeBase = declarative_base()\n\n class ItemModel(DeclarativeBase):\n __tablename__ = 'table_name'\n id = Column('id', String(10), primary_key=True)\n name = Column('name', String(60))\n description = Column('description', Text())\n url = Column('url', Text())\n md5sum = Column('md5sum', String(45))\n ...\n ...\n\nsettings.py\n^^^^^^^^^^^\n\n.. code:: python\n\n ...\n # Postgres settings, check other SQLAlchemy settings if you wish\n DATABASE = {\n 'drivername': 'postgresql+psycopg2',\n 'host': 'localhost',\n 'port': '5432',\n 'username': 'username',\n 'password': 'password',\n 'database': 'attack_mitre',\n }\n DECLARATIVE_BASE = 'SpiderProject.models.DeclarativeBase'\n ...\n ITEM_PIPELINES = {\n 'SpiderProject.pipelines.SpiderProjectDbPipeline': 300,\n }\n ...\n\npipelines.py\n^^^^^^^^^^^^\n\n.. code:: python\n\n from SpiderProject.models import ItemModel\n from scrapy_loaders.db_loaders import DBLoader\n from scrapy_loaders.pipelines import DbPipeline\n\n class ItemLoader(DBLoader):\n model = ItemModel\n hash_fields = ['name', 'description']\n update_fields = hash_fields + ['md5sum']\n ...\n\n class SpiderProjectDbPipeline(DbPipeline):\n db_loaders = {\n 'Item': ItemLoader,\n }\n ...\n\nFeatures\n========\n\nTests\n=====\n\nTODO: tests\n\n.. code:: bash\n\n nosetests --with-coverage --cover-inclusive --cover-package=scrapy_loaders --cover-html\n", "description_content_type": "", "docs_url": null, "download_url": "https://git.herrerosolis.com/Scrapers/scrapy_loaders/-/archive/v0.0.5/scrapy_loaders-v0.0.5.tar.gz", "downloads": { "last_day": -1, "last_month": -1, "last_week": -1 }, "home_page": "https://git.herrerosolis.com/Scrapers/scrapy_loaders", "keywords": "scrapy_loaders", "license": "MIT license", "maintainer": "", "maintainer_email": "", "name": "scrapy_loaders", "package_url": "https://pypi.org/project/scrapy_loaders/", "platform": "", "project_url": "https://pypi.org/project/scrapy_loaders/", "project_urls": { "Download": "https://git.herrerosolis.com/Scrapers/scrapy_loaders/-/archive/v0.0.5/scrapy_loaders-v0.0.5.tar.gz", "Homepage": "https://git.herrerosolis.com/Scrapers/scrapy_loaders" }, "release_url": "https://pypi.org/project/scrapy_loaders/0.0.5/", "requires_dist": null, "requires_python": "", "summary": "Scrapy Database Loader Wrapper with SQLAlchemy", "version": "0.0.5" }, "last_serial": 4160261, "releases": { "0.0.0": [ { "comment_text": "", "digests": { "md5": "9c7fa83a07ff7f293d38a7e6bdf68ee8", "sha256": "65b13727f90e2b797cad35aa17d68b0705ec60817d8fe4804273123e1cf6d813" }, "downloads": -1, "filename": "scrapy_loaders-0.0.0.tar.gz", "has_sig": false, "md5_digest": "9c7fa83a07ff7f293d38a7e6bdf68ee8", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 4281, "upload_time": "2018-08-10T23:38:55", "url": "https://files.pythonhosted.org/packages/77/78/571b5b1b88777c139062725c12b2e3b16bbab0d64b7756077035c63012be/scrapy_loaders-0.0.0.tar.gz" } ], "0.0.1": [ { "comment_text": "", "digests": { "md5": "c8b3020e4a7368254c2b352abd6a6964", "sha256": "3f79877e5e0626470fa84e0d4d4df1e918afdef81d40b05c1621f89ae41b7dcc" }, "downloads": -1, "filename": "scrapy_loaders-0.0.1.tar.gz", "has_sig": false, "md5_digest": "c8b3020e4a7368254c2b352abd6a6964", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 4305, "upload_time": "2018-08-10T23:59:22", "url": "https://files.pythonhosted.org/packages/85/b2/8c796fe987cc67c1462edae57ee85a9d2c81524a9dfeda3c0ff6bc0ecffc/scrapy_loaders-0.0.1.tar.gz" } ], "0.0.2": [ { "comment_text": "", "digests": { "md5": "b77b951267e4037735d154edf6d4c5dc", "sha256": "c4751941142f81796b0731b354a39733c7e361f9311996605a6e95a70688d0b8" }, "downloads": -1, "filename": "scrapy_loaders-0.0.2.tar.gz", "has_sig": false, "md5_digest": "b77b951267e4037735d154edf6d4c5dc", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 4301, "upload_time": "2018-08-11T00:58:18", "url": "https://files.pythonhosted.org/packages/94/ae/0e4efe0b0850b1b4325b2c8326fc94f85d6dc33915fd879c425742659f83/scrapy_loaders-0.0.2.tar.gz" } ], "0.0.3": [ { "comment_text": "", "digests": { "md5": "7497b8f7746cbca13ca64e303c7a0b62", "sha256": "88b40e7bd9d0754822ffba73acfb74baf37e78e25c1b4f4d015d5efb75b00043" }, "downloads": -1, "filename": "scrapy_loaders-0.0.3.tar.gz", "has_sig": false, "md5_digest": "7497b8f7746cbca13ca64e303c7a0b62", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 4323, "upload_time": "2018-08-11T11:44:30", "url": "https://files.pythonhosted.org/packages/53/3f/0e670117c827d2a7f563a25ba0616fbc6daa2bb8f100ef3fba197b2309c3/scrapy_loaders-0.0.3.tar.gz" } ], "0.0.4": [ { "comment_text": "", "digests": { "md5": "3a302fce695a0de27fd1d76071a81734", "sha256": "9619502dcf7ce1f6ccfe2f544f7e984f5f2f88d5903c877a92f1fd2e58d64cd2" }, "downloads": -1, "filename": "scrapy_loaders-0.0.4.tar.gz", "has_sig": false, "md5_digest": "3a302fce695a0de27fd1d76071a81734", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 4434, "upload_time": "2018-08-11T17:00:34", "url": "https://files.pythonhosted.org/packages/d8/cb/00ad0be942f1ee2465aa4412fa1f79ecb6cfa1c0552d31473f4556583ed6/scrapy_loaders-0.0.4.tar.gz" } ], "0.0.5": [ { "comment_text": "", "digests": { "md5": "ccd84d1655436ee90b2f2dea756f1402", "sha256": "316f2332306fb958e5bddce7dd2be1ff5145962aa6a665516f96548244499739" }, "downloads": -1, "filename": "scrapy_loaders-0.0.5.tar.gz", "has_sig": false, "md5_digest": "ccd84d1655436ee90b2f2dea756f1402", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 4464, "upload_time": "2018-08-11T20:04:52", "url": "https://files.pythonhosted.org/packages/e9/4e/a7494f34fed427d52b8c14ea03d69efc38a82265ada37b7373cca4a9869e/scrapy_loaders-0.0.5.tar.gz" } ] }, "urls": [ { "comment_text": "", "digests": { "md5": "ccd84d1655436ee90b2f2dea756f1402", "sha256": "316f2332306fb958e5bddce7dd2be1ff5145962aa6a665516f96548244499739" }, "downloads": -1, "filename": "scrapy_loaders-0.0.5.tar.gz", "has_sig": false, "md5_digest": "ccd84d1655436ee90b2f2dea756f1402", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 4464, "upload_time": "2018-08-11T20:04:52", "url": "https://files.pythonhosted.org/packages/e9/4e/a7494f34fed427d52b8c14ea03d69efc38a82265ada37b7373cca4a9869e/scrapy_loaders-0.0.5.tar.gz" } ] }