{ "info": { "author": "Srinivasan Rangarajan", "author_email": "srinivasanr@gmail.com", "bugtrack_url": null, "classifiers": [ "Development Status :: 4 - Beta", "Environment :: Console", "Framework :: Scrapy", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python" ], "description": "Scrapy Random User-Agent\n========================\n\nDoes your scrapy spider get identified and blocked by servers because\nyou use the default user-agent or a generic one?\n\nUse this ``random_useragent`` module and set a random user-agent for\nevery request. You are limited only by the number of different\nuser-agents you set in a text file.\n\nInstalling\n----------\n\nInstalling it is pretty simple.\n\n.. code-block:: python\n\n pip install scrapy-random-useragent\n\nUsage\n-----\n\nIn your ``settings.py`` file, update the ``DOWNLOADER_MIDDLEWARES``\nvariable like this.\n\n.. code-block:: python\n\n DOWNLOADER_MIDDLEWARES = {\n 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': None,\n 'random_useragent.RandomUserAgentMiddleware': 400\n }\n\nThis disables the default ``UserAgentMiddleware`` and enables the\n``RandomUserAgentMiddleware``.\n\nThen, create a new variable ``USER_AGENT_LIST`` with the path to your\ntext file which has the list of all user-agents\n(one user-agent per line).\n\n.. code-block:: python\n\n USER_AGENT_LIST = \"/path/to/useragents.txt\"\n\nNow all the requests from your crawler will have a random user-agent\npicked from the text file.", "description_content_type": null, "docs_url": null, "download_url": "UNKNOWN", "downloads": { "last_day": -1, "last_month": -1, "last_week": -1 }, "home_page": "https://github.com/cnu/scrapy-random-useragent", "keywords": "scrapy random user-agent", "license": "MIT", "maintainer": null, "maintainer_email": null, "name": "scrapy-random-useragent", "package_url": "https://pypi.org/project/scrapy-random-useragent/", "platform": "Any", "project_url": "https://pypi.org/project/scrapy-random-useragent/", "project_urls": { "Download": "UNKNOWN", "Homepage": "https://github.com/cnu/scrapy-random-useragent" }, "release_url": "https://pypi.org/project/scrapy-random-useragent/0.2/", "requires_dist": null, "requires_python": null, "summary": "Scrapy Middleware to set a random User-Agent for every Request.", "version": "0.2" }, "last_serial": 2161830, "releases": { "0.1": [ { "comment_text": "", "digests": { "md5": "ccb12a85c599fc1b18281ecc20fcacc2", "sha256": "b34520d4e960c377d1ba9e3a95cef6577803fb057161e59cb2a2ffe5d754d790" }, "downloads": -1, "filename": "scrapy-random-useragent-0.1.tar.gz", "has_sig": false, "md5_digest": "ccb12a85c599fc1b18281ecc20fcacc2", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 2324, "upload_time": "2014-12-25T14:29:30", "url": "https://files.pythonhosted.org/packages/e3/05/572ec810fbdca07bb16f2f5b23f0e36b46f2a2362a8e2377e8e27315e974/scrapy-random-useragent-0.1.tar.gz" } ], "0.2": [ { "comment_text": "", "digests": { "md5": "72f21e64f6edf1973441f19e036f62df", "sha256": "570f87e26438f7e1b69890219a6e052b8c510a745a21ae129da8f8fe8161e102" }, "downloads": -1, "filename": "scrapy-random-useragent-0.2.tar.gz", "has_sig": false, "md5_digest": "72f21e64f6edf1973441f19e036f62df", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 2619, "upload_time": "2016-06-11T07:41:56", "url": "https://files.pythonhosted.org/packages/23/2e/3a3ae91faf1d5d31526379186285817bda8ff66a221ec7085a9e549c1465/scrapy-random-useragent-0.2.tar.gz" } ] }, "urls": [ { "comment_text": "", "digests": { "md5": "72f21e64f6edf1973441f19e036f62df", "sha256": "570f87e26438f7e1b69890219a6e052b8c510a745a21ae129da8f8fe8161e102" }, "downloads": -1, "filename": "scrapy-random-useragent-0.2.tar.gz", "has_sig": false, "md5_digest": "72f21e64f6edf1973441f19e036f62df", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 2619, "upload_time": "2016-06-11T07:41:56", "url": "https://files.pythonhosted.org/packages/23/2e/3a3ae91faf1d5d31526379186285817bda8ff66a221ec7085a9e549c1465/scrapy-random-useragent-0.2.tar.gz" } ] }