{ "info": { "author": "Marquez Team", "author_email": "", "bugtrack_url": null, "classifiers": [], "description": "# marquez-airflow\n\n[![CircleCI](https://circleci.com/gh/MarquezProject/marquez-airflow/tree/master.svg?style=shield)](https://circleci.com/gh/MarquezProject/marquez-airflow/tree/master)\n\nA library that integrates Airflow DAGs with [Marquez](https://github.com/MarquezProject/marquez) for automatic metadata collection.\n\n## Status\n\nThis library is under active development at [The We Company](https://www.we.co).\n\n## Requirements\n\n - [Python 3.5.0](https://www.python.org/downloads)+\n - [Airflow 1.10.3](https://pypi.org/project/apache-airflow)+\n\n## Installation\n\n```bash\n$ pip install marquez-airflow\n```\n\nTo install from source run:\n\n```bash\n$ python setup.py install\n```\n \n ## Usage\n \n Once the library is installed in your system, your current DAGs need to be modified slightly by changing the import of `airflow.models.DAG` to `marquez.airflow.DAG`, see example below:\n \n```python\nfrom marquez_airflow import DAG\nfrom airflow.operators.dummy_operator import DummyOperator\n\n\nDAG_NAME = 'my_DAG_name'\n\ndefault_args = {\n 'marquez_location': 'github://data-dags/dag_location/',\n 'marquez_input_urns': [\"s3://some_data\", \"s3://more_data\"],\n 'marquez_output_urns': [\"s3://output_data\"],\n \n 'owner': ...,\n 'depends_on_past': False,\n 'start_date': ...,\n}\n\ndag = DAG(DAG_NAME, schedule_interval='*/10 * * * *',\n default_args=default_args, description=\"yet another DAG\")\n\nrun_this = DummyOperator(task_id='run_this', dag=dag)\nrun_this_too = DummyOperator(task_id='run_this_too', dag=dag)\nrun_this_too.set_upstream(run_this)\n```\n\n## Contributing\n\nSee [CONTRIBUTING.md](https://github.com/MarquezProject/marquez-airflow/blob/master/CONTRIBUTING.md) for more details about how to contribute.", "description_content_type": "text/markdown", "docs_url": null, "download_url": "", "downloads": { "last_day": -1, "last_month": -1, "last_week": -1 }, "home_page": "https://github.com/MarquezProject/marquez-airflow", "keywords": "", "license": "", "maintainer": "", "maintainer_email": "", "name": "marquez-airflow", "package_url": "https://pypi.org/project/marquez-airflow/", "platform": "", "project_url": "https://pypi.org/project/marquez-airflow/", "project_urls": { "Homepage": "https://github.com/MarquezProject/marquez-airflow" }, "release_url": "https://pypi.org/project/marquez-airflow/0.1.6/", "requires_dist": null, "requires_python": "", "summary": "Marquez integration with Airflow", "version": "0.1.6" }, "last_serial": 5359854, "releases": { "0.0.1": [ { "comment_text": "", "digests": { "md5": "8a817b45306cf972ea147aaf2a15c646", "sha256": "aa4346695426dcd925b2f223545cf01c81e74088441ece522517b2f1fbe7daea" }, "downloads": -1, "filename": "marquez_airflow-0.0.1-py3-none-any.whl", "has_sig": false, "md5_digest": "8a817b45306cf972ea147aaf2a15c646", "packagetype": "bdist_wheel", "python_version": "py3", "requires_python": null, "size": 7167, "upload_time": "2019-02-08T14:02:38", "url": "https://files.pythonhosted.org/packages/d3/8e/60cf7dc081239f308d48058411140b6a76ea547070484f1df0f9aa8b5539/marquez_airflow-0.0.1-py3-none-any.whl" }, { "comment_text": "", "digests": { "md5": "873ffa2b48c0765ca747675aea9b3168", "sha256": "ef753919551bbfc1dd0d85171928c6d29a9449875f503d92f02d6ffdef957260" }, "downloads": -1, "filename": "marquez-airflow-0.0.1.tar.gz", "has_sig": false, "md5_digest": "873ffa2b48c0765ca747675aea9b3168", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 2612, "upload_time": "2019-02-08T14:02:40", "url": "https://files.pythonhosted.org/packages/b5/50/864c4a5491f1ccf767939704920f19e5e5e3bc29691b9d6f0efa241d5e3f/marquez-airflow-0.0.1.tar.gz" } ], "0.0.2": [ { "comment_text": "", "digests": { "md5": "c8087b0c4d8fde19388cd7ff5b7f71b8", "sha256": "6dc4baf9d18c2e64968d30268573b8217bfeef479f26f2079e4b4cff0b35e1f4" }, "downloads": -1, "filename": "marquez_airflow-0.0.2-py3-none-any.whl", "has_sig": false, "md5_digest": "c8087b0c4d8fde19388cd7ff5b7f71b8", "packagetype": "bdist_wheel", "python_version": "py3", "requires_python": null, "size": 7176, "upload_time": "2019-02-08T14:21:54", "url": "https://files.pythonhosted.org/packages/19/63/6226a3c6ea025de5abe417ce6c0fd775e8e394848dcd2e03016fe117fea3/marquez_airflow-0.0.2-py3-none-any.whl" }, { "comment_text": "", "digests": { "md5": "ee369dc51567d74391e81fd1bd368418", "sha256": "8741981ddf13db4cfa7b3af2a4a1b5bf8d09ab2826926bdc156af85a805168ab" }, "downloads": -1, "filename": "marquez-airflow-0.0.2.tar.gz", "has_sig": false, "md5_digest": "ee369dc51567d74391e81fd1bd368418", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 2609, "upload_time": "2019-02-08T14:21:55", "url": "https://files.pythonhosted.org/packages/92/fe/cb2daa14840b51e1bb38500244b0f7bcbec915aee25e71815307a77a2c37/marquez-airflow-0.0.2.tar.gz" } ], "0.0.3": [ { "comment_text": "", "digests": { "md5": "6456f1e6955bbbd6b3cc08a283937545", "sha256": "b2f420eefa2aba23be556c900ee9b1e7b9082dd59ee090fc7fca939055a9d411" }, "downloads": -1, "filename": "marquez-airflow-0.0.3.tar.gz", "has_sig": false, "md5_digest": "6456f1e6955bbbd6b3cc08a283937545", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 3921, "upload_time": "2019-03-04T23:26:12", "url": "https://files.pythonhosted.org/packages/1d/7f/a707626d7c9a0813dd2d22860632f92f10430921b1fa6f0f37591327a3e4/marquez-airflow-0.0.3.tar.gz" } ], "0.1.2": [ { "comment_text": "", "digests": { "md5": "c927c453df9456c2acc7b44160644f4a", "sha256": "41b5c9735da841d75f590b79ee100dd876ccd9d7d90156cd32cc42b5494dfdee" }, "downloads": -1, "filename": "marquez-airflow-0.1.2.tar.gz", "has_sig": false, "md5_digest": "c927c453df9456c2acc7b44160644f4a", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 4444, "upload_time": "2019-03-20T22:52:46", "url": "https://files.pythonhosted.org/packages/1c/ec/20accd8dfcee16c77ea7ce85f8517ceb3bb9e070a7df298e26353ec77aa9/marquez-airflow-0.1.2.tar.gz" } ], "0.1.3": [ { "comment_text": "", "digests": { "md5": "ebb3eccea75502c0cbc72bb9b466d237", "sha256": "b6bb9cba9ffcc4be80cb608a75e64546f65d4c3a20a4714a5a1744b15f760883" }, "downloads": -1, "filename": "marquez-airflow-0.1.3.tar.gz", "has_sig": false, "md5_digest": "ebb3eccea75502c0cbc72bb9b466d237", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 4408, "upload_time": "2019-03-21T17:11:39", "url": "https://files.pythonhosted.org/packages/f5/96/e929bf1693062225e6897524af2a1ab08f6ad6a62e0672092910d810949c/marquez-airflow-0.1.3.tar.gz" } ], "0.1.4": [ { "comment_text": "", "digests": { "md5": "586b0097d395cfd8a2359e4fc0cc5f63", "sha256": "2c466bb53e349f7c90b7f4b870e1bfdd011c8dd0b787d8849aea0735ebe6b371" }, "downloads": -1, "filename": "marquez-airflow-0.1.4.tar.gz", "has_sig": false, "md5_digest": "586b0097d395cfd8a2359e4fc0cc5f63", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 4379, "upload_time": "2019-03-22T21:47:16", "url": "https://files.pythonhosted.org/packages/02/a7/d7ea8dd01122372b7072f2e6868fa2a21a29c418362a57cff3d5ad049e05/marquez-airflow-0.1.4.tar.gz" } ], "0.1.5": [ { "comment_text": "", "digests": { "md5": "ea2f37d5cad32f80b5cd4cf27012843e", "sha256": "e097bb8cf5650dae1ef61645224094e61019538893d5a3001027a66d82c228e8" }, "downloads": -1, "filename": "marquez-airflow-0.1.5.tar.gz", "has_sig": false, "md5_digest": "ea2f37d5cad32f80b5cd4cf27012843e", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 4414, "upload_time": "2019-04-04T20:50:52", "url": "https://files.pythonhosted.org/packages/cf/a0/da8e1a40fb4ddeaa0e43a41164fcb56d18092f632d282ddef308adf2f7df/marquez-airflow-0.1.5.tar.gz" } ], "0.1.6": [ { "comment_text": "", "digests": { "md5": "d7402a313ede9b161c6d650cfe6050de", "sha256": "8263a494064cbd0c9672e6792de9a60f460f1fef2ad8b9f5d740aceb676feed0" }, "downloads": -1, "filename": "marquez-airflow-0.1.6.tar.gz", "has_sig": false, "md5_digest": "d7402a313ede9b161c6d650cfe6050de", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 4196, "upload_time": "2019-06-04T22:02:01", "url": "https://files.pythonhosted.org/packages/84/1f/28b2b0715eb8b297eb58a32040d56dbda8f4105563bf9f705145361a111c/marquez-airflow-0.1.6.tar.gz" } ] }, "urls": [ { "comment_text": "", "digests": { "md5": "d7402a313ede9b161c6d650cfe6050de", "sha256": "8263a494064cbd0c9672e6792de9a60f460f1fef2ad8b9f5d740aceb676feed0" }, "downloads": -1, "filename": "marquez-airflow-0.1.6.tar.gz", "has_sig": false, "md5_digest": "d7402a313ede9b161c6d650cfe6050de", "packagetype": "sdist", "python_version": "source", "requires_python": null, "size": 4196, "upload_time": "2019-06-04T22:02:01", "url": "https://files.pythonhosted.org/packages/84/1f/28b2b0715eb8b297eb58a32040d56dbda8f4105563bf9f705145361a111c/marquez-airflow-0.1.6.tar.gz" } ] }