![]() ![]() """ ) load_task = PythonOperator ( task_id = "load", python_callable = load, ) load_task. ![]() This computed value is then put into xcom, so that it can be processed by the next task. doc_md = dedent ( """\ # Transform task A simple Transform task which takes in the collection of order data from xcom and computes the total order value. """ ) transform_task = PythonOperator ( task_id = "transform", python_callable = transform, ) transform_task. This data is then put into xcom, so that it can be processed by the next task. In this case, getting data is simulated by reading from a hardcoded JSON string. doc_md = dedent ( """\ # Extract task A simple Extract task to get data ready for the rest of the data pipeline. loads ( total_value_string ) print ( total_order_value ) extract_task = PythonOperator ( task_id = "extract", python_callable = extract, ) extract_task. xcom_pull ( task_ids = "transform", key = "total_order_value" ) total_order_value = json. xcom_push ( "total_order_value", total_value_json_string ) def load ( ** kwargs ): ti = kwargs total_value_string = ti. """ data_string = ' total_value_json_string = json. Documentation that goes along with the Airflow TaskFlow API tutorial is located () """ () def extract (): """ # Extract task A simple Extract task to get data ready for the rest of the data pipeline. datetime ( 2021, 1, 1, tz = "UTC" ), catchup = False, tags =, ) def tutorial_taskflow_api (): """ # TaskFlow API Tutorial Documentation This is a simple data pipeline example which demonstrates the use of the TaskFlow API using three simple tasks for Extract, Transform, and Load. Import json import pendulum from corators import dag, task ( schedule = None, start_date = pendulum.
0 Comments
Leave a Reply. |
AuthorWrite something about yourself. No need to be fancy, just an overview. ArchivesCategories |