apache-airflow-providers-google 12.0.0rc2__py3-none-any.whl → 13.0.0__py3-none-any.whl

airflow/providers/google/cloud/example_dags/example_facebook_ads_to_gcs.py

@@ -1,141 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-Example Airflow DAG that shows how to use FacebookAdsReportToGcsOperator.
-"""
-
-from __future__ import annotations
-
-import os
-from datetime import datetime
-
-from facebook_business.adobjects.adsinsights import AdsInsights
-
-from airflow.models.baseoperator import chain
-from airflow.models.dag import DAG
-from airflow.providers.google.cloud.operators.bigquery import (
-    BigQueryCreateEmptyDatasetOperator,
-    BigQueryCreateEmptyTableOperator,
-    BigQueryDeleteDatasetOperator,
-    BigQueryInsertJobOperator,
-)
-from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator
-from airflow.providers.google.cloud.transfers.facebook_ads_to_gcs import FacebookAdsReportToGcsOperator
-from airflow.providers.google.cloud.transfers.gcs_to_bigquery import GCSToBigQueryOperator
-
-# [START howto_GCS_env_variables]
-GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "free-tier-1997")
-GCS_BUCKET = os.environ.get("GCS_BUCKET", "airflow_bucket_fb")
-GCS_OBJ_PATH = os.environ.get("GCS_OBJ_PATH", "Temp/this_is_my_report_csv.csv")
-GCS_CONN_ID = os.environ.get("GCS_CONN_ID", "google_cloud_default")
-DATASET_NAME = os.environ.get("DATASET_NAME", "airflow_test_dataset")
-TABLE_NAME = os.environ.get("FB_TABLE_NAME", "airflow_test_datatable")
-# [END howto_GCS_env_variables]
-
-# [START howto_FB_ADS_variables]
-FIELDS = [
-    AdsInsights.Field.campaign_name,
-    AdsInsights.Field.campaign_id,
-    AdsInsights.Field.ad_id,
-    AdsInsights.Field.clicks,
-    AdsInsights.Field.impressions,
-]
-PARAMETERS = {"level": "ad", "date_preset": "yesterday"}
-# [END howto_FB_ADS_variables]
-
-with DAG(
-    "example_facebook_ads_to_gcs",
-    start_date=datetime(2021, 1, 1),
-    catchup=False,
-) as dag:
-    create_bucket = GCSCreateBucketOperator(
-        task_id="create_bucket",
-        bucket_name=GCS_BUCKET,
-        project_id=GCP_PROJECT_ID,
-    )
-
-    create_dataset = BigQueryCreateEmptyDatasetOperator(
-        task_id="create_dataset",
-        dataset_id=DATASET_NAME,
-    )
-
-    create_table = BigQueryCreateEmptyTableOperator(
-        task_id="create_table",
-        dataset_id=DATASET_NAME,
-        table_id=TABLE_NAME,
-        schema_fields=[
-            {"name": "campaign_name", "type": "STRING", "mode": "NULLABLE"},
-            {"name": "campaign_id", "type": "STRING", "mode": "NULLABLE"},
-            {"name": "ad_id", "type": "STRING", "mode": "NULLABLE"},
-            {"name": "clicks", "type": "STRING", "mode": "NULLABLE"},
-            {"name": "impressions", "type": "STRING", "mode": "NULLABLE"},
-        ],
-    )
-
-    # [START howto_operator_facebook_ads_to_gcs]
-    run_operator = FacebookAdsReportToGcsOperator(
-        task_id="run_fetch_data",
-        owner="airflow",
-        bucket_name=GCS_BUCKET,
-        parameters=PARAMETERS,
-        fields=FIELDS,
-        gcp_conn_id=GCS_CONN_ID,
-        object_name=GCS_OBJ_PATH,
-    )
-    # [END howto_operator_facebook_ads_to_gcs]
-
-    load_csv = GCSToBigQueryOperator(
-        task_id="gcs_to_bq_example",
-        bucket=GCS_BUCKET,
-        source_objects=[GCS_OBJ_PATH],
-        destination_project_dataset_table=f"{DATASET_NAME}.{TABLE_NAME}",
-        write_disposition="WRITE_TRUNCATE",
-    )
-
-    read_data_from_gcs_many_chunks = BigQueryInsertJobOperator(
-        task_id="read_data_from_gcs_many_chunks",
-        configuration={
-            "query": {
-                "query": f"SELECT COUNT(*) FROM `{GCP_PROJECT_ID}.{DATASET_NAME}.{TABLE_NAME}`",
-                "useLegacySql": False,
-            }
-        },
-    )
-
-    delete_bucket = GCSDeleteBucketOperator(
-        task_id="delete_bucket",
-        bucket_name=GCS_BUCKET,
-    )
-
-    delete_dataset = BigQueryDeleteDatasetOperator(
-        task_id="delete_dataset",
-        project_id=GCP_PROJECT_ID,
-        dataset_id=DATASET_NAME,
-        delete_contents=True,
-    )
-
-    chain(
-        create_bucket,
-        create_dataset,
-        create_table,
-        run_operator,
-        load_csv,
-        read_data_from_gcs_many_chunks,
-        delete_bucket,
-        delete_dataset,
-    )

airflow/providers/google/cloud/example_dags/example_looker.py

@@ -1,64 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-Example Airflow DAG that show how to use various Looker
-operators to submit PDT materialization job and manage it.
-"""
-
-from __future__ import annotations
-
-from datetime import datetime
-
-from airflow.models.dag import DAG
-from airflow.providers.google.cloud.operators.looker import LookerStartPdtBuildOperator
-from airflow.providers.google.cloud.sensors.looker import LookerCheckPdtBuildSensor
-
-with DAG(
-    dag_id="example_gcp_looker",
-    start_date=datetime(2021, 1, 1),
-    catchup=False,
-) as dag:
-    # [START cloud_looker_async_start_pdt_sensor]
-    start_pdt_task_async = LookerStartPdtBuildOperator(
-        task_id="start_pdt_task_async",
-        looker_conn_id="your_airflow_connection_for_looker",
-        model="your_lookml_model",
-        view="your_lookml_view",
-        asynchronous=True,
-    )
-
-    check_pdt_task_async_sensor = LookerCheckPdtBuildSensor(
-        task_id="check_pdt_task_async_sensor",
-        looker_conn_id="your_airflow_connection_for_looker",
-        materialization_id=start_pdt_task_async.output,
-        poke_interval=10,
-    )
-    # [END cloud_looker_async_start_pdt_sensor]
-
-    # [START how_to_cloud_looker_start_pdt_build_operator]
-    build_pdt_task = LookerStartPdtBuildOperator(
-        task_id="build_pdt_task",
-        looker_conn_id="your_airflow_connection_for_looker",
-        model="your_lookml_model",
-        view="your_lookml_view",
-    )
-    # [END how_to_cloud_looker_start_pdt_build_operator]
-
-    start_pdt_task_async >> check_pdt_task_async_sensor
-
-    build_pdt_task

airflow/providers/google/cloud/example_dags/example_presto_to_gcs.py

@@ -1,194 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-Example DAG using PrestoToGCSOperator.
-"""
-
-from __future__ import annotations
-
-import os
-import re
-from datetime import datetime
-
-from airflow.models.dag import DAG
-from airflow.providers.google.cloud.operators.bigquery import (
-    BigQueryCreateEmptyDatasetOperator,
-    BigQueryCreateExternalTableOperator,
-    BigQueryDeleteDatasetOperator,
-    BigQueryInsertJobOperator,
-)
-from airflow.providers.google.cloud.transfers.presto_to_gcs import PrestoToGCSOperator
-
-GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project")
-GCS_BUCKET = os.environ.get("GCP_PRESTO_TO_GCS_BUCKET_NAME", "INVALID BUCKET NAME")
-DATASET_NAME = os.environ.get("GCP_PRESTO_TO_GCS_DATASET_NAME", "test_presto_to_gcs_dataset")
-
-SOURCE_MULTIPLE_TYPES = "memory.default.test_multiple_types"
-SOURCE_CUSTOMER_TABLE = "tpch.sf1.customer"
-
-
-def safe_name(s: str) -> str:
-    """
-    Remove invalid characters for filename
-    """
-    return re.sub("[^0-9a-zA-Z_]+", "_", s)
-
-
-with DAG(
-    dag_id="example_presto_to_gcs",
-    start_date=datetime(2021, 1, 1),
-    catchup=False,
-    tags=["example"],
-) as dag:
-    create_dataset = BigQueryCreateEmptyDatasetOperator(task_id="create-dataset", dataset_id=DATASET_NAME)
-
-    delete_dataset = BigQueryDeleteDatasetOperator(
-        task_id="delete_dataset", dataset_id=DATASET_NAME, delete_contents=True
-    )
-
-    # [START howto_operator_presto_to_gcs_basic]
-    presto_to_gcs_basic = PrestoToGCSOperator(
-        task_id="presto_to_gcs_basic",
-        sql=f"select * from {SOURCE_MULTIPLE_TYPES}",
-        bucket=GCS_BUCKET,
-        filename=f"{safe_name(SOURCE_MULTIPLE_TYPES)}.{{}}.json",
-    )
-    # [END howto_operator_presto_to_gcs_basic]
-
-    # [START howto_operator_presto_to_gcs_multiple_types]
-    presto_to_gcs_multiple_types = PrestoToGCSOperator(
-        task_id="presto_to_gcs_multiple_types",
-        sql=f"select * from {SOURCE_MULTIPLE_TYPES}",
-        bucket=GCS_BUCKET,
-        filename=f"{safe_name(SOURCE_MULTIPLE_TYPES)}.{{}}.json",
-        schema_filename=f"{safe_name(SOURCE_MULTIPLE_TYPES)}-schema.json",
-        gzip=False,
-    )
-    # [END howto_operator_presto_to_gcs_multiple_types]
-
-    # [START howto_operator_create_external_table_multiple_types]
-    create_external_table_multiple_types = BigQueryCreateExternalTableOperator(
-        task_id="create_external_table_multiple_types",
-        bucket=GCS_BUCKET,
-        source_objects=[f"{safe_name(SOURCE_MULTIPLE_TYPES)}.*.json"],
-        table_resource={
-            "tableReference": {
-                "projectId": GCP_PROJECT_ID,
-                "datasetId": DATASET_NAME,
-                "tableId": f"{safe_name(SOURCE_MULTIPLE_TYPES)}",
-            },
-            "schema": {
-                "fields": [
-                    {"name": "name", "type": "STRING"},
-                    {"name": "post_abbr", "type": "STRING"},
-                ]
-            },
-            "externalDataConfiguration": {
-                "sourceFormat": "NEWLINE_DELIMITED_JSON",
-                "compression": "NONE",
-                "csvOptions": {"skipLeadingRows": 1},
-            },
-        },
-        schema_object=f"{safe_name(SOURCE_MULTIPLE_TYPES)}-schema.json",
-    )
-    # [END howto_operator_create_external_table_multiple_types]
-
-    read_data_from_gcs_multiple_types = BigQueryInsertJobOperator(
-        task_id="read_data_from_gcs_multiple_types",
-        configuration={
-            "query": {
-                "query": f"SELECT COUNT(*) FROM `{GCP_PROJECT_ID}.{DATASET_NAME}."
-                f"{safe_name(SOURCE_MULTIPLE_TYPES)}`",
-                "useLegacySql": False,
-            }
-        },
-    )
-
-    # [START howto_operator_presto_to_gcs_many_chunks]
-    presto_to_gcs_many_chunks = PrestoToGCSOperator(
-        task_id="presto_to_gcs_many_chunks",
-        sql=f"select * from {SOURCE_CUSTOMER_TABLE}",
-        bucket=GCS_BUCKET,
-        filename=f"{safe_name(SOURCE_CUSTOMER_TABLE)}.{{}}.json",
-        schema_filename=f"{safe_name(SOURCE_CUSTOMER_TABLE)}-schema.json",
-        approx_max_file_size_bytes=10_000_000,
-        gzip=False,
-    )
-    # [END howto_operator_presto_to_gcs_many_chunks]
-
-    create_external_table_many_chunks = BigQueryCreateExternalTableOperator(
-        task_id="create_external_table_many_chunks",
-        bucket=GCS_BUCKET,
-        table_resource={
-            "tableReference": {
-                "projectId": GCP_PROJECT_ID,
-                "datasetId": DATASET_NAME,
-                "tableId": f"{safe_name(SOURCE_CUSTOMER_TABLE)}",
-            },
-            "schema": {
-                "fields": [
-                    {"name": "name", "type": "STRING"},
-                    {"name": "post_abbr", "type": "STRING"},
-                ]
-            },
-            "externalDataConfiguration": {
-                "sourceFormat": "NEWLINE_DELIMITED_JSON",
-                "compression": "NONE",
-                "csvOptions": {"skipLeadingRows": 1},
-            },
-        },
-        source_objects=[f"{safe_name(SOURCE_CUSTOMER_TABLE)}.*.json"],
-        schema_object=f"{safe_name(SOURCE_CUSTOMER_TABLE)}-schema.json",
-    )
-
-    # [START howto_operator_read_data_from_gcs_many_chunks]
-    read_data_from_gcs_many_chunks = BigQueryInsertJobOperator(
-        task_id="read_data_from_gcs_many_chunks",
-        configuration={
-            "query": {
-                "query": f"SELECT COUNT(*) FROM `{GCP_PROJECT_ID}.{DATASET_NAME}."
-                f"{safe_name(SOURCE_CUSTOMER_TABLE)}`",
-                "useLegacySql": False,
-            }
-        },
-    )
-    # [END howto_operator_read_data_from_gcs_many_chunks]
-
-    # [START howto_operator_presto_to_gcs_csv]
-    presto_to_gcs_csv = PrestoToGCSOperator(
-        task_id="presto_to_gcs_csv",
-        sql=f"select * from {SOURCE_MULTIPLE_TYPES}",
-        bucket=GCS_BUCKET,
-        filename=f"{safe_name(SOURCE_MULTIPLE_TYPES)}.{{}}.csv",
-        schema_filename=f"{safe_name(SOURCE_MULTIPLE_TYPES)}-schema.json",
-        export_format="csv",
-    )
-    # [END howto_operator_presto_to_gcs_csv]
-
-    create_dataset >> presto_to_gcs_basic
-    create_dataset >> presto_to_gcs_multiple_types
-    create_dataset >> presto_to_gcs_many_chunks
-    create_dataset >> presto_to_gcs_csv
-
-    presto_to_gcs_multiple_types >> create_external_table_multiple_types >> read_data_from_gcs_multiple_types
-    presto_to_gcs_many_chunks >> create_external_table_many_chunks >> read_data_from_gcs_many_chunks
-
-    presto_to_gcs_basic >> delete_dataset
-    presto_to_gcs_csv >> delete_dataset
-    read_data_from_gcs_multiple_types >> delete_dataset
-    read_data_from_gcs_many_chunks >> delete_dataset

airflow/providers/google/cloud/example_dags/example_salesforce_to_gcs.py

@@ -1,129 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-Example Airflow DAG that shows how to use SalesforceToGcsOperator.
-"""
-
-from __future__ import annotations
-
-import os
-from datetime import datetime
-
-from airflow.models.dag import DAG
-from airflow.providers.google.cloud.operators.bigquery import (
-    BigQueryCreateEmptyDatasetOperator,
-    BigQueryCreateEmptyTableOperator,
-    BigQueryDeleteDatasetOperator,
-    BigQueryInsertJobOperator,
-)
-from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator
-from airflow.providers.google.cloud.transfers.gcs_to_bigquery import GCSToBigQueryOperator
-from airflow.providers.google.cloud.transfers.salesforce_to_gcs import SalesforceToGcsOperator
-
-GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project")
-GCS_BUCKET = os.environ.get("GCS_BUCKET", "airflow-salesforce-bucket")
-DATASET_NAME = os.environ.get("SALESFORCE_DATASET_NAME", "salesforce_test_dataset")
-TABLE_NAME = os.environ.get("SALESFORCE_TABLE_NAME", "salesforce_test_datatable")
-GCS_OBJ_PATH = os.environ.get("GCS_OBJ_PATH", "results.csv")
-QUERY = "SELECT Id, Name, Company, Phone, Email, CreatedDate, LastModifiedDate, IsDeleted FROM Lead"
-GCS_CONN_ID = os.environ.get("GCS_CONN_ID", "google_cloud_default")
-SALESFORCE_CONN_ID = os.environ.get("SALESFORCE_CONN_ID", "salesforce_default")
-
-
-with DAG(
-    "example_salesforce_to_gcs",
-    start_date=datetime(2021, 1, 1),
-    catchup=False,
-) as dag:
-    create_bucket = GCSCreateBucketOperator(
-        task_id="create_bucket",
-        bucket_name=GCS_BUCKET,
-        project_id=GCP_PROJECT_ID,
-        gcp_conn_id=GCS_CONN_ID,
-    )
-
-    # [START howto_operator_salesforce_to_gcs]
-    gcs_upload_task = SalesforceToGcsOperator(
-        query=QUERY,
-        include_deleted=True,
-        bucket_name=GCS_BUCKET,
-        object_name=GCS_OBJ_PATH,
-        salesforce_conn_id=SALESFORCE_CONN_ID,
-        export_format="csv",
-        coerce_to_timestamp=False,
-        record_time_added=False,
-        gcp_conn_id=GCS_CONN_ID,
-        task_id="upload_to_gcs",
-        dag=dag,
-    )
-    # [END howto_operator_salesforce_to_gcs]
-
-    create_dataset = BigQueryCreateEmptyDatasetOperator(
-        task_id="create_dataset", dataset_id=DATASET_NAME, project_id=GCP_PROJECT_ID, gcp_conn_id=GCS_CONN_ID
-    )
-
-    create_table = BigQueryCreateEmptyTableOperator(
-        task_id="create_table",
-        dataset_id=DATASET_NAME,
-        table_id=TABLE_NAME,
-        schema_fields=[
-            {"name": "id", "type": "STRING", "mode": "NULLABLE"},
-            {"name": "name", "type": "STRING", "mode": "NULLABLE"},
-            {"name": "company", "type": "STRING", "mode": "NULLABLE"},
-            {"name": "phone", "type": "STRING", "mode": "NULLABLE"},
-            {"name": "email", "type": "STRING", "mode": "NULLABLE"},
-            {"name": "createddate", "type": "STRING", "mode": "NULLABLE"},
-            {"name": "lastmodifieddate", "type": "STRING", "mode": "NULLABLE"},
-            {"name": "isdeleted", "type": "BOOL", "mode": "NULLABLE"},
-        ],
-    )
-
-    load_csv = GCSToBigQueryOperator(
-        task_id="gcs_to_bq",
-        bucket=GCS_BUCKET,
-        source_objects=[GCS_OBJ_PATH],
-        destination_project_dataset_table=f"{DATASET_NAME}.{TABLE_NAME}",
-        write_disposition="WRITE_TRUNCATE",
-    )
-
-    read_data_from_gcs = BigQueryInsertJobOperator(
-        task_id="read_data_from_gcs",
-        configuration={
-            "query": {
-                "query": f"SELECT COUNT(*) FROM `{GCP_PROJECT_ID}.{DATASET_NAME}.{TABLE_NAME}`",
-                "useLegacySql": False,
-            }
-        },
-    )
-
-    delete_bucket = GCSDeleteBucketOperator(
-        task_id="delete_bucket",
-        bucket_name=GCS_BUCKET,
-    )
-
-    delete_dataset = BigQueryDeleteDatasetOperator(
-        task_id="delete_dataset",
-        project_id=GCP_PROJECT_ID,
-        dataset_id=DATASET_NAME,
-        delete_contents=True,
-    )
-
-    create_bucket >> gcs_upload_task >> load_csv
-    create_dataset >> create_table >> load_csv
-    load_csv >> read_data_from_gcs
-    read_data_from_gcs >> delete_bucket
-    read_data_from_gcs >> delete_dataset

airflow/providers/google/marketing_platform/example_dags/__init__.py

@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.