apache-airflow-providers-google 10.17.0rc1__py3-none-any.whl → 10.18.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/google/__init__.py +3 -3
- airflow/providers/google/cloud/hooks/automl.py +1 -1
- airflow/providers/google/cloud/hooks/bigquery.py +64 -33
- airflow/providers/google/cloud/hooks/cloud_composer.py +250 -2
- airflow/providers/google/cloud/hooks/cloud_sql.py +154 -7
- airflow/providers/google/cloud/hooks/cloud_storage_transfer_service.py +7 -2
- airflow/providers/google/cloud/hooks/compute_ssh.py +2 -1
- airflow/providers/google/cloud/hooks/dataflow.py +246 -32
- airflow/providers/google/cloud/hooks/dataplex.py +6 -2
- airflow/providers/google/cloud/hooks/dlp.py +14 -14
- airflow/providers/google/cloud/hooks/gcs.py +6 -2
- airflow/providers/google/cloud/hooks/gdm.py +2 -2
- airflow/providers/google/cloud/hooks/kubernetes_engine.py +2 -2
- airflow/providers/google/cloud/hooks/mlengine.py +8 -4
- airflow/providers/google/cloud/hooks/pubsub.py +1 -1
- airflow/providers/google/cloud/hooks/secret_manager.py +252 -4
- airflow/providers/google/cloud/hooks/vertex_ai/custom_job.py +1431 -74
- airflow/providers/google/cloud/links/vertex_ai.py +2 -1
- airflow/providers/google/cloud/log/gcs_task_handler.py +2 -1
- airflow/providers/google/cloud/operators/automl.py +13 -12
- airflow/providers/google/cloud/operators/bigquery.py +36 -22
- airflow/providers/google/cloud/operators/bigquery_dts.py +4 -3
- airflow/providers/google/cloud/operators/bigtable.py +7 -6
- airflow/providers/google/cloud/operators/cloud_build.py +12 -11
- airflow/providers/google/cloud/operators/cloud_composer.py +147 -2
- airflow/providers/google/cloud/operators/cloud_memorystore.py +17 -16
- airflow/providers/google/cloud/operators/cloud_sql.py +60 -17
- airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py +35 -16
- airflow/providers/google/cloud/operators/compute.py +12 -11
- airflow/providers/google/cloud/operators/datacatalog.py +21 -20
- airflow/providers/google/cloud/operators/dataflow.py +59 -42
- airflow/providers/google/cloud/operators/datafusion.py +11 -10
- airflow/providers/google/cloud/operators/datapipeline.py +3 -2
- airflow/providers/google/cloud/operators/dataprep.py +5 -4
- airflow/providers/google/cloud/operators/dataproc.py +19 -16
- airflow/providers/google/cloud/operators/datastore.py +8 -7
- airflow/providers/google/cloud/operators/dlp.py +31 -30
- airflow/providers/google/cloud/operators/functions.py +4 -3
- airflow/providers/google/cloud/operators/gcs.py +66 -41
- airflow/providers/google/cloud/operators/kubernetes_engine.py +232 -12
- airflow/providers/google/cloud/operators/life_sciences.py +2 -1
- airflow/providers/google/cloud/operators/mlengine.py +11 -10
- airflow/providers/google/cloud/operators/pubsub.py +6 -5
- airflow/providers/google/cloud/operators/spanner.py +7 -6
- airflow/providers/google/cloud/operators/speech_to_text.py +2 -1
- airflow/providers/google/cloud/operators/stackdriver.py +11 -10
- airflow/providers/google/cloud/operators/tasks.py +14 -13
- airflow/providers/google/cloud/operators/text_to_speech.py +2 -1
- airflow/providers/google/cloud/operators/translate_speech.py +2 -1
- airflow/providers/google/cloud/operators/vertex_ai/custom_job.py +333 -26
- airflow/providers/google/cloud/operators/vertex_ai/generative_model.py +20 -12
- airflow/providers/google/cloud/operators/vertex_ai/pipeline_job.py +0 -1
- airflow/providers/google/cloud/operators/vision.py +13 -12
- airflow/providers/google/cloud/operators/workflows.py +10 -9
- airflow/providers/google/cloud/secrets/secret_manager.py +2 -1
- airflow/providers/google/cloud/sensors/bigquery_dts.py +2 -1
- airflow/providers/google/cloud/sensors/bigtable.py +2 -1
- airflow/providers/google/cloud/sensors/cloud_storage_transfer_service.py +2 -1
- airflow/providers/google/cloud/sensors/dataflow.py +239 -52
- airflow/providers/google/cloud/sensors/datafusion.py +2 -1
- airflow/providers/google/cloud/sensors/dataproc.py +3 -2
- airflow/providers/google/cloud/sensors/gcs.py +14 -12
- airflow/providers/google/cloud/sensors/tasks.py +2 -1
- airflow/providers/google/cloud/sensors/workflows.py +2 -1
- airflow/providers/google/cloud/transfers/adls_to_gcs.py +8 -2
- airflow/providers/google/cloud/transfers/azure_blob_to_gcs.py +7 -1
- airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py +7 -1
- airflow/providers/google/cloud/transfers/bigquery_to_gcs.py +2 -1
- airflow/providers/google/cloud/transfers/bigquery_to_mssql.py +1 -1
- airflow/providers/google/cloud/transfers/bigquery_to_sql.py +1 -0
- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +5 -6
- airflow/providers/google/cloud/transfers/gcs_to_gcs.py +22 -12
- airflow/providers/google/cloud/triggers/bigquery.py +14 -3
- airflow/providers/google/cloud/triggers/cloud_composer.py +68 -0
- airflow/providers/google/cloud/triggers/cloud_sql.py +2 -1
- airflow/providers/google/cloud/triggers/cloud_storage_transfer_service.py +2 -1
- airflow/providers/google/cloud/triggers/dataflow.py +504 -4
- airflow/providers/google/cloud/triggers/dataproc.py +110 -26
- airflow/providers/google/cloud/triggers/mlengine.py +2 -1
- airflow/providers/google/cloud/triggers/vertex_ai.py +94 -0
- airflow/providers/google/common/hooks/base_google.py +45 -7
- airflow/providers/google/firebase/hooks/firestore.py +2 -2
- airflow/providers/google/firebase/operators/firestore.py +2 -1
- airflow/providers/google/get_provider_info.py +3 -2
- {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0rc1.dist-info}/METADATA +8 -8
- {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0rc1.dist-info}/RECORD +88 -89
- airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py +0 -289
- {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_google-10.17.0rc1.dist-info → apache_airflow_providers_google-10.18.0rc1.dist-info}/entry_points.txt +0 -0
@@ -25,7 +25,8 @@ if TYPE_CHECKING:
|
|
25
25
|
|
26
26
|
VERTEX_AI_BASE_LINK = "/vertex-ai"
|
27
27
|
VERTEX_AI_MODEL_LINK = (
|
28
|
-
VERTEX_AI_BASE_LINK
|
28
|
+
VERTEX_AI_BASE_LINK
|
29
|
+
+ "/models/locations/{region}/models/{model_id}/versions/default/properties?project={project_id}"
|
29
30
|
)
|
30
31
|
VERTEX_AI_MODEL_LIST_LINK = VERTEX_AI_BASE_LINK + "/models?project={project_id}"
|
31
32
|
VERTEX_AI_MODEL_EXPORT_LINK = "/storage/browser/{bucket_name}/model-{model_id}?project={project_id}"
|
@@ -32,6 +32,7 @@ from airflow.exceptions import AirflowNotFoundException
|
|
32
32
|
from airflow.providers.google.cloud.hooks.gcs import GCSHook, _parse_gcs_url
|
33
33
|
from airflow.providers.google.cloud.utils.credentials_provider import get_credentials_and_project_id
|
34
34
|
from airflow.providers.google.common.consts import CLIENT_INFO
|
35
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
35
36
|
from airflow.utils.log.file_task_handler import FileTaskHandler
|
36
37
|
from airflow.utils.log.logging_mixin import LoggingMixin
|
37
38
|
|
@@ -82,7 +83,7 @@ class GCSTaskHandler(FileTaskHandler, LoggingMixin):
|
|
82
83
|
gcp_key_path: str | None = None,
|
83
84
|
gcp_keyfile_dict: dict | None = None,
|
84
85
|
gcp_scopes: Collection[str] | None = _DEFAULT_SCOPESS,
|
85
|
-
project_id: str
|
86
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
86
87
|
**kwargs,
|
87
88
|
):
|
88
89
|
super().__init__(base_log_folder, filename_template)
|
@@ -43,6 +43,7 @@ from airflow.providers.google.cloud.links.automl import (
|
|
43
43
|
AutoMLModelTrainLink,
|
44
44
|
)
|
45
45
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
46
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
46
47
|
|
47
48
|
if TYPE_CHECKING:
|
48
49
|
from google.api_core.retry import Retry
|
@@ -100,7 +101,7 @@ class AutoMLTrainModelOperator(GoogleCloudBaseOperator):
|
|
100
101
|
*,
|
101
102
|
model: dict,
|
102
103
|
location: str,
|
103
|
-
project_id: str
|
104
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
104
105
|
metadata: MetaData = (),
|
105
106
|
timeout: float | None = None,
|
106
107
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -209,7 +210,7 @@ class AutoMLPredictOperator(GoogleCloudBaseOperator):
|
|
209
210
|
location: str,
|
210
211
|
payload: dict,
|
211
212
|
operation_params: dict[str, str] | None = None,
|
212
|
-
project_id: str
|
213
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
213
214
|
metadata: MetaData = (),
|
214
215
|
timeout: float | None = None,
|
215
216
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -312,7 +313,7 @@ class AutoMLBatchPredictOperator(GoogleCloudBaseOperator):
|
|
312
313
|
input_config: dict,
|
313
314
|
output_config: dict,
|
314
315
|
location: str,
|
315
|
-
project_id: str
|
316
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
316
317
|
prediction_params: dict[str, str] | None = None,
|
317
318
|
metadata: MetaData = (),
|
318
319
|
timeout: float | None = None,
|
@@ -409,7 +410,7 @@ class AutoMLCreateDatasetOperator(GoogleCloudBaseOperator):
|
|
409
410
|
*,
|
410
411
|
dataset: dict,
|
411
412
|
location: str,
|
412
|
-
project_id: str
|
413
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
413
414
|
metadata: MetaData = (),
|
414
415
|
timeout: float | None = None,
|
415
416
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -504,7 +505,7 @@ class AutoMLImportDataOperator(GoogleCloudBaseOperator):
|
|
504
505
|
dataset_id: str,
|
505
506
|
location: str,
|
506
507
|
input_config: dict,
|
507
|
-
project_id: str
|
508
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
508
509
|
metadata: MetaData = (),
|
509
510
|
timeout: float | None = None,
|
510
511
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -608,7 +609,7 @@ class AutoMLTablesListColumnSpecsOperator(GoogleCloudBaseOperator):
|
|
608
609
|
field_mask: dict | None = None,
|
609
610
|
filter_: str | None = None,
|
610
611
|
page_size: int | None = None,
|
611
|
-
project_id: str
|
612
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
612
613
|
metadata: MetaData = (),
|
613
614
|
timeout: float | None = None,
|
614
615
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -790,7 +791,7 @@ class AutoMLGetModelOperator(GoogleCloudBaseOperator):
|
|
790
791
|
*,
|
791
792
|
model_id: str,
|
792
793
|
location: str,
|
793
|
-
project_id: str
|
794
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
794
795
|
metadata: MetaData = (),
|
795
796
|
timeout: float | None = None,
|
796
797
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -876,7 +877,7 @@ class AutoMLDeleteModelOperator(GoogleCloudBaseOperator):
|
|
876
877
|
*,
|
877
878
|
model_id: str,
|
878
879
|
location: str,
|
879
|
-
project_id: str
|
880
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
880
881
|
metadata: MetaData = (),
|
881
882
|
timeout: float | None = None,
|
882
883
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -962,7 +963,7 @@ class AutoMLDeployModelOperator(GoogleCloudBaseOperator):
|
|
962
963
|
*,
|
963
964
|
model_id: str,
|
964
965
|
location: str,
|
965
|
-
project_id: str
|
966
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
966
967
|
image_detection_metadata: dict | None = None,
|
967
968
|
metadata: Sequence[tuple[str, str]] = (),
|
968
969
|
timeout: float | None = None,
|
@@ -1053,7 +1054,7 @@ class AutoMLTablesListTableSpecsOperator(GoogleCloudBaseOperator):
|
|
1053
1054
|
location: str,
|
1054
1055
|
page_size: int | None = None,
|
1055
1056
|
filter_: str | None = None,
|
1056
|
-
project_id: str
|
1057
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1057
1058
|
metadata: MetaData = (),
|
1058
1059
|
timeout: float | None = None,
|
1059
1060
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -1141,7 +1142,7 @@ class AutoMLListDatasetOperator(GoogleCloudBaseOperator):
|
|
1141
1142
|
self,
|
1142
1143
|
*,
|
1143
1144
|
location: str,
|
1144
|
-
project_id: str
|
1145
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1145
1146
|
metadata: MetaData = (),
|
1146
1147
|
timeout: float | None = None,
|
1147
1148
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -1226,7 +1227,7 @@ class AutoMLDeleteDatasetOperator(GoogleCloudBaseOperator):
|
|
1226
1227
|
*,
|
1227
1228
|
dataset_id: str | list[str],
|
1228
1229
|
location: str,
|
1229
|
-
project_id: str
|
1230
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1230
1231
|
metadata: MetaData = (),
|
1231
1232
|
timeout: float | None = None,
|
1232
1233
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -56,6 +56,7 @@ from airflow.providers.google.cloud.triggers.bigquery import (
|
|
56
56
|
BigQueryValueCheckTrigger,
|
57
57
|
)
|
58
58
|
from airflow.providers.google.cloud.utils.bigquery import convert_job_id
|
59
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
59
60
|
|
60
61
|
if TYPE_CHECKING:
|
61
62
|
from google.api_core.retry import Retry
|
@@ -322,8 +323,26 @@ class BigQueryCheckOperator(_BigQueryDbHookMixin, SQLCheckOperator):
|
|
322
323
|
),
|
323
324
|
method_name="execute_complete",
|
324
325
|
)
|
326
|
+
self._handle_job_error(job)
|
327
|
+
# job.result() returns a RowIterator. Mypy expects an instance of SupportsNext[Any] for
|
328
|
+
# the next() call which the RowIterator does not resemble to. Hence, ignore the arg-type error.
|
329
|
+
# Row passed to _validate_records is a collection of values only, without column names.
|
330
|
+
self._validate_records(next(iter(job.result()), [])) # type: ignore[arg-type]
|
325
331
|
self.log.info("Current state of job %s is %s", job.job_id, job.state)
|
326
332
|
|
333
|
+
@staticmethod
|
334
|
+
def _handle_job_error(job: BigQueryJob | UnknownJob) -> None:
|
335
|
+
if job.error_result:
|
336
|
+
raise AirflowException(f"BigQuery job {job.job_id} failed: {job.error_result}")
|
337
|
+
|
338
|
+
def _validate_records(self, records) -> None:
|
339
|
+
if not records:
|
340
|
+
raise AirflowException(f"The following query returned zero rows: {self.sql}")
|
341
|
+
elif not all(records):
|
342
|
+
self._raise_exception( # type: ignore[attr-defined]
|
343
|
+
f"Test failed.\nQuery:\n{self.sql}\nResults:\n{records!s}"
|
344
|
+
)
|
345
|
+
|
327
346
|
def execute_complete(self, context: Context, event: dict[str, Any]) -> None:
|
328
347
|
"""Act as a callback for when the trigger fires.
|
329
348
|
|
@@ -333,13 +352,7 @@ class BigQueryCheckOperator(_BigQueryDbHookMixin, SQLCheckOperator):
|
|
333
352
|
if event["status"] == "error":
|
334
353
|
raise AirflowException(event["message"])
|
335
354
|
|
336
|
-
|
337
|
-
if not records:
|
338
|
-
raise AirflowException("The query returned empty results")
|
339
|
-
elif not all(records):
|
340
|
-
self._raise_exception( # type: ignore[attr-defined]
|
341
|
-
f"Test failed.\nQuery:\n{self.sql}\nResults:\n{records!s}"
|
342
|
-
)
|
355
|
+
self._validate_records(event["records"])
|
343
356
|
self.log.info("Record: %s", event["records"])
|
344
357
|
self.log.info("Success.")
|
345
358
|
|
@@ -454,8 +467,8 @@ class BigQueryValueCheckOperator(_BigQueryDbHookMixin, SQLValueCheckOperator):
|
|
454
467
|
self._handle_job_error(job)
|
455
468
|
# job.result() returns a RowIterator. Mypy expects an instance of SupportsNext[Any] for
|
456
469
|
# the next() call which the RowIterator does not resemble to. Hence, ignore the arg-type error.
|
457
|
-
|
458
|
-
self.check_value(
|
470
|
+
# Row passed to check_value is a collection of values only, without column names.
|
471
|
+
self.check_value(next(iter(job.result()), [])) # type: ignore[arg-type]
|
459
472
|
self.log.info("Current state of job %s is %s", job.job_id, job.state)
|
460
473
|
|
461
474
|
@staticmethod
|
@@ -542,7 +555,7 @@ class BigQueryIntervalCheckOperator(_BigQueryDbHookMixin, SQLIntervalCheckOperat
|
|
542
555
|
labels: dict | None = None,
|
543
556
|
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
544
557
|
poll_interval: float = 4.0,
|
545
|
-
project_id: str
|
558
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
546
559
|
**kwargs,
|
547
560
|
) -> None:
|
548
561
|
super().__init__(
|
@@ -946,7 +959,7 @@ class BigQueryGetDataOperator(GoogleCloudBaseOperator):
|
|
946
959
|
table_id: str,
|
947
960
|
table_project_id: str | None = None,
|
948
961
|
job_project_id: str | None = None,
|
949
|
-
project_id: str
|
962
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
950
963
|
max_results: int = 100,
|
951
964
|
selected_fields: str | None = None,
|
952
965
|
gcp_conn_id: str = "google_cloud_default",
|
@@ -1447,7 +1460,7 @@ class BigQueryCreateEmptyTableOperator(GoogleCloudBaseOperator):
|
|
1447
1460
|
dataset_id: str,
|
1448
1461
|
table_id: str,
|
1449
1462
|
table_resource: dict[str, Any] | None = None,
|
1450
|
-
project_id: str
|
1463
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1451
1464
|
schema_fields: list | None = None,
|
1452
1465
|
gcs_schema_object: str | None = None,
|
1453
1466
|
time_partitioning: dict | None = None,
|
@@ -1907,7 +1920,7 @@ class BigQueryDeleteDatasetOperator(GoogleCloudBaseOperator):
|
|
1907
1920
|
self,
|
1908
1921
|
*,
|
1909
1922
|
dataset_id: str,
|
1910
|
-
project_id: str
|
1923
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1911
1924
|
delete_contents: bool = False,
|
1912
1925
|
gcp_conn_id: str = "google_cloud_default",
|
1913
1926
|
impersonation_chain: str | Sequence[str] | None = None,
|
@@ -1987,7 +2000,7 @@ class BigQueryCreateEmptyDatasetOperator(GoogleCloudBaseOperator):
|
|
1987
2000
|
self,
|
1988
2001
|
*,
|
1989
2002
|
dataset_id: str | None = None,
|
1990
|
-
project_id: str
|
2003
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
1991
2004
|
dataset_reference: dict | None = None,
|
1992
2005
|
location: str | None = None,
|
1993
2006
|
gcp_conn_id: str = "google_cloud_default",
|
@@ -2091,7 +2104,7 @@ class BigQueryGetDatasetOperator(GoogleCloudBaseOperator):
|
|
2091
2104
|
self,
|
2092
2105
|
*,
|
2093
2106
|
dataset_id: str,
|
2094
|
-
project_id: str
|
2107
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2095
2108
|
gcp_conn_id: str = "google_cloud_default",
|
2096
2109
|
impersonation_chain: str | Sequence[str] | None = None,
|
2097
2110
|
**kwargs,
|
@@ -2154,7 +2167,7 @@ class BigQueryGetDatasetTablesOperator(GoogleCloudBaseOperator):
|
|
2154
2167
|
self,
|
2155
2168
|
*,
|
2156
2169
|
dataset_id: str,
|
2157
|
-
project_id: str
|
2170
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2158
2171
|
max_results: int | None = None,
|
2159
2172
|
gcp_conn_id: str = "google_cloud_default",
|
2160
2173
|
impersonation_chain: str | Sequence[str] | None = None,
|
@@ -2223,7 +2236,7 @@ class BigQueryPatchDatasetOperator(GoogleCloudBaseOperator):
|
|
2223
2236
|
*,
|
2224
2237
|
dataset_id: str,
|
2225
2238
|
dataset_resource: dict,
|
2226
|
-
project_id: str
|
2239
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2227
2240
|
gcp_conn_id: str = "google_cloud_default",
|
2228
2241
|
impersonation_chain: str | Sequence[str] | None = None,
|
2229
2242
|
**kwargs,
|
@@ -2296,7 +2309,7 @@ class BigQueryUpdateTableOperator(GoogleCloudBaseOperator):
|
|
2296
2309
|
fields: list[str] | None = None,
|
2297
2310
|
dataset_id: str | None = None,
|
2298
2311
|
table_id: str | None = None,
|
2299
|
-
project_id: str
|
2312
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2300
2313
|
gcp_conn_id: str = "google_cloud_default",
|
2301
2314
|
impersonation_chain: str | Sequence[str] | None = None,
|
2302
2315
|
**kwargs,
|
@@ -2380,7 +2393,7 @@ class BigQueryUpdateDatasetOperator(GoogleCloudBaseOperator):
|
|
2380
2393
|
dataset_resource: dict[str, Any],
|
2381
2394
|
fields: list[str] | None = None,
|
2382
2395
|
dataset_id: str | None = None,
|
2383
|
-
project_id: str
|
2396
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2384
2397
|
gcp_conn_id: str = "google_cloud_default",
|
2385
2398
|
impersonation_chain: str | Sequence[str] | None = None,
|
2386
2399
|
**kwargs,
|
@@ -2516,7 +2529,7 @@ class BigQueryUpsertTableOperator(GoogleCloudBaseOperator):
|
|
2516
2529
|
*,
|
2517
2530
|
dataset_id: str,
|
2518
2531
|
table_resource: dict,
|
2519
|
-
project_id: str
|
2532
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2520
2533
|
gcp_conn_id: str = "google_cloud_default",
|
2521
2534
|
location: str | None = None,
|
2522
2535
|
impersonation_chain: str | Sequence[str] | None = None,
|
@@ -2623,7 +2636,7 @@ class BigQueryUpdateTableSchemaOperator(GoogleCloudBaseOperator):
|
|
2623
2636
|
dataset_id: str,
|
2624
2637
|
table_id: str,
|
2625
2638
|
include_policy_tags: bool = False,
|
2626
|
-
project_id: str
|
2639
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2627
2640
|
gcp_conn_id: str = "google_cloud_default",
|
2628
2641
|
impersonation_chain: str | Sequence[str] | None = None,
|
2629
2642
|
**kwargs,
|
@@ -2732,7 +2745,7 @@ class BigQueryInsertJobOperator(GoogleCloudBaseOperator, _BigQueryOpenLineageMix
|
|
2732
2745
|
def __init__(
|
2733
2746
|
self,
|
2734
2747
|
configuration: dict[str, Any],
|
2735
|
-
project_id: str
|
2748
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
2736
2749
|
location: str | None = None,
|
2737
2750
|
job_id: str | None = None,
|
2738
2751
|
force_rerun: bool = True,
|
@@ -2903,6 +2916,7 @@ class BigQueryInsertJobOperator(GoogleCloudBaseOperator, _BigQueryOpenLineageMix
|
|
2903
2916
|
location=self.location or hook.location,
|
2904
2917
|
poll_interval=self.poll_interval,
|
2905
2918
|
impersonation_chain=self.impersonation_chain,
|
2919
|
+
cancel_on_kill=self.cancel_on_kill,
|
2906
2920
|
),
|
2907
2921
|
method_name="execute_complete",
|
2908
2922
|
)
|
@@ -37,6 +37,7 @@ from airflow.providers.google.cloud.hooks.bigquery_dts import BiqQueryDataTransf
|
|
37
37
|
from airflow.providers.google.cloud.links.bigquery_dts import BigQueryDataTransferConfigLink
|
38
38
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
39
39
|
from airflow.providers.google.cloud.triggers.bigquery_dts import BigQueryDataTransferRunTrigger
|
40
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
40
41
|
|
41
42
|
if TYPE_CHECKING:
|
42
43
|
from google.api_core.retry import Retry
|
@@ -94,7 +95,7 @@ class BigQueryCreateDataTransferOperator(GoogleCloudBaseOperator):
|
|
94
95
|
self,
|
95
96
|
*,
|
96
97
|
transfer_config: dict,
|
97
|
-
project_id: str
|
98
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
98
99
|
location: str | None = None,
|
99
100
|
authorization_code: str | None = None,
|
100
101
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -187,7 +188,7 @@ class BigQueryDeleteDataTransferConfigOperator(GoogleCloudBaseOperator):
|
|
187
188
|
self,
|
188
189
|
*,
|
189
190
|
transfer_config_id: str,
|
190
|
-
project_id: str
|
191
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
191
192
|
location: str | None = None,
|
192
193
|
retry: Retry | _MethodDefault = DEFAULT,
|
193
194
|
timeout: float | None = None,
|
@@ -273,7 +274,7 @@ class BigQueryDataTransferServiceStartTransferRunsOperator(GoogleCloudBaseOperat
|
|
273
274
|
self,
|
274
275
|
*,
|
275
276
|
transfer_config_id: str,
|
276
|
-
project_id: str
|
277
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
277
278
|
location: str | None = None,
|
278
279
|
requested_time_range: dict | None = None,
|
279
280
|
requested_run_time: dict | None = None,
|
@@ -31,6 +31,7 @@ from airflow.providers.google.cloud.links.bigtable import (
|
|
31
31
|
BigtableTablesLink,
|
32
32
|
)
|
33
33
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
34
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
34
35
|
|
35
36
|
if TYPE_CHECKING:
|
36
37
|
import enum
|
@@ -112,7 +113,7 @@ class BigtableCreateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
|
|
112
113
|
instance_id: str,
|
113
114
|
main_cluster_id: str,
|
114
115
|
main_cluster_zone: str,
|
115
|
-
project_id: str
|
116
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
116
117
|
replica_clusters: list[dict[str, str]] | None = None,
|
117
118
|
instance_display_name: str | None = None,
|
118
119
|
instance_type: enums.Instance.Type | None = None,
|
@@ -218,7 +219,7 @@ class BigtableUpdateInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
|
|
218
219
|
self,
|
219
220
|
*,
|
220
221
|
instance_id: str,
|
221
|
-
project_id: str
|
222
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
222
223
|
instance_display_name: str | None = None,
|
223
224
|
instance_type: enums.Instance.Type | enum.IntEnum | None = None,
|
224
225
|
instance_labels: dict | None = None,
|
@@ -298,7 +299,7 @@ class BigtableDeleteInstanceOperator(GoogleCloudBaseOperator, BigtableValidation
|
|
298
299
|
self,
|
299
300
|
*,
|
300
301
|
instance_id: str,
|
301
|
-
project_id: str
|
302
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
302
303
|
gcp_conn_id: str = "google_cloud_default",
|
303
304
|
impersonation_chain: str | Sequence[str] | None = None,
|
304
305
|
**kwargs,
|
@@ -374,7 +375,7 @@ class BigtableCreateTableOperator(GoogleCloudBaseOperator, BigtableValidationMix
|
|
374
375
|
*,
|
375
376
|
instance_id: str,
|
376
377
|
table_id: str,
|
377
|
-
project_id: str
|
378
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
378
379
|
initial_split_keys: list | None = None,
|
379
380
|
column_families: dict[str, GarbageCollectionRule] | None = None,
|
380
381
|
gcp_conn_id: str = "google_cloud_default",
|
@@ -478,7 +479,7 @@ class BigtableDeleteTableOperator(GoogleCloudBaseOperator, BigtableValidationMix
|
|
478
479
|
*,
|
479
480
|
instance_id: str,
|
480
481
|
table_id: str,
|
481
|
-
project_id: str
|
482
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
482
483
|
app_profile_id: str | None = None,
|
483
484
|
gcp_conn_id: str = "google_cloud_default",
|
484
485
|
impersonation_chain: str | Sequence[str] | None = None,
|
@@ -559,7 +560,7 @@ class BigtableUpdateClusterOperator(GoogleCloudBaseOperator, BigtableValidationM
|
|
559
560
|
instance_id: str,
|
560
561
|
cluster_id: str,
|
561
562
|
nodes: int,
|
562
|
-
project_id: str
|
563
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
563
564
|
gcp_conn_id: str = "google_cloud_default",
|
564
565
|
impersonation_chain: str | Sequence[str] | None = None,
|
565
566
|
**kwargs,
|
@@ -40,6 +40,7 @@ from airflow.providers.google.cloud.links.cloud_build import (
|
|
40
40
|
from airflow.providers.google.cloud.operators.cloud_base import GoogleCloudBaseOperator
|
41
41
|
from airflow.providers.google.cloud.triggers.cloud_build import CloudBuildCreateBuildTrigger
|
42
42
|
from airflow.providers.google.common.consts import GOOGLE_DEFAULT_DEFERRABLE_METHOD_NAME
|
43
|
+
from airflow.providers.google.common.hooks.base_google import PROVIDE_PROJECT_ID
|
43
44
|
from airflow.utils import yaml
|
44
45
|
from airflow.utils.helpers import exactly_one
|
45
46
|
|
@@ -87,7 +88,7 @@ class CloudBuildCancelBuildOperator(GoogleCloudBaseOperator):
|
|
87
88
|
self,
|
88
89
|
*,
|
89
90
|
id_: str,
|
90
|
-
project_id: str
|
91
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
91
92
|
retry: Retry | _MethodDefault = DEFAULT,
|
92
93
|
timeout: float | None = None,
|
93
94
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -171,7 +172,7 @@ class CloudBuildCreateBuildOperator(GoogleCloudBaseOperator):
|
|
171
172
|
self,
|
172
173
|
*,
|
173
174
|
build: dict | Build,
|
174
|
-
project_id: str
|
175
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
175
176
|
wait: bool = True,
|
176
177
|
retry: Retry | _MethodDefault = DEFAULT,
|
177
178
|
timeout: float | None = None,
|
@@ -316,7 +317,7 @@ class CloudBuildCreateBuildTriggerOperator(GoogleCloudBaseOperator):
|
|
316
317
|
self,
|
317
318
|
*,
|
318
319
|
trigger: dict | BuildTrigger,
|
319
|
-
project_id: str
|
320
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
320
321
|
retry: Retry | _MethodDefault = DEFAULT,
|
321
322
|
timeout: float | None = None,
|
322
323
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -399,7 +400,7 @@ class CloudBuildDeleteBuildTriggerOperator(GoogleCloudBaseOperator):
|
|
399
400
|
self,
|
400
401
|
*,
|
401
402
|
trigger_id: str,
|
402
|
-
project_id: str
|
403
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
403
404
|
retry: Retry | _MethodDefault = DEFAULT,
|
404
405
|
timeout: float | None = None,
|
405
406
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -473,7 +474,7 @@ class CloudBuildGetBuildOperator(GoogleCloudBaseOperator):
|
|
473
474
|
self,
|
474
475
|
*,
|
475
476
|
id_: str,
|
476
|
-
project_id: str
|
477
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
477
478
|
retry: Retry | _MethodDefault = DEFAULT,
|
478
479
|
timeout: float | None = None,
|
479
480
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -549,7 +550,7 @@ class CloudBuildGetBuildTriggerOperator(GoogleCloudBaseOperator):
|
|
549
550
|
self,
|
550
551
|
*,
|
551
552
|
trigger_id: str,
|
552
|
-
project_id: str
|
553
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
553
554
|
retry: Retry | _MethodDefault = DEFAULT,
|
554
555
|
timeout: float | None = None,
|
555
556
|
metadata: Sequence[tuple[str, str]] = (),
|
@@ -627,7 +628,7 @@ class CloudBuildListBuildTriggersOperator(GoogleCloudBaseOperator):
|
|
627
628
|
self,
|
628
629
|
*,
|
629
630
|
location: str = "global",
|
630
|
-
project_id: str
|
631
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
631
632
|
page_size: int | None = None,
|
632
633
|
page_token: str | None = None,
|
633
634
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -707,7 +708,7 @@ class CloudBuildListBuildsOperator(GoogleCloudBaseOperator):
|
|
707
708
|
self,
|
708
709
|
*,
|
709
710
|
location: str = "global",
|
710
|
-
project_id: str
|
711
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
711
712
|
page_size: int | None = None,
|
712
713
|
filter_: str | None = None,
|
713
714
|
retry: Retry | _MethodDefault = DEFAULT,
|
@@ -783,7 +784,7 @@ class CloudBuildRetryBuildOperator(GoogleCloudBaseOperator):
|
|
783
784
|
self,
|
784
785
|
*,
|
785
786
|
id_: str,
|
786
|
-
project_id: str
|
787
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
787
788
|
wait: bool = True,
|
788
789
|
retry: Retry | _MethodDefault = DEFAULT,
|
789
790
|
timeout: float | None = None,
|
@@ -868,7 +869,7 @@ class CloudBuildRunBuildTriggerOperator(GoogleCloudBaseOperator):
|
|
868
869
|
*,
|
869
870
|
trigger_id: str,
|
870
871
|
source: dict | RepoSource,
|
871
|
-
project_id: str
|
872
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
872
873
|
wait: bool = True,
|
873
874
|
retry: Retry | _MethodDefault = DEFAULT,
|
874
875
|
timeout: float | None = None,
|
@@ -953,7 +954,7 @@ class CloudBuildUpdateBuildTriggerOperator(GoogleCloudBaseOperator):
|
|
953
954
|
*,
|
954
955
|
trigger_id: str,
|
955
956
|
trigger: dict | BuildTrigger,
|
956
|
-
project_id: str
|
957
|
+
project_id: str = PROVIDE_PROJECT_ID,
|
957
958
|
retry: Retry | _MethodDefault = DEFAULT,
|
958
959
|
timeout: float | None = None,
|
959
960
|
metadata: Sequence[tuple[str, str]] = (),
|