apache-airflow-providers-apache-hive 6.3.0__tar.gz → 6.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/PKG-INFO +6 -6
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/README.rst +3 -3
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/__init__.py +1 -1
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/get_provider_info.py +2 -1
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/hooks/hive.py +16 -13
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/macros/hive.py +3 -3
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/operators/hive.py +5 -1
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/pyproject.toml +3 -3
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/LICENSE +0 -0
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/hooks/__init__.py +0 -0
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/macros/__init__.py +0 -0
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/operators/__init__.py +0 -0
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/operators/hive_stats.py +0 -0
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/plugins/__init__.py +0 -0
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/plugins/hive.py +0 -0
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/sensors/__init__.py +0 -0
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/sensors/hive_partition.py +0 -0
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/sensors/metastore_partition.py +0 -0
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/sensors/named_hive_partition.py +0 -0
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/transfers/__init__.py +0 -0
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/transfers/hive_to_mysql.py +0 -0
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/transfers/hive_to_samba.py +0 -0
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/transfers/mssql_to_hive.py +0 -0
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/transfers/mysql_to_hive.py +0 -0
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/transfers/s3_to_hive.py +0 -0
- {apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/airflow/providers/apache/hive/transfers/vertica_to_hive.py +0 -0
{apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: apache-airflow-providers-apache-hive
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.4.0
|
|
4
4
|
Summary: Provider package apache-airflow-providers-apache-hive for Apache Airflow
|
|
5
5
|
Keywords: airflow-provider,apache.hive,airflow,integration
|
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
@@ -34,8 +34,8 @@ Requires-Dist: apache-airflow-providers-presto ; extra == "presto"
|
|
|
34
34
|
Requires-Dist: apache-airflow-providers-samba ; extra == "samba"
|
|
35
35
|
Requires-Dist: apache-airflow-providers-vertica ; extra == "vertica"
|
|
36
36
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
37
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/6.
|
|
38
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/6.
|
|
37
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/6.4.0/changelog.html
|
|
38
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/6.4.0
|
|
39
39
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
40
40
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
41
41
|
Project-URL: Twitter, https://twitter.com/ApacheAirflow
|
|
@@ -92,7 +92,7 @@ Provides-Extra: vertica
|
|
|
92
92
|
|
|
93
93
|
Package ``apache-airflow-providers-apache-hive``
|
|
94
94
|
|
|
95
|
-
Release: ``6.
|
|
95
|
+
Release: ``6.4.0``
|
|
96
96
|
|
|
97
97
|
|
|
98
98
|
`Apache Hive <https://hive.apache.org/>`__
|
|
@@ -105,7 +105,7 @@ This is a provider package for ``apache.hive`` provider. All classes for this pr
|
|
|
105
105
|
are in ``airflow.providers.apache.hive`` python package.
|
|
106
106
|
|
|
107
107
|
You can find package information and changelog for the provider
|
|
108
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/6.
|
|
108
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/6.4.0/>`_.
|
|
109
109
|
|
|
110
110
|
Installation
|
|
111
111
|
------------
|
|
@@ -156,4 +156,4 @@ Dependent package
|
|
|
156
156
|
====================================================================================================================== ===================
|
|
157
157
|
|
|
158
158
|
The changelog for the provider package can be found in the
|
|
159
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/6.
|
|
159
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/6.4.0/changelog.html>`_.
|
{apache_airflow_providers_apache_hive-6.3.0 → apache_airflow_providers_apache_hive-6.4.0}/README.rst
RENAMED
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
|
|
43
43
|
Package ``apache-airflow-providers-apache-hive``
|
|
44
44
|
|
|
45
|
-
Release: ``6.
|
|
45
|
+
Release: ``6.4.0``
|
|
46
46
|
|
|
47
47
|
|
|
48
48
|
`Apache Hive <https://hive.apache.org/>`__
|
|
@@ -55,7 +55,7 @@ This is a provider package for ``apache.hive`` provider. All classes for this pr
|
|
|
55
55
|
are in ``airflow.providers.apache.hive`` python package.
|
|
56
56
|
|
|
57
57
|
You can find package information and changelog for the provider
|
|
58
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/6.
|
|
58
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/6.4.0/>`_.
|
|
59
59
|
|
|
60
60
|
Installation
|
|
61
61
|
------------
|
|
@@ -106,4 +106,4 @@ Dependent package
|
|
|
106
106
|
====================================================================================================================== ===================
|
|
107
107
|
|
|
108
108
|
The changelog for the provider package can be found in the
|
|
109
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/6.
|
|
109
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/6.4.0/changelog.html>`_.
|
|
@@ -28,8 +28,9 @@ def get_provider_info():
|
|
|
28
28
|
"name": "Apache Hive",
|
|
29
29
|
"description": "`Apache Hive <https://hive.apache.org/>`__\n",
|
|
30
30
|
"suspended": False,
|
|
31
|
-
"source-date-epoch":
|
|
31
|
+
"source-date-epoch": 1703288109,
|
|
32
32
|
"versions": [
|
|
33
|
+
"6.4.0",
|
|
33
34
|
"6.3.0",
|
|
34
35
|
"6.2.0",
|
|
35
36
|
"6.1.6",
|
|
@@ -80,6 +80,7 @@ class HiveCliHook(BaseHook):
|
|
|
80
80
|
This can make monitoring easier.
|
|
81
81
|
:param hive_cli_params: Space separated list of hive command parameters to add to the
|
|
82
82
|
hive command.
|
|
83
|
+
:param proxy_user: Run HQL code as this user.
|
|
83
84
|
"""
|
|
84
85
|
|
|
85
86
|
conn_name_attr = "hive_cli_conn_id"
|
|
@@ -96,6 +97,7 @@ class HiveCliHook(BaseHook):
|
|
|
96
97
|
mapred_job_name: str | None = None,
|
|
97
98
|
hive_cli_params: str = "",
|
|
98
99
|
auth: str | None = None,
|
|
100
|
+
proxy_user: str | None = None,
|
|
99
101
|
) -> None:
|
|
100
102
|
super().__init__()
|
|
101
103
|
conn = self.get_connection(hive_cli_conn_id)
|
|
@@ -105,7 +107,6 @@ class HiveCliHook(BaseHook):
|
|
|
105
107
|
self.conn = conn
|
|
106
108
|
self.run_as = run_as
|
|
107
109
|
self.sub_process: Any = None
|
|
108
|
-
|
|
109
110
|
if mapred_queue_priority:
|
|
110
111
|
mapred_queue_priority = mapred_queue_priority.upper()
|
|
111
112
|
if mapred_queue_priority not in HIVE_QUEUE_PRIORITIES:
|
|
@@ -116,6 +117,7 @@ class HiveCliHook(BaseHook):
|
|
|
116
117
|
self.mapred_queue = mapred_queue or conf.get("hive", "default_hive_mapred_queue")
|
|
117
118
|
self.mapred_queue_priority = mapred_queue_priority
|
|
118
119
|
self.mapred_job_name = mapred_job_name
|
|
120
|
+
self.proxy_user = proxy_user
|
|
119
121
|
|
|
120
122
|
def _get_proxy_user(self) -> str:
|
|
121
123
|
"""Set the proper proxy_user value in case the user overwrite the default."""
|
|
@@ -126,6 +128,8 @@ class HiveCliHook(BaseHook):
|
|
|
126
128
|
return f"hive.server2.proxy.user={conn.login}"
|
|
127
129
|
if proxy_user_value == "owner" and self.run_as:
|
|
128
130
|
return f"hive.server2.proxy.user={self.run_as}"
|
|
131
|
+
if proxy_user_value == "as_param" and self.proxy_user:
|
|
132
|
+
return f"hive.server2.proxy.user={self.proxy_user}"
|
|
129
133
|
if proxy_user_value != "": # There is a custom proxy user
|
|
130
134
|
return f"hive.server2.proxy.user={proxy_user_value}"
|
|
131
135
|
return proxy_user_value # The default proxy user (undefined)
|
|
@@ -582,12 +586,11 @@ class HiveMetastoreHook(BaseHook):
|
|
|
582
586
|
|
|
583
587
|
:param schema: Name of hive schema (database) @table belongs to
|
|
584
588
|
:param table: Name of hive table @partition belongs to
|
|
585
|
-
:param partition: Expression that matches the partitions to check for
|
|
586
|
-
(eg `a = 'b' AND c = 'd'`)
|
|
589
|
+
:param partition: Expression that matches the partitions to check for (e.g. `a = 'b' AND c = 'd'`)
|
|
587
590
|
|
|
588
591
|
>>> hh = HiveMetastoreHook()
|
|
589
|
-
>>> t =
|
|
590
|
-
>>> hh.check_for_partition(
|
|
592
|
+
>>> t = "static_babynames_partitioned"
|
|
593
|
+
>>> hh.check_for_partition("airflow", t, "ds='2015-01-01'")
|
|
591
594
|
True
|
|
592
595
|
"""
|
|
593
596
|
with self.metastore as client:
|
|
@@ -606,10 +609,10 @@ class HiveMetastoreHook(BaseHook):
|
|
|
606
609
|
:param partition_name: Name of the partitions to check for (eg `a=b/c=d`)
|
|
607
610
|
|
|
608
611
|
>>> hh = HiveMetastoreHook()
|
|
609
|
-
>>> t =
|
|
610
|
-
>>> hh.check_for_named_partition(
|
|
612
|
+
>>> t = "static_babynames_partitioned"
|
|
613
|
+
>>> hh.check_for_named_partition("airflow", t, "ds=2015-01-01")
|
|
611
614
|
True
|
|
612
|
-
>>> hh.check_for_named_partition(
|
|
615
|
+
>>> hh.check_for_named_partition("airflow", t, "ds=xxx")
|
|
613
616
|
False
|
|
614
617
|
"""
|
|
615
618
|
with self.metastore as client:
|
|
@@ -619,7 +622,7 @@ class HiveMetastoreHook(BaseHook):
|
|
|
619
622
|
"""Get a metastore table object.
|
|
620
623
|
|
|
621
624
|
>>> hh = HiveMetastoreHook()
|
|
622
|
-
>>> t = hh.get_table(db=
|
|
625
|
+
>>> t = hh.get_table(db="airflow", table_name="static_babynames")
|
|
623
626
|
>>> t.tableName
|
|
624
627
|
'static_babynames'
|
|
625
628
|
>>> [col.name for col in t.sd.cols]
|
|
@@ -649,8 +652,8 @@ class HiveMetastoreHook(BaseHook):
|
|
|
649
652
|
For subpartitioned table, the number might easily exceed this.
|
|
650
653
|
|
|
651
654
|
>>> hh = HiveMetastoreHook()
|
|
652
|
-
>>> t =
|
|
653
|
-
>>> parts = hh.get_partitions(schema=
|
|
655
|
+
>>> t = "static_babynames_partitioned"
|
|
656
|
+
>>> parts = hh.get_partitions(schema="airflow", table_name=t)
|
|
654
657
|
>>> len(parts)
|
|
655
658
|
1
|
|
656
659
|
>>> parts
|
|
@@ -765,9 +768,9 @@ class HiveMetastoreHook(BaseHook):
|
|
|
765
768
|
Check if table exists.
|
|
766
769
|
|
|
767
770
|
>>> hh = HiveMetastoreHook()
|
|
768
|
-
>>> hh.table_exists(db=
|
|
771
|
+
>>> hh.table_exists(db="airflow", table_name="static_babynames")
|
|
769
772
|
True
|
|
770
|
-
>>> hh.table_exists(db=
|
|
773
|
+
>>> hh.table_exists(db="airflow", table_name="does_not_exist")
|
|
771
774
|
False
|
|
772
775
|
"""
|
|
773
776
|
try:
|
|
@@ -39,7 +39,7 @@ def max_partition(
|
|
|
39
39
|
:param field: the field to get the max value from. If there's only
|
|
40
40
|
one partition field, this will be inferred
|
|
41
41
|
|
|
42
|
-
>>> max_partition(
|
|
42
|
+
>>> max_partition("airflow.static_babynames_partitioned")
|
|
43
43
|
'2015-01-01'
|
|
44
44
|
"""
|
|
45
45
|
from airflow.providers.apache.hive.hooks.hive import HiveMetastoreHook
|
|
@@ -94,8 +94,8 @@ def closest_ds_partition(
|
|
|
94
94
|
:param metastore_conn_id: which metastore connection to use
|
|
95
95
|
:returns: The closest date
|
|
96
96
|
|
|
97
|
-
>>> tbl =
|
|
98
|
-
>>> closest_ds_partition(tbl,
|
|
97
|
+
>>> tbl = "airflow.static_babynames_partitioned"
|
|
98
|
+
>>> closest_ds_partition(tbl, "2015-01-02")
|
|
99
99
|
'2015-01-01'
|
|
100
100
|
"""
|
|
101
101
|
from airflow.providers.apache.hive.hooks.hive import HiveMetastoreHook
|
|
@@ -62,6 +62,7 @@ class HiveOperator(BaseOperator):
|
|
|
62
62
|
This can make monitoring easier.
|
|
63
63
|
:param hive_cli_params: parameters passed to hive CLO
|
|
64
64
|
:param auth: optional authentication option passed for the Hive connection
|
|
65
|
+
:param proxy_user: Run HQL code as this user.
|
|
65
66
|
"""
|
|
66
67
|
|
|
67
68
|
template_fields: Sequence[str] = (
|
|
@@ -72,6 +73,7 @@ class HiveOperator(BaseOperator):
|
|
|
72
73
|
"hiveconfs",
|
|
73
74
|
"mapred_job_name",
|
|
74
75
|
"mapred_queue_priority",
|
|
76
|
+
"proxy_user",
|
|
75
77
|
)
|
|
76
78
|
template_ext: Sequence[str] = (
|
|
77
79
|
".hql",
|
|
@@ -95,6 +97,7 @@ class HiveOperator(BaseOperator):
|
|
|
95
97
|
mapred_job_name: str | None = None,
|
|
96
98
|
hive_cli_params: str = "",
|
|
97
99
|
auth: str | None = None,
|
|
100
|
+
proxy_user: str | None = None,
|
|
98
101
|
**kwargs: Any,
|
|
99
102
|
) -> None:
|
|
100
103
|
super().__init__(**kwargs)
|
|
@@ -112,7 +115,7 @@ class HiveOperator(BaseOperator):
|
|
|
112
115
|
self.mapred_job_name = mapred_job_name
|
|
113
116
|
self.hive_cli_params = hive_cli_params
|
|
114
117
|
self.auth = auth
|
|
115
|
-
|
|
118
|
+
self.proxy_user = proxy_user
|
|
116
119
|
job_name_template = conf.get_mandatory_value(
|
|
117
120
|
"hive",
|
|
118
121
|
"mapred_job_name_template",
|
|
@@ -131,6 +134,7 @@ class HiveOperator(BaseOperator):
|
|
|
131
134
|
mapred_job_name=self.mapred_job_name,
|
|
132
135
|
hive_cli_params=self.hive_cli_params,
|
|
133
136
|
auth=self.auth,
|
|
137
|
+
proxy_user=self.proxy_user,
|
|
134
138
|
)
|
|
135
139
|
|
|
136
140
|
@deprecated(reason="use `hook` property instead.", category=AirflowProviderDeprecationWarning)
|
|
@@ -28,7 +28,7 @@ build-backend = "flit_core.buildapi"
|
|
|
28
28
|
|
|
29
29
|
[project]
|
|
30
30
|
name = "apache-airflow-providers-apache-hive"
|
|
31
|
-
version = "6.
|
|
31
|
+
version = "6.4.0"
|
|
32
32
|
description = "Provider package apache-airflow-providers-apache-hive for Apache Airflow"
|
|
33
33
|
readme = "README.rst"
|
|
34
34
|
authors = [
|
|
@@ -64,8 +64,8 @@ dependencies = [
|
|
|
64
64
|
]
|
|
65
65
|
|
|
66
66
|
[project.urls]
|
|
67
|
-
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/6.
|
|
68
|
-
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/6.
|
|
67
|
+
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/6.4.0"
|
|
68
|
+
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/6.4.0/changelog.html"
|
|
69
69
|
"Bug Tracker" = "https://github.com/apache/airflow/issues"
|
|
70
70
|
"Source Code" = "https://github.com/apache/airflow"
|
|
71
71
|
"Slack Chat" = "https://s.apache.org/airflow-slack"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|