apache-airflow-providers-amazon 9.7.0rc1__tar.gz → 9.7.0rc2__tar.gz

{apache_airflow_providers_amazon-9.7.0rc1 → apache_airflow_providers_amazon-9.7.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-amazon
-Version: 9.7.0rc1
+Version: 9.7.0rc2
 Summary: Provider package apache-airflow-providers-amazon for Apache Airflow
 Keywords: airflow-provider,amazon,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -22,7 +22,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: apache-airflow>=2.10.0rc1
 Requires-Dist: apache-airflow-providers-common-compat>=1.6.1rc1
-Requires-Dist: apache-airflow-providers-common-sql>=1.20.0rc1
+Requires-Dist: apache-airflow-providers-common-sql>=1.27.0rc1
 Requires-Dist: apache-airflow-providers-http
 Requires-Dist: boto3>=1.37.0
 Requires-Dist: botocore>=1.37.0
@@ -39,7 +39,6 @@ Requires-Dist: lxml<5.4.0,>=5.3.2
 Requires-Dist: sagemaker-studio>=1.0.9
 Requires-Dist: marshmallow>=3
 Requires-Dist: aiobotocore[boto3]>=2.21.1 ; extra == "aiobotocore"
-Requires-Dist: boto3<1.37.2,>=1.37.0 ; extra == "aiobotocore"
 Requires-Dist: apache-airflow-providers-apache-hive ; extra == "apache-hive"
 Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0rc1 ; extra == "cncf-kubernetes"
 Requires-Dist: apache-airflow-providers-common-messaging>=1.0.1rc1 ; extra == "common-messaging"
@@ -50,8 +49,7 @@ Requires-Dist: apache-airflow-providers-google ; extra == "google"
 Requires-Dist: apache-airflow-providers-imap ; extra == "imap"
 Requires-Dist: apache-airflow-providers-microsoft-azure ; extra == "microsoft-azure"
 Requires-Dist: apache-airflow-providers-mongo ; extra == "mongo"
-Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
-Requires-Dist: pandas>=2.1.2,<2.2 ; extra == "pandas"
+Requires-Dist: apache-airflow-providers-openlineage>=2.3.0rc1 ; extra == "openlineage"
 Requires-Dist: python3-saml>=1.16.0 ; extra == "python3-saml"
 Requires-Dist: s3fs>=2023.10.0 ; extra == "s3fs"
 Requires-Dist: apache-airflow-providers-salesforce ; extra == "salesforce"
@@ -76,7 +74,6 @@ Provides-Extra: imap
 Provides-Extra: microsoft-azure
 Provides-Extra: mongo
 Provides-Extra: openlineage
-Provides-Extra: pandas
 Provides-Extra: python3-saml
 Provides-Extra: s3fs
 Provides-Extra: salesforce
@@ -140,7 +137,7 @@ PIP package                                 Version required
 ==========================================  =====================
 ``apache-airflow``                          ``>=2.10.0``
 ``apache-airflow-providers-common-compat``  ``>=1.6.1``
-``apache-airflow-providers-common-sql``     ``>=1.20.0``
+``apache-airflow-providers-common-sql``     ``>=1.27.0``
 ``apache-airflow-providers-http``
 ``boto3``                                   ``>=1.37.0``
 ``botocore``                                ``>=1.37.0``

{apache_airflow_providers_amazon-9.7.0rc1 → apache_airflow_providers_amazon-9.7.0rc2}/README.rst

@@ -55,7 +55,7 @@ PIP package                                 Version required
 ==========================================  =====================
 ``apache-airflow``                          ``>=2.10.0``
 ``apache-airflow-providers-common-compat``  ``>=1.6.1``
-``apache-airflow-providers-common-sql``     ``>=1.20.0``
+``apache-airflow-providers-common-sql``     ``>=1.27.0``
 ``apache-airflow-providers-http``
 ``boto3``                                   ``>=1.37.0``
 ``botocore``                                ``>=1.37.0``

{apache_airflow_providers_amazon-9.7.0rc1 → apache_airflow_providers_amazon-9.7.0rc2}/docs/changelog.rst

@@ -53,6 +53,9 @@ Misc
 * ``Amazon EMR Sensors/Operators inherit AWS Base Classes (#49486)``
 * ``Introduce lower bind to lxml as 5.4.0 (#49612)``
 * ``Remove Marshmallow from Core (#49388)``
+* ``Migrate 'HiveToDynamoDBOperator' and 'SqlToS3Operator' to use 'get_df' (#50126)``
+* ``add root parent information to OpenLineage events (#49237)``
+* ``Remove limits for aiobotocore limiting boto3 (#50285)``
 
 .. Below changes are excluded from the changelog. Move them to
    appropriate section above if needed. Do not delete the lines(!):
@@ -73,6 +76,8 @@ Misc
    * ``capitalize the term airflow (#49450)``
    * ``Use Label class from task sdk in providers (#49398)``
    * ``Add Stop AutoML Job to the Sagemaker system test to clean up. (#49325)``
+   * ``Update description of provider.yaml dependencies (#50231)``
+   * ``Prepare ad hoc release for providers May 2025 (#50166)``
 
 9.6.1
 .....

{apache_airflow_providers_amazon-9.7.0rc1 → apache_airflow_providers_amazon-9.7.0rc2}/docs/index.rst

@@ -112,7 +112,7 @@ PIP package                                 Version required
 ==========================================  =====================
 ``apache-airflow``                          ``>=2.10.0``
 ``apache-airflow-providers-common-compat``  ``>=1.6.1``
-``apache-airflow-providers-common-sql``     ``>=1.20.0``
+``apache-airflow-providers-common-sql``     ``>=1.27.0``
 ``apache-airflow-providers-http``
 ``boto3``                                   ``>=1.37.0``
 ``botocore``                                ``>=1.37.0``

{apache_airflow_providers_amazon-9.7.0rc1 → apache_airflow_providers_amazon-9.7.0rc2}/provider.yaml

@@ -23,7 +23,10 @@ description: |
 
 state: ready
 source-date-epoch: 1744788746
-# note that those versions are maintained by release manager - do not update them manually
+# Note that those versions are maintained by release manager - do not update them manually
+# with the exception of case where other provider in sources has >= new provider version.
+# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
+# to be done in the same PR
 versions:
   - 9.7.0
   - 9.6.1

{apache_airflow_providers_amazon-9.7.0rc1 → apache_airflow_providers_amazon-9.7.0rc2}/pyproject.toml

@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-amazon"
-version = "9.7.0rc1"
+version = "9.7.0rc2"
 description = "Provider package apache-airflow-providers-amazon for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -59,7 +59,7 @@ requires-python = "~=3.9"
 dependencies = [
     "apache-airflow>=2.10.0rc1",
     "apache-airflow-providers-common-compat>=1.6.1rc1",
-    "apache-airflow-providers-common-sql>=1.20.0rc1",
+    "apache-airflow-providers-common-sql>=1.27.0rc1",
     "apache-airflow-providers-http",
     # We should update minimum version of boto3 and here regularly to avoid `pip` backtracking with the number
     # of candidates to consider. Make sure to configure boto3 version here as well as in all the tools below
@@ -88,21 +88,11 @@ dependencies = [
 # The optional dependencies should be modified in place in the generated file
 # Any change in the dependencies is preserved when the file is regenerated
 [project.optional-dependencies]
-"pandas" = [
-    # In pandas 2.2 minimal version of the sqlalchemy is 2.0
-    # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
-    # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
-    # In addition FAB also limit sqlalchemy to < 2.0
-    "pandas>=2.1.2,<2.2",
-]
 # There is conflict between boto3 and aiobotocore dependency botocore.
 # TODO: We can remove it once boto3 and aiobotocore both have compatible botocore version or
 # boto3 have native async support and we move away from aio aiobotocore
 "aiobotocore" = [
     "aiobotocore[boto3]>=2.21.1",
-    # boto3 here should be synchronized with latest aiobotocore version otherwise pip might get
-    # into a backtracking loop and fail to install the package
-    "boto3<1.37.2,>=1.37.0"
 ]
 "cncf.kubernetes" = [
     "apache-airflow-providers-cncf-kubernetes>=7.2.0rc1",
@@ -138,7 +128,7 @@ dependencies = [
     "apache-airflow-providers-mongo"
 ]
 "openlineage" = [
-    "apache-airflow-providers-openlineage"
+    "apache-airflow-providers-openlineage>=2.3.0rc1"
 ]
 "salesforce" = [
     "apache-airflow-providers-salesforce"
@@ -187,6 +177,7 @@ dev = [
     "openapi-spec-validator>=0.7.1",
     "opensearch-py>=2.2.0",
     "responses>=0.25.0",
+    "apache-airflow-providers-common-sql[pandas,polars]",
 ]
 
 # To build docs:

{apache_airflow_providers_amazon-9.7.0rc1 → apache_airflow_providers_amazon-9.7.0rc2}/src/airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py

@@ -93,7 +93,7 @@ class HiveToDynamoDBOperator(BaseOperator):
         self.log.info("Extracting data from Hive")
         self.log.info(self.sql)
 
-        data = hive.get_pandas_df(self.sql, schema=self.schema)
+        data = hive.get_df(self.sql, schema=self.schema, df_type="pandas")
         dynamodb = DynamoDBHook(
             aws_conn_id=self.aws_conn_id,
             table_name=self.table_name,

{apache_airflow_providers_amazon-9.7.0rc1 → apache_airflow_providers_amazon-9.7.0rc2}/src/airflow/providers/amazon/aws/transfers/sql_to_s3.py

@@ -187,7 +187,7 @@ class SqlToS3Operator(BaseOperator):
     def execute(self, context: Context) -> None:
         sql_hook = self._get_hook()
         s3_conn = S3Hook(aws_conn_id=self.aws_conn_id, verify=self.verify)
-        data_df = sql_hook.get_pandas_df(sql=self.query, parameters=self.parameters)
+        data_df = sql_hook.get_df(sql=self.query, parameters=self.parameters, df_type="pandas")
         self.log.info("Data from SQL obtained")
         self._fix_dtypes(data_df, self.file_format)
         file_options = FILE_OPTIONS_MAP[self.file_format]
@@ -233,8 +233,6 @@ class SqlToS3Operator(BaseOperator):
         self.log.debug("Get connection for %s", self.sql_conn_id)
         conn = BaseHook.get_connection(self.sql_conn_id)
         hook = conn.get_hook(hook_params=self.sql_hook_params)
-        if not callable(getattr(hook, "get_pandas_df", None)):
-            raise AirflowException(
-                "This hook is not supported. The hook class must have get_pandas_df method."
-            )
+        if not callable(getattr(hook, "get_df", None)):
+            raise AirflowException("This hook is not supported. The hook class must have get_df method.")
         return hook

{apache_airflow_providers_amazon-9.7.0rc1 → apache_airflow_providers_amazon-9.7.0rc2}/tests/unit/amazon/aws/transfers/test_hive_to_dynamodb.py

@@ -51,11 +51,11 @@ class TestHiveToDynamoDBOperator:
         assert hook.get_conn() is not None
 
     @mock.patch(
-        "airflow.providers.apache.hive.hooks.hive.HiveServer2Hook.get_pandas_df",
+        "airflow.providers.apache.hive.hooks.hive.HiveServer2Hook.get_df",
         return_value=pd.DataFrame(data=[("1", "sid")], columns=["id", "name"]),
     )
     @mock_aws
-    def test_get_records_with_schema(self, mock_get_pandas_df):
+    def test_get_records_with_schema(self, mock_get_df):
         # this table needs to be created in production
         self.hook.get_conn().create_table(
             TableName="test_airflow",
@@ -81,11 +81,11 @@ class TestHiveToDynamoDBOperator:
         assert table.item_count == 1
 
     @mock.patch(
-        "airflow.providers.apache.hive.hooks.hive.HiveServer2Hook.get_pandas_df",
+        "airflow.providers.apache.hive.hooks.hive.HiveServer2Hook.get_df",
         return_value=pd.DataFrame(data=[("1", "sid"), ("1", "gupta")], columns=["id", "name"]),
     )
     @mock_aws
-    def test_pre_process_records_with_schema(self, mock_get_pandas_df):
+    def test_pre_process_records_with_schema(self, mock_get_df):
         # this table needs to be created in production
         self.hook.get_conn().create_table(
             TableName="test_airflow",

{apache_airflow_providers_amazon-9.7.0rc1 → apache_airflow_providers_amazon-9.7.0rc2}/tests/unit/amazon/aws/transfers/test_sql_to_s3.py

@@ -38,8 +38,8 @@ class TestSqlToS3Operator:
 
         mock_dbapi_hook = mock.Mock()
         test_df = pd.DataFrame({"a": "1", "b": "2"}, index=[0, 1])
-        get_pandas_df_mock = mock_dbapi_hook.return_value.get_pandas_df
-        get_pandas_df_mock.return_value = test_df
+        get_df_mock = mock_dbapi_hook.return_value.get_df
+        get_df_mock.return_value = test_df
         with NamedTemporaryFile() as f:
             temp_mock.return_value.__enter__.return_value.name = f.name
 
@@ -58,7 +58,7 @@ class TestSqlToS3Operator:
             op.execute(None)
             mock_s3_hook.assert_called_once_with(aws_conn_id="aws_conn_id", verify=None)
 
-            get_pandas_df_mock.assert_called_once_with(sql=query, parameters=None)
+            get_df_mock.assert_called_once_with(sql=query, parameters=None, df_type="pandas")
 
             temp_mock.assert_called_once_with(mode="r+", suffix=".csv")
             mock_s3_hook.return_value.load_file.assert_called_once_with(
@@ -78,8 +78,8 @@ class TestSqlToS3Operator:
         mock_dbapi_hook = mock.Mock()
 
         test_df = pd.DataFrame({"a": "1", "b": "2"}, index=[0, 1])
-        get_pandas_df_mock = mock_dbapi_hook.return_value.get_pandas_df
-        get_pandas_df_mock.return_value = test_df
+        get_df_mock = mock_dbapi_hook.return_value.get_df
+        get_df_mock.return_value = test_df
         with NamedTemporaryFile() as f:
             temp_mock.return_value.__enter__.return_value.name = f.name
 
@@ -98,7 +98,7 @@ class TestSqlToS3Operator:
             op.execute(None)
             mock_s3_hook.assert_called_once_with(aws_conn_id="aws_conn_id", verify=None)
 
-            get_pandas_df_mock.assert_called_once_with(sql=query, parameters=None)
+            get_df_mock.assert_called_once_with(sql=query, parameters=None, df_type="pandas")
 
             temp_mock.assert_called_once_with(mode="rb+", suffix=".parquet")
             mock_s3_hook.return_value.load_file.assert_called_once_with(
@@ -114,8 +114,8 @@ class TestSqlToS3Operator:
 
         mock_dbapi_hook = mock.Mock()
         test_df = pd.DataFrame({"a": "1", "b": "2"}, index=[0, 1])
-        get_pandas_df_mock = mock_dbapi_hook.return_value.get_pandas_df
-        get_pandas_df_mock.return_value = test_df
+        get_df_mock = mock_dbapi_hook.return_value.get_df
+        get_df_mock.return_value = test_df
         with NamedTemporaryFile() as f:
             temp_mock.return_value.__enter__.return_value.name = f.name
 
@@ -135,7 +135,7 @@ class TestSqlToS3Operator:
             op.execute(None)
             mock_s3_hook.assert_called_once_with(aws_conn_id="aws_conn_id", verify=None)
 
-            get_pandas_df_mock.assert_called_once_with(sql=query, parameters=None)
+            get_df_mock.assert_called_once_with(sql=query, parameters=None, df_type="pandas")
 
             temp_mock.assert_called_once_with(mode="r+", suffix=".json")
             mock_s3_hook.return_value.load_file.assert_called_once_with(