apache-airflow-providers-teradata 1.0.10__py3-none-any.whl → 2.0.0__py3-none-any.whl

airflow/providers/teradata/transfers/teradata_to_teradata.py

@@ -0,0 +1,101 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from functools import cached_property
+from typing import TYPE_CHECKING, Sequence
+
+from airflow.models import BaseOperator
+from airflow.providers.teradata.hooks.teradata import TeradataHook
+
+if TYPE_CHECKING:
+    from airflow.utils.context import Context
+
+
+class TeradataToTeradataOperator(BaseOperator):
+    """
+    Moves data from Teradata source database to Teradata destination database.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:TeradataToTeradataOperator`
+
+    :param dest_teradata_conn_id: destination Teradata connection.
+    :param destination_table: destination table to insert rows.
+    :param source_teradata_conn_id: :ref:`Source Teradata connection <howto/connection:Teradata>`.
+    :param sql: SQL query to execute against the source Teradata database
+    :param sql_params: Parameters to use in sql query.
+    :param rows_chunk: number of rows per chunk to commit.
+    """
+
+    template_fields: Sequence[str] = (
+        "sql",
+        "sql_params",
+    )
+    template_ext: Sequence[str] = (".sql",)
+    template_fields_renderers = {"sql": "sql", "sql_params": "py"}
+    ui_color = "#e07c24"
+
+    def __init__(
+        self,
+        *,
+        dest_teradata_conn_id: str,
+        destination_table: str,
+        source_teradata_conn_id: str,
+        sql: str,
+        sql_params: dict | None = None,
+        rows_chunk: int = 5000,
+        **kwargs,
+    ) -> None:
+        super().__init__(**kwargs)
+        if sql_params is None:
+            sql_params = {}
+        self.dest_teradata_conn_id = dest_teradata_conn_id
+        self.destination_table = destination_table
+        self.source_teradata_conn_id = source_teradata_conn_id
+        self.sql = sql
+        self.sql_params = sql_params
+        self.rows_chunk = rows_chunk
+
+    @cached_property
+    def src_hook(self) -> TeradataHook:
+        return TeradataHook(teradata_conn_id=self.source_teradata_conn_id)
+
+    @cached_property
+    def dest_hook(self) -> TeradataHook:
+        return TeradataHook(teradata_conn_id=self.dest_teradata_conn_id)
+
+    def execute(self, context: Context) -> None:
+        src_hook = self.src_hook
+        dest_hook = self.dest_hook
+        with src_hook.get_conn() as src_conn:
+            cursor = src_conn.cursor()
+            cursor.execute(self.sql, self.sql_params)
+            target_fields = [field[0] for field in cursor.description]
+            rows_total = 0
+            if len(target_fields) != 0:
+                for rows in iter(lambda: cursor.fetchmany(self.rows_chunk), []):
+                    dest_hook.bulk_insert_rows(
+                        self.destination_table,
+                        rows,
+                        target_fields=target_fields,
+                        commit_every=self.rows_chunk,
+                    )
+                    rows_total += len(rows)
+            self.log.info("Finished data transfer. Total number of rows transferred - %s", rows_total)
+            cursor.close()

apache_airflow_providers_teradata-2.0.0.dist-info/METADATA

@@ -0,0 +1,137 @@
+Metadata-Version: 2.1
+Name: apache-airflow-providers-teradata
+Version: 2.0.0
+Summary: Provider package apache-airflow-providers-teradata for Apache Airflow
+Keywords: airflow-provider,teradata,airflow,integration
+Author-email: Apache Software Foundation <dev@airflow.apache.org>
+Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
+Requires-Python: ~=3.8
+Description-Content-Type: text/x-rst
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
+Classifier: Environment :: Web Environment
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: System Administrators
+Classifier: Framework :: Apache Airflow
+Classifier: Framework :: Apache Airflow :: Provider
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: System :: Monitoring
+Requires-Dist: apache-airflow-providers-common-sql>=1.3.1
+Requires-Dist: apache-airflow>=2.6.0
+Requires-Dist: teradatasql>=17.20.0.28
+Requires-Dist: teradatasqlalchemy>=17.20.0.0
+Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
+Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-teradata/2.0.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-teradata/2.0.0
+Project-URL: Slack Chat, https://s.apache.org/airflow-slack
+Project-URL: Source Code, https://github.com/apache/airflow
+Project-URL: Twitter, https://twitter.com/ApacheAirflow
+Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
+Provides-Extra: common.sql
+
+
+.. Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License.
+
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
+    OVERWRITTEN WHEN PREPARING PACKAGES.
+
+ .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+    `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
+
+
+Package ``apache-airflow-providers-teradata``
+
+Release: ``2.0.0``
+
+
+`Teradata <https://www.teradata.com/>`__
+
+
+Provider package
+----------------
+
+This is a provider package for ``teradata`` provider. All classes for this provider package
+are in ``airflow.providers.teradata`` python package.
+
+You can find package information and changelog for the provider
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-teradata/2.0.0/>`_.
+
+Installation
+------------
+
+You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below
+for the minimum Airflow version supported) via
+``pip install apache-airflow-providers-teradata``
+
+The package supports the following python versions: 3.8,3.9,3.10,3.11
+
+Requirements
+------------
+
+=======================================  ==================
+PIP package                              Version required
+=======================================  ==================
+``apache-airflow``                       ``>=2.6.0``
+``apache-airflow-providers-common-sql``  ``>=1.3.1``
+``teradatasqlalchemy``                   ``>=17.20.0.0``
+``teradatasql``                          ``>=17.20.0.28``
+=======================================  ==================
+
+Cross provider package dependencies
+-----------------------------------
+
+Those are dependencies that might be needed in order to use all the features of the package.
+You need to install the specified provider packages in order to use them.
+
+You can install such cross-provider dependencies when installing from PyPI. For example:
+
+.. code-block:: bash
+
+    pip install apache-airflow-providers-teradata[common.sql]
+
+
+============================================================================================================  ==============
+Dependent package                                                                                             Extra
+============================================================================================================  ==============
+`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_  ``common.sql``
+============================================================================================================  ==============
+
+The changelog for the provider package can be found in the
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-teradata/2.0.0/changelog.html>`_.

apache_airflow_providers_teradata-2.0.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+airflow/providers/teradata/LICENSE,sha256=ywUBpKZc7Jb96rVt5I3IDbg7dIJAbUSHkuoDcF3jbH4,13569
+airflow/providers/teradata/__init__.py,sha256=RV5q0cL-SrYdknuU-pF-iEKaPMaVq_IeH3_tcjc6o60,1583
+airflow/providers/teradata/get_provider_info.py,sha256=qOUPkhJIwdFfTsm926Ub8yacOBDIJ-lWWZ6NG2H6vBM,2872
+airflow/providers/teradata/hooks/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
+airflow/providers/teradata/hooks/teradata.py,sha256=LTV18WhtmXbc9pgKlhuLdICbSArVrTBVaO3O8uW3GlQ,8094
+airflow/providers/teradata/operators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
+airflow/providers/teradata/operators/teradata.py,sha256=UqEH6bn0e9NJgjBVVXgM-lQj-50MigBeJyWlbJ-1WG8,2373
+airflow/providers/teradata/transfers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
+airflow/providers/teradata/transfers/teradata_to_teradata.py,sha256=WketoakWYtYdMn7W3hR55otQJqQSKB_dKHTtYLc4mco,3834
+apache_airflow_providers_teradata-2.0.0.dist-info/entry_points.txt,sha256=JbigXoUoKVSNWG-_-029FCCuehMOmAvuSnNGZ9Bz1Kc,104
+apache_airflow_providers_teradata-2.0.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+apache_airflow_providers_teradata-2.0.0.dist-info/METADATA,sha256=5c6TrSER-wMo6RYKdjEtmRPHssCcFACf8Qq0O0sInCU,6191
+apache_airflow_providers_teradata-2.0.0.dist-info/RECORD,,

{apache_airflow_providers_teradata-1.0.10.dist-info → apache_airflow_providers_teradata-2.0.0.dist-info}/WHEEL

@@ -1,5 +1,4 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.36.2)
+Generator: flit 3.9.0
 Root-Is-Purelib: true
 Tag: py3-none-any
-

apache_airflow_providers_teradata-2.0.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[apache_airflow_provider]
+provider_info=airflow.providers.teradata.get_provider_info:get_provider_info
+

airflow/providers/teradata/example_dags/example_execute_bteq.py

@@ -1,51 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-This is an example dag that runs a BTEQ sentence to teradata
-"""
-from datetime import timedelta
-
-from airflow import DAG
-from airflow.providers.teradata.operators.bteq import BteqOperator
-from airflow.utils.dates import days_ago
-
-DEFAULT_ARGS = {
-    'owner': 'airflow',
-    'depends_on_past': False,
-    'email': ['airflow@example.com'],
-    'email_on_failure': False,
-    'email_on_retry': False,
-}
-
-with DAG(
-    dag_id='example_execute_bteq',
-    default_args=DEFAULT_ARGS,
-    dagrun_timeout=timedelta(hours=2),
-    schedule_interval=None,
-    start_date=days_ago(2),
-    tags=['example'],
-) as dag:
-    task_execute_bteq = BteqOperator(
-        task_id='task_execute_bteq',
-        bteq='''
-                 SELECT CURRENT_DATE;
-                .IF ERRORCODE <> 0 THEN .QUIT 0301;
-                .QUIT 0;
-             ''',
-
-    )

airflow/providers/teradata/example_dags/example_export_csv.py

@@ -1,48 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-This is an example dag that export select queries from Teradata Database to CSV file (without header)
-"""
-from datetime import timedelta
-
-from airflow import DAG
-from airflow.providers.teradata.operators.fastexport import FastExportOperator
-from airflow.utils.dates import days_ago
-
-DEFAULT_ARGS = {
-    'owner': 'airflow',
-    'depends_on_past': False,
-    'email': ['airflow@example.com'],
-    'email_on_failure': False,
-    'email_on_retry': False,
-}
-
-with DAG(
-    dag_id='example_export_csv_to_teradata',
-    default_args=DEFAULT_ARGS,
-    dagrun_timeout=timedelta(hours=2),
-    schedule_interval=None,
-    start_date=days_ago(2),
-    tags=['example'],
-) as dag:
-    export_csv_to_teradata = FastExportOperator(
-        task_id='export_csv_to_teradata',
-        ttu_conn_id='ttu_default',
-        sql_select_stmt='''SEL UserName, CreateTimeStamp, SpoolSpace FROM dbc.Users''',
-        output_file='dummy_import.csv',
-    )

airflow/providers/teradata/example_dags/example_load_csv.py

@@ -1,69 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-This is an example dag that loads a CSV file without header to Teradata Database
-"""
-from datetime import timedelta
-
-from airflow import DAG
-from airflow.providers.teradata.operators.fastload import FastLoadOperator
-from airflow.utils.dates import days_ago
-
-DEFAULT_ARGS = {
-    'owner': 'airflow',
-    'depends_on_past': False,
-    'email': ['airflow@example.com'],
-    'email_on_failure': False,
-    'email_on_retry': False,
-}
-
-with DAG(
-    dag_id='example_load_csv_to_teradata',
-    default_args=DEFAULT_ARGS,
-    dagrun_timeout=timedelta(hours=2),
-    schedule_interval=None,
-    start_date=days_ago(2),
-    tags=['example'],
-) as dag:
-    load_csv_to_teradata = FastLoadOperator(
-        task_id='load_csv_to_teradata',
-        ttu_conn_id='ttu_default',
-        input_file='/files/dags/dummy_load.csv',
-        ## Pre-operator BTEQ for creating the table before data load ocurrs
-        ## You can use macros too!
-        preoperator_bteq="""
-                    DROP TABLE SQLJ.dummy_load_UV;
-                    DROP TABLE SQLJ.dummy_load_Log;
-                    DROP TABLE SQLJ.dummy_load_ET;
-                    DROP TABLE SQLJ.dummy_load;
-                    CREATE TABLE SQLJ.dummy_load (
-                        id int,
-                        first_name varchar(100),
-                        mail varchar(320),
-                        phone_number varchar(20)
-                    )
-                    PRIMARY INDEX (id);
-                    .IF ERRORCODE <> 0 THEN .QUIT 0301;
-                    .QUIT 0;
-        """,
-        target_table='SQLJ.dummy_load',
-        ## Working db is for saving internal tables of fastload process (error tables)
-        working_database='SQLJ',
-        delimiter='|',
-        encoding='UTF8'
-    )