dagster-teradata 0.0.1__tar.gz

dagster_teradata-0.0.1/PKG-INFO

@@ -0,0 +1,148 @@
+Metadata-Version: 2.2
+Name: dagster-teradata
+Version: 0.0.1
+Summary: <insert description here>
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: dagster>=1.8.0
+Requires-Dist: teradatasql
+Provides-Extra: azure
+Requires-Dist: dagster-azure; extra == "azure"
+Provides-Extra: aws
+Requires-Dist: boto3; extra == "aws"
+Requires-Dist: dagster-aws; extra == "aws"
+
+# dagster-teradata
+
+A dagster module that provides integration with [Teradata Vantage](https://www.teradata.com/).
+
+## Installation
+The `dagster_teradata` module is available as a PyPI package - install with your preferred python 
+environment manager.
+
+```
+source .venv/bin/activate
+pip install dagster-teradata
+```
+
+## Example Usage
+
+This offers seamless integration with Teradata Vantage, facilitating efficient workflows for data processing, management, 
+and transformation. This module supports a range of scenarios, such as executing queries, managing tables, 
+and integrating with cloud storage solutions like AWS S3 and Azure Data Lake Storage (ADLS). Additionally, 
+it enables compute cluster management for Teradata Vantage Cloud Lake.
+
+```python
+import os
+import pytest
+from dagster import job, op, EnvVar
+from dagster_teradata import TeradataResource
+
+td_resource = TeradataResource(
+    host=EnvVar("TERADATA_HOST"),
+    user=EnvVar("TERADATA_USER"),
+    password=EnvVar("TERADATA_PASSWORD"),
+    database=EnvVar("TERADATA_DATABASE"),
+)
+
+def test_execute_query(tmp_path):
+    @op(required_resource_keys={"teradata"})
+    def example_test_execute_query(context):
+        result = context.resources.teradata.execute_queries(
+            ["select order_id from orders_24", "select order_id from orders_25"], True
+        )
+        context.log.info(result)
+
+    @job(resource_defs={"teradata": td_resource})
+    def example_job():
+        example_test_execute_query()
+
+    example_job.execute_in_process(resources={"teradata": td_resource})
+```
+```python
+import os
+import pytest
+from dagster import job, op, EnvVar
+from dagster_teradata import TeradataResource
+
+td_resource = TeradataResource(
+    host=EnvVar("TERADATA_HOST"),
+    user=EnvVar("TERADATA_USER"),
+    password=EnvVar("TERADATA_PASSWORD"),
+    database=EnvVar("TERADATA_DATABASE"),
+)
+
+def test_drop_table(tmp_path):
+    @op(required_resource_keys={"teradata"})
+    def example_test_drop_table(context):
+        result = context.resources.teradata.drop_table(["process_tmp1", "process_tmp2"])
+        context.log.info(result)
+
+    @job(resource_defs={"teradata": td_resource})
+    def example_job():
+        example_test_drop_table()
+
+    example_job.execute_in_process(resources={"teradata": td_resource})
+```
+
+Here is another example of compute cluster management in Teradata VantageCloud Lake:
+
+```python
+import os
+
+import pytest
+from dagster import job, op, EnvVar
+from dagster_teradata import teradata_resource
+
+def test_create_teradata_compute_cluster(tmp_path):
+    @op(required_resource_keys={"teradata"})
+    def example_create_teradata_compute_cluster(context):
+        """Args for create_teradata_compute_cluster():
+        compute_profile_name: Name of the Compute Profile to manage.
+        compute_group_name: Name of compute group to which compute profile belongs.
+        query_strategy: Query strategy to use. Refers to the approach or method used by the
+                Teradata Optimizer to execute SQL queries efficiently within a Teradata computer cluster.
+                Valid query_strategy value is either 'STANDARD' or 'ANALYTIC'. Default at database level is STANDARD
+        compute_map: ComputeMapName of the compute map. The compute_map in a compute cluster profile refers
+                to the mapping of compute resources to a specific node or set of nodes within the cluster.
+        compute_attribute: Optional attributes of compute profile. Example compute attribute
+                MIN_COMPUTE_COUNT(1) MAX_COMPUTE_COUNT(5) INITIALLY_SUSPENDED('FALSE')
+                   compute_attribute (str, optional): Additional attributes for compute profile. Defaults to None.
+        """
+        context.resources.teradata.create_teradata_compute_cluster(
+            "ShippingCG01",
+            "Shipping",
+            "STANDARD",
+            "TD_COMPUTE_MEDIUM",
+            "MIN_COMPUTE_COUNT(1) MAX_COMPUTE_COUNT(1) INITIALLY_SUSPENDED('FALSE')",
+        )
+
+    @job(resource_defs={"teradata": teradata_resource})
+    def example_job():
+        example_create_teradata_compute_cluster()
+
+    example_job.execute_in_process(
+        run_config={
+            "resources": {
+                "teradata": {
+                    "config": {
+                        "host": EnvVar("TERADATA_HOST"),
+                        "user": EnvVar("TERADATA_USER"),
+                        "password": EnvVar("TERADATA_PASSWORD"),
+                        "database": EnvVar("TERADATA_DATABASE"),
+                    }
+                }
+            }
+        }
+    )
+```
+
+## Development
+
+The `Makefile` provides the tools required to test and lint your local installation.
+
+```sh
+make test
+make ruff
+make check
+```

dagster_teradata-0.0.1/README.md

@@ -0,0 +1,134 @@
+# dagster-teradata
+
+A dagster module that provides integration with [Teradata Vantage](https://www.teradata.com/).
+
+## Installation
+The `dagster_teradata` module is available as a PyPI package - install with your preferred python 
+environment manager.
+
+```
+source .venv/bin/activate
+pip install dagster-teradata
+```
+
+## Example Usage
+
+This offers seamless integration with Teradata Vantage, facilitating efficient workflows for data processing, management, 
+and transformation. This module supports a range of scenarios, such as executing queries, managing tables, 
+and integrating with cloud storage solutions like AWS S3 and Azure Data Lake Storage (ADLS). Additionally, 
+it enables compute cluster management for Teradata Vantage Cloud Lake.
+
+```python
+import os
+import pytest
+from dagster import job, op, EnvVar
+from dagster_teradata import TeradataResource
+
+td_resource = TeradataResource(
+    host=EnvVar("TERADATA_HOST"),
+    user=EnvVar("TERADATA_USER"),
+    password=EnvVar("TERADATA_PASSWORD"),
+    database=EnvVar("TERADATA_DATABASE"),
+)
+
+def test_execute_query(tmp_path):
+    @op(required_resource_keys={"teradata"})
+    def example_test_execute_query(context):
+        result = context.resources.teradata.execute_queries(
+            ["select order_id from orders_24", "select order_id from orders_25"], True
+        )
+        context.log.info(result)
+
+    @job(resource_defs={"teradata": td_resource})
+    def example_job():
+        example_test_execute_query()
+
+    example_job.execute_in_process(resources={"teradata": td_resource})
+```
+```python
+import os
+import pytest
+from dagster import job, op, EnvVar
+from dagster_teradata import TeradataResource
+
+td_resource = TeradataResource(
+    host=EnvVar("TERADATA_HOST"),
+    user=EnvVar("TERADATA_USER"),
+    password=EnvVar("TERADATA_PASSWORD"),
+    database=EnvVar("TERADATA_DATABASE"),
+)
+
+def test_drop_table(tmp_path):
+    @op(required_resource_keys={"teradata"})
+    def example_test_drop_table(context):
+        result = context.resources.teradata.drop_table(["process_tmp1", "process_tmp2"])
+        context.log.info(result)
+
+    @job(resource_defs={"teradata": td_resource})
+    def example_job():
+        example_test_drop_table()
+
+    example_job.execute_in_process(resources={"teradata": td_resource})
+```
+
+Here is another example of compute cluster management in Teradata VantageCloud Lake:
+
+```python
+import os
+
+import pytest
+from dagster import job, op, EnvVar
+from dagster_teradata import teradata_resource
+
+def test_create_teradata_compute_cluster(tmp_path):
+    @op(required_resource_keys={"teradata"})
+    def example_create_teradata_compute_cluster(context):
+        """Args for create_teradata_compute_cluster():
+        compute_profile_name: Name of the Compute Profile to manage.
+        compute_group_name: Name of compute group to which compute profile belongs.
+        query_strategy: Query strategy to use. Refers to the approach or method used by the
+                Teradata Optimizer to execute SQL queries efficiently within a Teradata computer cluster.
+                Valid query_strategy value is either 'STANDARD' or 'ANALYTIC'. Default at database level is STANDARD
+        compute_map: ComputeMapName of the compute map. The compute_map in a compute cluster profile refers
+                to the mapping of compute resources to a specific node or set of nodes within the cluster.
+        compute_attribute: Optional attributes of compute profile. Example compute attribute
+                MIN_COMPUTE_COUNT(1) MAX_COMPUTE_COUNT(5) INITIALLY_SUSPENDED('FALSE')
+                   compute_attribute (str, optional): Additional attributes for compute profile. Defaults to None.
+        """
+        context.resources.teradata.create_teradata_compute_cluster(
+            "ShippingCG01",
+            "Shipping",
+            "STANDARD",
+            "TD_COMPUTE_MEDIUM",
+            "MIN_COMPUTE_COUNT(1) MAX_COMPUTE_COUNT(1) INITIALLY_SUSPENDED('FALSE')",
+        )
+
+    @job(resource_defs={"teradata": teradata_resource})
+    def example_job():
+        example_create_teradata_compute_cluster()
+
+    example_job.execute_in_process(
+        run_config={
+            "resources": {
+                "teradata": {
+                    "config": {
+                        "host": EnvVar("TERADATA_HOST"),
+                        "user": EnvVar("TERADATA_USER"),
+                        "password": EnvVar("TERADATA_PASSWORD"),
+                        "database": EnvVar("TERADATA_DATABASE"),
+                    }
+                }
+            }
+        }
+    )
+```
+
+## Development
+
+The `Makefile` provides the tools required to test and lint your local installation.
+
+```sh
+make test
+make ruff
+make check
+```

dagster_teradata-0.0.1/dagster_teradata/__init__.py

@@ -0,0 +1,6 @@
+from dagster_teradata.resources import (
+    TeradataDagsterConnection as TeradataDagsterConnection,
+    TeradataResource as TeradataResource,
+    fetch_last_updated_timestamps as fetch_last_updated_timestamps,
+    teradata_resource as teradata_resource,
+)

dagster_teradata-0.0.1/dagster_teradata/constants.py

@@ -0,0 +1,23 @@
+"""Define constants for dagster-teradata."""
+
+CC_OPR_SUCCESS_STATUS_MSG = "Compute Cluster %s  %s operation completed successfully."
+CC_OPR_FAILURE_STATUS_MSG = "Compute Cluster %s  %s operation has failed."
+CC_OPR_INITIALIZING_STATUS_MSG = (
+    "The environment is currently initializing. Please wait."
+)
+CC_OPR_EMPTY_PROFILE_ERROR_MSG = (
+    "Please provide a valid name for the compute cluster profile."
+)
+CC_GRP_PRP_NON_EXISTS_MSG = (
+    "The specified Compute cluster is not present or The user doesn't have permission to "
+    "access compute cluster."
+)
+CC_GRP_PRP_UN_AUTHORIZED_MSG = "The %s operation is not authorized for the user."
+CC_GRP_LAKE_SUPPORT_ONLY_MSG = "Compute Groups is supported only on Vantage Cloud Lake."
+CC_OPR_TIMEOUT_ERROR = "There is an issue with the %s operation. Kindly consult the administrator for assistance."
+CC_GRP_PRP_EXISTS_MSG = "The specified Compute cluster is already exists."
+CC_OPR_EMPTY_COPY_PROFILE_ERROR_MSG = (
+    "Please provide a valid name for the source and target compute profile."
+)
+CC_OPR_TIME_OUT = 1200
+CC_POLL_INTERVAL = 60