dynamicio 7.2.0__tar.gz → 7.2.1__tar.gz

{dynamicio-7.2.0 → dynamicio-7.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dynamicio
-Version: 7.2.0
+Version: 7.2.1
 Summary: Panda's wrapper for IO operations
 License: Apache-2.0
 Keywords: data,pandas,io
@@ -18,7 +18,7 @@ Requires-Dist: awscli (>=1.22.24)
 Requires-Dist: boto3 (>=1.20.24)
 Requires-Dist: confluent-kafka (>=2.4.0,<2.5.0)
 Requires-Dist: fastparquet (>=0.8.0)
-Requires-Dist: fsspec (==2022.3.0)
+Requires-Dist: fsspec (==2022.11.0)
 Requires-Dist: libcst (==1.5.1)
 Requires-Dist: logzero (>=1.7.0)
 Requires-Dist: magic-logger (>=1.0.2)
@@ -27,6 +27,7 @@ Requires-Dist: numpy (>=1.24.0,<2.0.0)
 Requires-Dist: pandas (>=1.5.1)
 Requires-Dist: psycopg2-binary (>=2.9.3,<2.10.0)
 Requires-Dist: pyarrow (>=10.0.1)
+Requires-Dist: pyathena
 Requires-Dist: pydantic (>=1.9.2,<3)
 Requires-Dist: python-json-logger (>=2.0.1,<2.1.0)
 Requires-Dist: s3fs (==0.4.2)
@@ -34,8 +35,8 @@ Requires-Dist: simplejson (>=3.17.2,<3.18.0)
 Requires-Dist: tables (>=3.10.1)
 Description-Content-Type: text/markdown
 
-[![CircleCI](https://dl.circleci.com/status-badge/img/gh/VorTECHsa/dynamicio/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/VorTECHsa/dynamicio/tree/master)
-[![Coverage Status](https://github.com/VorTECHsa/dynamicio/blob/master/docs/coverage_report/coverage-badge.svg?raw=True)]()
+[![Build Status](https://github.com/VorTECHsa/dynamicio/actions/workflows/build_and_publish.yml/badge.svg?branch=master)](https://github.com/VorTECHsa/dynamicio/actions/workflows/build_and_publish.yml)
+[![codecov](https://codecov.io/gh/VorTECHsa/dynamicio/graph/badge.svg?token=McZFwD2QPh)](https://codecov.io/gh/VorTECHsa/dynamicio)
 [<img src="https://img.shields.io/badge/slack-@vortexa/dynamicio_public-purple.svg?logo=slack">](https://join.slack.com/share/enQtMzg2Nzk3ODY3MzEzNi0yNTU1ZmIyN2JkMGFhZjhhZWVjNzA2OWUzNWIyMjMyYmYzZmE4MzBjYWQ3YjdhNjU1MGU2NjFkNzMyZDllMzE2?raw=True)
 
 <img src="https://github.com/VorTECHsa/dynamicio/blob/master/docs/images/logo-transparent.png?raw=True" width="500"> <img src="https://github.com/VorTECHsa/dynamicio/blob/master/docs/images/wrapped-panda.png?raw=True" width="100">

{dynamicio-7.2.0 → dynamicio-7.2.1}/README.md

@@ -1,5 +1,5 @@
-[![CircleCI](https://dl.circleci.com/status-badge/img/gh/VorTECHsa/dynamicio/tree/master.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/VorTECHsa/dynamicio/tree/master)
-[![Coverage Status](https://github.com/VorTECHsa/dynamicio/blob/master/docs/coverage_report/coverage-badge.svg?raw=True)]()
+[![Build Status](https://github.com/VorTECHsa/dynamicio/actions/workflows/build_and_publish.yml/badge.svg?branch=master)](https://github.com/VorTECHsa/dynamicio/actions/workflows/build_and_publish.yml)
+[![codecov](https://codecov.io/gh/VorTECHsa/dynamicio/graph/badge.svg?token=McZFwD2QPh)](https://codecov.io/gh/VorTECHsa/dynamicio)
 [<img src="https://img.shields.io/badge/slack-@vortexa/dynamicio_public-purple.svg?logo=slack">](https://join.slack.com/share/enQtMzg2Nzk3ODY3MzEzNi0yNTU1ZmIyN2JkMGFhZjhhZWVjNzA2OWUzNWIyMjMyYmYzZmE4MzBjYWQ3YjdhNjU1MGU2NjFkNzMyZDllMzE2?raw=True)
 
 <img src="https://github.com/VorTECHsa/dynamicio/blob/master/docs/images/logo-transparent.png?raw=True" width="500"> <img src="https://github.com/VorTECHsa/dynamicio/blob/master/docs/images/wrapped-panda.png?raw=True" width="100">

{dynamicio-7.2.0 → dynamicio-7.2.1}/dynamicio/__init__.py

@@ -1,13 +1,14 @@
 """A package for wrapping your I/O operations."""
+# pylint: disable=abstract-method
 import os
 from contextlib import suppress
 
 from magic_logger import logger
 
 try:
-    from importlib.metadata import version, PackageNotFoundError
+    from importlib.metadata import PackageNotFoundError, version
 except ImportError:
-    from importlib_metadata import version, PackageNotFoundError  # type: ignore
+    from importlib_metadata import PackageNotFoundError, version  # type: ignore
 
 with suppress(Exception):
     try:
@@ -15,13 +16,14 @@ with suppress(Exception):
     except PackageNotFoundError:
         __version__ = "0.0.0"
 
+# Application Imports
 from dynamicio.core import DynamicDataIO
-from dynamicio.mixins import WithKafka, WithLocal, WithLocalBatch, WithPostgres, WithS3File, WithS3PathPrefix
+from dynamicio.mixins import WithAthena, WithKafka, WithLocal, WithLocalBatch, WithPostgres, WithS3File, WithS3PathPrefix
 
 os.environ["LC_CTYPE"] = "en_US.UTF"  # Set your locale to a unicode-compatible one
 
 
-class UnifiedIO(WithS3File, WithS3PathPrefix, WithLocalBatch, WithLocal, WithKafka, WithPostgres, DynamicDataIO):  # type: ignore
+class UnifiedIO(WithS3File, WithS3PathPrefix, WithLocalBatch, WithLocal, WithKafka, WithAthena, WithPostgres, DynamicDataIO):  # type: ignore
     """A unified io composed of dynamicio.mixins."""
 
 

{dynamicio-7.2.0 → dynamicio-7.2.1}/dynamicio/config/pydantic/__init__.py

@@ -9,5 +9,6 @@ from dynamicio.config.pydantic.io_resources import (
     PostgresDataEnvironment,
     S3DataEnvironment,
     S3PathPrefixEnvironment,
+    AthenaDataEnvironment
 )
 from dynamicio.config.pydantic.table_schema import DataframeSchema, SchemaColumn

{dynamicio-7.2.0 → dynamicio-7.2.1}/dynamicio/config/pydantic/io_resources.py

@@ -9,6 +9,7 @@ from typing import Mapping, Optional, Union
 import pydantic
 from pydantic import BaseModel
 
+# Application Imports
 import dynamicio.config.pydantic.table_schema as table_spec
 
 
@@ -65,9 +66,10 @@ class IOBinding(BaseModel):
             DataBackendType.s3_path_prefix: S3PathPrefixEnvironment,
             DataBackendType.kafka: KafkaDataEnvironment,
             DataBackendType.postgres: PostgresDataEnvironment,
+            DataBackendType.athena: AthenaDataEnvironment,
         }
         out_dict = {}
-        for (env_name, env_data) in info.items():
+        for env_name, env_data in info.items():
             base_obj: IOEnvironment = IOEnvironment(**env_data)
             override_cls = config_cls_overrides.get(base_obj.data_backend_type)
             if override_cls:
@@ -82,7 +84,7 @@ class IOBinding(BaseModel):
         if not isinstance(values, Mapping):
             raise ValueError(f"IOBinding must be a dict at the top level. (got {values!r} instead)")
         remapped_value = {"environments": {}}
-        for (key, value) in values.items():
+        for key, value in values.items():
             if key in ("__binding_name__", "schema"):
                 # Passthrough params
                 remapped_value[key] = value
@@ -270,4 +272,20 @@ class PostgresDataEnvironment(IOEnvironment):
     postgres: PostgresDataSubSection
 
 
+class AthenaDataSubSection(BaseModel):
+    """AWS Athena configuration section."""
+
+    s3_staging_dir: str
+    region_name: str
+
+    # Optional fields, one must be provided via YAML or mixin options
+    query: Optional[str] = None
+
+
+class AthenaDataEnvironment(IOEnvironment):
+    """Parent section for Athena source config."""
+
+    athena: AthenaDataSubSection
+
+
 IOBinding.update_forward_refs()

dynamicio-7.2.1/dynamicio/mixins/__init__.py

@@ -0,0 +1,8 @@
+"""Default dynamicio mixins module."""
+
+# Application Imports
+from dynamicio.mixins.with_athena import WithAthena
+from dynamicio.mixins.with_kafka import WithKafka
+from dynamicio.mixins.with_local import WithLocal, WithLocalBatch
+from dynamicio.mixins.with_postgres import WithPostgres
+from dynamicio.mixins.with_s3 import WithS3File, WithS3PathPrefix

dynamicio-7.2.1/dynamicio/mixins/with_athena.py

@@ -0,0 +1,55 @@
+# pylint: disable=no-member, protected-access, too-few-public-methods
+
+"""This module provides mixins that support AWS Athena I/O."""
+import inspect
+from typing import Any, MutableMapping
+
+import pandas as pd
+from magic_logger import logger
+from pyathena import connect
+from pyathena.connection import Connection
+from pyathena.pandas.cursor import PandasCursor
+from pyathena.pandas.result_set import AthenaPandasResultSet
+
+# Application Imports
+from dynamicio.config.pydantic import AthenaDataEnvironment
+from dynamicio.mixins.utils import allow_options
+
+allowed_athena_options = set(inspect.signature(AthenaPandasResultSet.__init__).parameters.keys()) - {"self"}
+
+
+class WithAthena:
+    """Handles I/O operations for AWS Athena."""
+
+    sources_config: AthenaDataEnvironment
+    options: MutableMapping[str, Any]
+
+    def _read_from_athena(self) -> pd.DataFrame:
+        """Reads data from AWS Athena.
+
+        Expected config:
+            - query
+            - s3_staging_dir
+            - region_name
+        """
+        cfg = self.sources_config.athena
+        query = self.options.pop("query", None)
+
+        assert query, "A 'query' must be provided for Athena read"
+
+        conn = connect(s3_staging_dir=cfg.s3_staging_dir, region_name=cfg.region_name, cursor_class=PandasCursor)
+        return self._run_query(conn, query, **self.options)
+
+    @allow_options(allowed_athena_options)
+    def _run_query(self, conn: Connection, query: str, **options: Any) -> pd.DataFrame:
+        logger.info(f"[athena] Executing query: {query}")
+        cursor = conn.cursor()
+        cursor.execute(query)
+
+        rows = cursor.fetchall(**options)
+        columns = [col[0] for col in cursor.description]
+        return pd.DataFrame(rows, columns=columns)
+
+    def _write_to_athena(self, df: pd.DataFrame):
+        """Athena does not support direct writing. Raise NotImplementedError."""
+        raise NotImplementedError("Athena does not support direct writes via pandas. Consider writing to S3 or using Glue instead.")

{dynamicio-7.2.0 → dynamicio-7.2.1}/dynamicio/mixins/with_kafka.py

@@ -8,6 +8,7 @@ import simplejson
 from confluent_kafka import Producer
 from magic_logger import logger
 
+# Application Imports
 from dynamicio.config.pydantic import DataframeSchema, KafkaDataEnvironment
 from dynamicio.mixins import utils
 
@@ -242,6 +243,10 @@ class WithKafka:
         "transactional.id",
     }
 
+    def _read_from_kafka(self, df: pd.DataFrame):
+        """Kafka is not ideal for reading files as part of batch ETL jobs. Raise NotImplementedError."""
+        raise NotImplementedError("Dynamicio does not support reading from Kafka Topics.")
+
     def _write_to_kafka(self, df: pd.DataFrame) -> None:
         """Given a dataframe where each row is a message to be sent to a Kafka Topic, iterate through all rows and send them to a Kafka topic.
 

{dynamicio-7.2.0 → dynamicio-7.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dynamicio"
-version = "7.2.0"  # dummy, will be overridden by git tag
+version = "7.2.1"  # dummy, will be overridden by git tag
 description = "Panda's wrapper for IO operations"
 authors = [
     "Christos Hadjinikolis <christos.hadjinikolis@gmail.com>",
@@ -20,8 +20,9 @@ classifiers = [
 python = ">=3.10,<3.13"
 awscli = ">=1.22.24"
 boto3 = ">=1.20.24"
+pyathena = "*"
 fastparquet = ">=0.8.0"
-fsspec = "==2022.3.0"
+fsspec = "==2022.11.0"
 confluent-kafka = "~2.4.0"
 logzero = ">=1.7.0"
 magic-logger = ">=1.0.2"
@@ -40,7 +41,7 @@ tables = ">=3.10.1"
 libcst = "==1.5.1"
 
 [tool.poetry.group.dev.dependencies]
-black = "==22.10.0"
+black = "==24.3.0"
 coverage-badge = "==1.1.0"
 flake8-import-order = "*"
 flake8-print = "*"
@@ -65,7 +66,7 @@ yamllint = "==1.28.0"
 pdoc3 = "==0.10.0"
 
 [tool.poetry.group.build.dependencies]
-setuptools = "==65.5.1"
+setuptools = "==70.0.0"
 twine = "==4.0.1"
 wheel = "==0.38.4"
 

dynamicio-7.2.0/dynamicio/mixins/__init__.py

@@ -1,16 +0,0 @@
-"""Default dynamicio mixins module"""
-
-from dynamicio.mixins.with_kafka import (
-    WithKafka,
-)
-from dynamicio.mixins.with_local import (
-    WithLocal,
-    WithLocalBatch,
-)
-from dynamicio.mixins.with_postgres import (
-    WithPostgres,
-)
-from dynamicio.mixins.with_s3 import (
-    WithS3File,
-    WithS3PathPrefix,
-)