etlplus 0.14.3__py3-none-any.whl → 0.16.0__py3-none-any.whl

etlplus/connector/file.py

@@ -0,0 +1,120 @@
+"""
+:mod:`etlplus.connector.file` module.
+
+File connector configuration dataclass.
+
+Notes
+-----
+- TypedDicts in this module are intentionally ``total=False`` and are not
+    enforced at runtime.
+- :meth:`*.from_obj` constructors accept :class:`Mapping[str, Any]` and perform
+    tolerant parsing and light casting. This keeps the runtime permissive while
+    improving autocomplete and static analysis for contributors.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from dataclasses import field
+from typing import Any
+from typing import Self
+from typing import TypedDict
+from typing import overload
+
+from ..types import StrAnyMap
+from ..utils import coerce_dict
+from .core import ConnectorBase
+from .enums import DataConnectorType
+from .types import ConnectorType
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'ConnectorFile',
+    'ConnectorFileConfigMap',
+]
+
+
+# SECTION: TYPED DICTS ====================================================== #
+
+
+class ConnectorFileConfigMap(TypedDict, total=False):
+    """
+    Shape accepted by :meth:`ConnectorFile.from_obj` (all keys optional).
+
+    See Also
+    --------
+    - :meth:`etlplus.connector.file.ConnectorFile.from_obj`
+    """
+
+    name: str
+    type: ConnectorType
+    format: str
+    path: str
+    options: StrAnyMap
+
+
+# SECTION: DATA CLASSES ===================================================== #
+
+
+@dataclass(kw_only=True, slots=True)
+class ConnectorFile(ConnectorBase):
+    """
+    Configuration for a file-based data connector.
+
+    Attributes
+    ----------
+    type : ConnectorType
+        Connector kind, always ``'file'``.
+    format : str | None
+        File format (e.g., ``'json'``, ``'csv'``).
+    path : str | None
+        File path or URI.
+    options : dict[str, Any]
+        Reader/writer format options.
+    """
+
+    # -- Attributes -- #
+
+    type: ConnectorType = DataConnectorType.FILE
+    format: str | None = None
+    path: str | None = None
+    options: dict[str, Any] = field(default_factory=dict)
+
+    # -- Class Methods -- #
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: ConnectorFileConfigMap) -> Self: ...
+
+    @classmethod
+    @overload
+    def from_obj(cls, obj: StrAnyMap) -> Self: ...
+
+    @classmethod
+    def from_obj(
+        cls,
+        obj: StrAnyMap,
+    ) -> Self:
+        """
+        Parse a mapping into a ``ConnectorFile`` instance.
+
+        Parameters
+        ----------
+        obj : StrAnyMap
+            Mapping with at least ``name``.
+
+        Returns
+        -------
+        Self
+            Parsed connector instance.
+        """
+        name = cls._require_name(obj, kind='File')
+
+        return cls(
+            name=name,
+            format=obj.get('format'),
+            path=obj.get('path'),
+            options=coerce_dict(obj.get('options')),
+        )

etlplus/connector/types.py

@@ -0,0 +1,40 @@
+"""
+:mod:`etlplus.connector.types` module.
+
+Connector type aliases for :mod:`etlplus.connector`.
+
+Examples
+--------
+>>> from etlplus.connector import Connector
+>>> src: Connector = {
+>>>     "type": "file",
+>>>     "path": "/data/input.csv",
+>>> }
+>>> tgt: Connector = {
+>>>     "type": "database",
+>>>     "connection_string": "postgresql://user:pass@localhost/db",
+>>> }
+>>> from etlplus.api import RetryPolicy
+>>> rp: RetryPolicy = {"max_attempts": 3, "backoff": 0.5}
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from .enums import DataConnectorType
+
+# SECTION: EXPORTS  ========================================================= #
+
+
+__all__ = [
+    # Type Aliases
+    'ConnectorType',
+]
+
+
+# SECTION: TYPE ALIASES ===================================================== #
+
+
+# Literal type for supported connector kinds (strings or enum members)
+type ConnectorType = DataConnectorType | Literal['api', 'database', 'file']

etlplus/connector/utils.py

@@ -0,0 +1,122 @@
+"""
+:mod:`etlplus.connector.utils` module.
+
+Shared connector parsing helpers.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from .api import ConnectorApi
+from .connector import Connector
+from .database import ConnectorDb
+from .enums import DataConnectorType
+from .file import ConnectorFile
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Functions
+    'parse_connector',
+]
+
+
+# SECTION: INTERNAL FUNCTIONS =============================================== #
+
+
+def _coerce_connector_type(
+    obj: Mapping[str, Any],
+) -> DataConnectorType:
+    """
+    Normalize and validate the connector ``type`` field.
+
+    Parameters
+    ----------
+    obj : Mapping[str, Any]
+        Mapping with a ``type`` entry.
+
+    Returns
+    -------
+    DataConnectorType
+        Normalized connector type enum.
+
+    Raises
+    ------
+    TypeError
+        If ``type`` is missing or unsupported.
+    """
+    if 'type' not in obj:
+        raise TypeError('Connector requires a "type"')
+    try:
+        return DataConnectorType.coerce(obj.get('type'))
+    except ValueError as exc:
+        allowed = ', '.join(DataConnectorType.choices())
+        raise TypeError(
+            f'Unsupported connector type: {obj.get("type")!r}. '
+            f'Expected one of {allowed}.',
+        ) from exc
+
+
+def _load_connector(
+    kind: DataConnectorType,
+) -> type[Connector]:
+    """
+    Resolve the connector class for the requested kind.
+
+    Parameters
+    ----------
+    kind : DataConnectorType
+        Connector kind enum.
+
+    Returns
+    -------
+    type[Connector]
+        Connector class corresponding to *kind*.
+    """
+    match kind:
+        case DataConnectorType.API:
+            return ConnectorApi
+        case DataConnectorType.DATABASE:
+            return ConnectorDb
+        case DataConnectorType.FILE:
+            return ConnectorFile
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def parse_connector(
+    obj: Mapping[str, Any],
+) -> Connector:
+    """
+    Dispatch to a concrete connector constructor based on ``type``.
+
+    Parameters
+    ----------
+    obj : Mapping[str, Any]
+        Mapping with at least ``name`` and ``type``.
+
+    Returns
+    -------
+    Connector
+        Concrete connector instance.
+
+    Raises
+    ------
+    TypeError
+        If the mapping is invalid or the connector type is unsupported.
+
+    Notes
+    -----
+    Delegates to the tolerant ``from_obj`` constructors for each connector
+    kind. Connector types are normalized via
+    :class:`etlplus.connector.enums.DataConnectorType`, so common aliases
+    (e.g., ``'db'`` or ``'http'``) are accepted.
+    """
+    if not isinstance(obj, Mapping):
+        raise TypeError('Connector configuration must be a mapping.')
+    connector_cls = _load_connector(_coerce_connector_type(obj))
+    return connector_cls.from_obj(obj)

etlplus/database/README.md

@@ -1,4 +1,4 @@
-# etlplus.database subpackage
+# `etlplus.database` Subpackage
 
 Documentation for the `etlplus.database` subpackage: database engine, schema, and ORM helpers.
 
@@ -9,7 +9,7 @@ Documentation for the `etlplus.database` subpackage: database engine, schema, an
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.database subpackage](#etlplusdatabase-subpackage)
+- [`etlplus.database` Subpackage](#etlplusdatabase-subpackage)
   - [Database Engine and Connections](#database-engine-and-connections)
   - [Schema and DDL Helpers](#schema-and-ddl-helpers)
   - [ORM Utilities](#orm-utilities)

etlplus/database/ddl.py

@@ -233,7 +233,7 @@ def render_table_sql(
     template : TemplateKey | None, optional
         Template key to use (default: 'ddl').
     template_path : str | None, optional
-        Path to a custom template file (overrides ``template``).
+        Path to a custom template file (overrides *template*).
 
     Returns
     -------
@@ -264,7 +264,7 @@ def render_tables(
     template : TemplateKey | None, optional
         Template key to use (default: 'ddl').
     template_path : str | None, optional
-        Path to a custom template file (overrides ``template``).
+        Path to a custom template file (overrides *template*).
 
     Returns
     -------

etlplus/database/engine.py

@@ -87,7 +87,7 @@ def load_database_url_from_config(
     Extract a database URL/DSN from a YAML/JSON config file.
 
     The loader is schema-tolerant: it looks for a top-level "databases" map
-    and then for a named entry (``name``). Each entry may contain either a
+    and then for a named entry (*name*). Each entry may contain either a
     ``connection_string``/``url``/``dsn`` or a nested ``default`` block with
     those fields.
 
@@ -136,9 +136,25 @@ def load_database_url_from_config(
     return url
 
 
-def make_engine(url: str | None = None, **engine_kwargs: Any) -> Engine:
-    """Create a SQLAlchemy Engine, defaulting to env config if no URL given."""
+def make_engine(
+    url: str | None = None,
+    **engine_kwargs: Any,
+) -> Engine:
+    """
+    Create a SQLAlchemy Engine, defaulting to env config if no URL given.
+
+    Parameters
+    ----------
+    url : str | None, optional
+        Database URL/DSN string. When omitted, ``DATABASE_URL`` is used.
+    **engine_kwargs : Any
+        Extra keyword arguments forwarded to ``create_engine``.
 
+    Returns
+    -------
+    Engine
+        Configured SQLAlchemy engine instance.
+    """
     resolved_url = url or DATABASE_URL
     return create_engine(resolved_url, pool_pre_ping=True, **engine_kwargs)
 

etlplus/database/orm.py

@@ -201,12 +201,14 @@ def build_models(
 ) -> ModelRegistry:
     """
     Build SQLAlchemy ORM models from table specifications.
+
     Parameters
     ----------
     specs : list[TableSpec]
         List of table specifications.
     base : type[DeclarativeBase], optional
         Base class for the ORM models (default: :class:`Base`).
+
     Returns
     -------
     ModelRegistry

etlplus/enums.py

@@ -22,7 +22,6 @@ __all__ = [
     # Enums
     'AggregateName',
     'CoercibleStrEnum',
-    'DataConnectorType',
     'OperatorName',
     'PipelineStep',
 ]
@@ -76,7 +75,7 @@ class CoercibleStrEnum(enum.StrEnum):
     @classmethod
     def coerce(cls, value: Self | str | object) -> Self:
         """
-        Convert an enum member or string-like input to a member of ``cls``.
+        Convert an enum member or string-like input to a member of *cls*.
 
         Parameters
         ----------
@@ -168,37 +167,6 @@ class AggregateName(CoercibleStrEnum):
         return lambda xs, n: (fmean(xs) if xs else 0.0)
 
 
-class DataConnectorType(CoercibleStrEnum):
-    """Supported data connector types."""
-
-    # -- Constants -- #
-
-    API = 'api'
-    DATABASE = 'database'
-    FILE = 'file'
-
-    # -- Class Methods -- #
-
-    @classmethod
-    def aliases(cls) -> StrStrMap:
-        """
-        Return a mapping of common aliases for each enum member.
-
-        Returns
-        -------
-        StrStrMap
-            A mapping of alias names to their corresponding enum member names.
-        """
-        return {
-            'http': 'api',
-            'https': 'api',
-            'rest': 'api',
-            'db': 'database',
-            'filesystem': 'file',
-            'fs': 'file',
-        }
-
-
 class OperatorName(CoercibleStrEnum):
     """Supported comparison operators with helpers."""
 

etlplus/file/README.md

@@ -1,4 +1,4 @@
-# etlplus.file subpackage
+# `etlplus.file` Subpackage
 
 Documentation for the `etlplus.file` subpackage: unified file format support and helpers for reading
 and writing data files.
@@ -11,7 +11,7 @@ and writing data files.
 
 Back to project overview: see the top-level [README](../../README.md).
 
-- [etlplus.file subpackage](#etlplusfile-subpackage)
+- [`etlplus.file` Subpackage](#etlplusfile-subpackage)
   - [Supported File Formats](#supported-file-formats)
   - [Inferring File Format and Compression](#inferring-file-format-and-compression)
   - [Reading and Writing Files](#reading-and-writing-files)

etlplus/file/_imports.py

@@ -12,6 +12,7 @@ from typing import Any
 # SECTION: INTERNAL CONSTANTS =============================================== #
 
 
+# Optional Python module support (lazy-loaded to avoid hard dependency)
 _MODULE_CACHE: dict[str, Any] = {}
 
 

etlplus/file/_io.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import csv
 from pathlib import Path
+from typing import Any
 from typing import cast
 
 from ..types import JSONData
@@ -17,6 +18,44 @@ from ..types import JSONList
 # SECTION: FUNCTIONS ======================================================== #
 
 
+def coerce_record_payload(
+    payload: Any,
+    *,
+    format_name: str,
+) -> JSONData:
+    """
+    Validate that *payload* is an object or list of objects.
+
+    Parameters
+    ----------
+    payload : Any
+        Parsed payload to validate.
+    format_name : str
+        Human-readable format name for error messages.
+
+    Returns
+    -------
+    JSONData
+        *payload* when it is a dict or a list of dicts.
+
+    Raises
+    ------
+    TypeError
+        If the payload is not a dict or list of dicts.
+    """
+    if isinstance(payload, dict):
+        return cast(JSONDict, payload)
+    if isinstance(payload, list):
+        if all(isinstance(item, dict) for item in payload):
+            return cast(JSONList, payload)
+        raise TypeError(
+            f'{format_name} array must contain only objects (dicts)',
+        )
+    raise TypeError(
+        f'{format_name} root must be an object or an array of objects',
+    )
+
+
 def normalize_records(
     data: JSONData,
     format_name: str,
@@ -50,9 +89,13 @@ def normalize_records(
     return [cast(JSONDict, data)]
 
 
-def read_delimited(path: Path, *, delimiter: str) -> JSONList:
+def read_delimited(
+    path: Path,
+    *,
+    delimiter: str,
+) -> JSONList:
     """
-    Read delimited content from ``path``.
+    Read delimited content from *path*.
 
     Parameters
     ----------
@@ -79,9 +122,14 @@ def read_delimited(path: Path, *, delimiter: str) -> JSONList:
     return rows
 
 
-def write_delimited(path: Path, data: JSONData, *, delimiter: str) -> int:
+def write_delimited(
+    path: Path,
+    data: JSONData,
+    *,
+    delimiter: str,
+) -> int:
     """
-    Write ``data`` to a delimited file and return record count.
+    Write *data* to a delimited file and return record count.
 
     Parameters
     ----------

etlplus/file/accdb.py

@@ -28,6 +28,7 @@ from . import stub
 
 
 __all__ = [
+    # Functions
     'read',
     'write',
 ]
@@ -40,7 +41,7 @@ def read(
     path: Path,
 ) -> JSONList:
     """
-    Read ACCDB content from ``path``.
+    Read ACCDB content from *path*.
 
     Parameters
     ----------
@@ -60,7 +61,7 @@ def write(
     data: JSONData,
 ) -> int:
     """
-    Write ``data`` to ACCDB at ``path`` and return record count.
+    Write *data* to ACCDB at *path* and return record count.
 
     Parameters
     ----------

etlplus/file/arrow.py

@@ -28,6 +28,7 @@ from . import stub
 
 
 __all__ = [
+    # Functions
     'read',
     'write',
 ]
@@ -40,7 +41,7 @@ def read(
     path: Path,
 ) -> JSONList:
     """
-    Read ARROW content from ``path``.
+    Read ARROW content from *path*.
 
     Parameters
     ----------
@@ -60,7 +61,7 @@ def write(
     data: JSONData,
 ) -> int:
     """
-    Write ``data`` to ARROW at ``path`` and return record count.
+    Write *data* to ARROW at *path* and return record count.
 
     Parameters
     ----------

etlplus/file/avro.py

@@ -33,6 +33,7 @@ from ._io import normalize_records
 
 
 __all__ = [
+    # Functions
     'read',
     'write',
 ]
@@ -124,7 +125,7 @@ def read(
     path: Path,
 ) -> JSONList:
     """
-    Read AVRO content from ``path``.
+    Read AVRO content from *path*.
 
     Parameters
     ----------
@@ -147,7 +148,7 @@ def write(
     data: JSONData,
 ) -> int:
     """
-    Write ``data`` to AVRO at ``path`` and return record count.
+    Write *data* to AVRO at *path* and return record count.
 
     Parameters
     ----------

etlplus/file/bson.py

@@ -27,6 +27,7 @@ from . import stub
 
 
 __all__ = [
+    # Functions
     'read',
     'write',
 ]
@@ -39,7 +40,7 @@ def read(
     path: Path,
 ) -> JSONList:
     """
-    Read BSON content from ``path``.
+    Read BSON content from *path*.
 
     Parameters
     ----------
@@ -59,7 +60,7 @@ def write(
     data: JSONData,
 ) -> int:
     """
-    Write ``data`` to BSON at ``path`` and return record count.
+    Write *data* to BSON at *path* and return record count.
 
     Parameters
     ----------

etlplus/file/cbor.py

@@ -28,6 +28,7 @@ from . import stub
 
 
 __all__ = [
+    # Functions
     'read',
     'write',
 ]
@@ -40,7 +41,7 @@ def read(
     path: Path,
 ) -> JSONList:
     """
-    Read CBOR content from ``path``.
+    Read CBOR content from *path*.
 
     Parameters
     ----------
@@ -60,7 +61,7 @@ def write(
     data: JSONData,
 ) -> int:
     """
-    Write ``data`` to CBOR at ``path`` and return record count.
+    Write *data* to CBOR at *path* and return record count.
 
     Parameters
     ----------

etlplus/file/cfg.py

@@ -29,6 +29,7 @@ from . import stub
 
 
 __all__ = [
+    # Functions
     'read',
     'write',
 ]
@@ -41,7 +42,7 @@ def read(
     path: Path,
 ) -> JSONList:
     """
-    Read CFG content from ``path``.
+    Read CFG content from *path*.
 
     Parameters
     ----------
@@ -61,7 +62,7 @@ def write(
     data: JSONData,
 ) -> int:
     """
-    Write ``data`` to CFG file at ``path`` and return record count.
+    Write *data* to CFG file at *path* and return record count.
 
     Parameters
     ----------