etlplus 0.12.9__tar.gz → 0.13.0__tar.gz

{etlplus-0.12.9/etlplus.egg-info → etlplus-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: etlplus
-Version: 0.12.9
+Version: 0.13.0
 Summary: A Swiss Army knife for simple ETL operations
 Home-page: https://github.com/Dagitali/ETLPlus
 Author: ETLPlus Team
@@ -77,8 +77,12 @@ package and command-line interface for data extraction, validation, transformati
       - [Semi-Structured Text](#semi-structured-text)
       - [Columnar / Analytics-Friendly](#columnar--analytics-friendly)
       - [Binary Serialization and Interchange](#binary-serialization-and-interchange)
+      - [Databases and Embedded Storage](#databases-and-embedded-storage)
       - [Spreadsheets](#spreadsheets)
+      - [Statistical / Scientific / Numeric Computing](#statistical--scientific--numeric-computing)
+      - [Logs and Event Streams](#logs-and-event-streams)
       - [Data Archives](#data-archives)
+      - [Templates](#templates)
   - [Usage](#usage)
     - [Command Line Interface](#command-line-interface)
       - [Argument Order and Required Options](#argument-order-and-required-options)
@@ -219,65 +223,122 @@ DDL can be rendered from table specs for migrations or schema checks.
 
 ### Files (`file`)
 
-File formats are grouped as in `FileFormat`. Support is marked as:
+Recognized file formats are listed in the tables below. Support for reading to or writing from a recognized file format is marked as:
 
 - **Y**: implemented (may require optional dependencies)
 - **N**: stubbed or not yet implemented
 
 #### Stubbed / Placeholder
 
-| Format | Supported | Description |
-| --- | --- | --- |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
 | `stub` | N | Placeholder format for tests and future connectors. |
 
 #### Tabular & Delimited Text
 
-| Format | Supported | Description |
-| --- | --- | --- |
-| `csv` | Y | Comma-Separated Values |
-| `fwf` | N | Fixed-Width Fields |
-| `dat` | N | Generic data file, often delimited or fixed-width |
-| `psv` | N | Pipe-Separated Values |
-| `tab` | N | Often synonymous with TSV |
-| `tsv` | Y | Tab-Separated Values |
-| `txt` | Y | Plain text, often delimited or fixed-width |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `csv` | Y | Y | Comma-Separated Values |
+| `dat` | N | N | Generic data file, often delimited or fixed-width |
+| `fwf` | N | N | Fixed-Width Fields |
+| `psv` | N | N | Pipe-Separated Values |
+| `tab` | N | N | Often synonymous with TSV |
+| `tsv` | Y | Y | Tab-Separated Values |
+| `txt` | Y | Y | Plain text, often delimited or fixed-width |
 
 #### Semi-Structured Text
 
-| Format | Supported | Description |
-| --- | --- | --- |
-| `json` | Y | JavaScript Object Notation |
-| `ndjson` | Y | Newline-Delimited JSON |
-| `xml` | Y | Extensible Markup Language |
-| `yaml` | Y | YAML Ain't Markup Language |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `cfg` | N | N | Config-style key-value pairs |
+| `conf` | N | N | Config-style key-value pairs |
+| `ini` | N | N | Config-style key-value pairs |
+| `json` | Y | Y | JavaScript Object Notation |
+| `ndjson` | Y | Y | Newline-Delimited JSON |
+| `properties` | N | N | Java-style key-value pairs |
+| `toml` | N | N | Tom's Obvious Minimal Language |
+| `xml` | Y | Y | Extensible Markup Language |
+| `yaml` | Y | Y | YAML Ain't Markup Language |
 
 #### Columnar / Analytics-Friendly
 
-| Format | Supported | Description |
-| --- | --- | --- |
-| `feather` | Y | Apache Arrow Feather |
-| `orc` | Y | Optimized Row Columnar; common in Hadoop |
-| `parquet` | Y | Apache Parquet; common in Big Data |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `arrow` | N | N | Apache Arrow IPC |
+| `feather` | Y | Y | Apache Arrow Feather |
+| `orc` | Y | Y | Optimized Row Columnar; common in Hadoop |
+| `parquet` | Y | Y | Apache Parquet; common in Big Data |
 
 #### Binary Serialization and Interchange
 
-| Format | Supported | Description |
-| --- | --- | --- |
-| `avro` | Y | Apache Avro |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `avro` | Y | Y | Apache Avro |
+| `bson` | N | N | Binary JSON; common with MongoDB exports/dumps |
+| `cbor` | N | N | Concise Binary Object Representation |
+| `ion` | N | N | Amazon Ion |
+| `msgpack` | N | N | MessagePack |
+| `pb` | N | N | Protocol Buffers (Google Protobuf) |
+| `pbf` | N | N | Protocolbuffer Binary Format; often for GIS data |
+| `proto` | N | N | Protocol Buffers schema; often in .pb / .bin |
+
+#### Databases and Embedded Storage
+
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `accdb` | N | N | Microsoft Access (newer format) |
+| `duckdb` | N | N | DuckDB |
+| `mdb` | N | N | Microsoft Access (older format) |
+| `sqlite` | N | N | SQLite |
 
 #### Spreadsheets
 
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `numbers` | N | N | Apple Numbers |
+| `ods` | N | N | OpenDocument |
+| `wks` | N | N | Lotus 1-2-3  |
+| `xls` | Y | Y | Microsoft Excel (BIFF) |
+| `xlsm` | N | N | Microsoft Excel Macro-Enabled (Open XML) |
+| `xlsx` | Y | Y | Microsoft Excel (Open XML) |
+
+#### Statistical / Scientific / Numeric Computing
+
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `dta` | N | N | Stata |
+| `hdf5` | N | N | Hierarchical Data Format |
+| `mat` | N | N | MATLAB |
+| `nc` | N | N | NetCDF |
+| `rda` | N | N | RData workspace/object |
+| `rds` | N | N | R data |
+| `sas7bdat` | N | N | SAS data |
+| `sav` | N | N | SPSS data |
+| `sylk` | N | N | Symbolic Link |
+| `xpt` | N | N | SAS Transport |
+| `zsav` | N | N | Compressed SPSS data |
+
+#### Logs and Event Streams
+
 | Format | Supported | Description |
 | --- | --- | --- |
-| `xls` | Y | Microsoft Excel (BIFF); read-only |
-| `xlsx` | Y | Microsoft Excel (Open XML) |
+| `log` | N | N | Generic log file |
 
 #### Data Archives
 
-| Format | Supported | Description |
-| --- | --- | --- |
-| `gz` | Y | Gzip-compressed file |
-| `zip` | Y | ZIP archive |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `gz` | Y | Y | Gzip-compressed file |
+| `zip` | Y | Y | ZIP archive |
+
+#### Templates
+
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `hbs` | N | N | Handlebars |
+| `jinja2` | N | N | Jinja2 |
+| `mustache` | N | N | Mustache |
+| `vm` | N | N | Apache Velocity |
 
 ## Usage
 

{etlplus-0.12.9 → etlplus-0.13.0}/README.md

@@ -27,8 +27,12 @@ package and command-line interface for data extraction, validation, transformati
       - [Semi-Structured Text](#semi-structured-text)
       - [Columnar / Analytics-Friendly](#columnar--analytics-friendly)
       - [Binary Serialization and Interchange](#binary-serialization-and-interchange)
+      - [Databases and Embedded Storage](#databases-and-embedded-storage)
       - [Spreadsheets](#spreadsheets)
+      - [Statistical / Scientific / Numeric Computing](#statistical--scientific--numeric-computing)
+      - [Logs and Event Streams](#logs-and-event-streams)
       - [Data Archives](#data-archives)
+      - [Templates](#templates)
   - [Usage](#usage)
     - [Command Line Interface](#command-line-interface)
       - [Argument Order and Required Options](#argument-order-and-required-options)
@@ -169,65 +173,122 @@ DDL can be rendered from table specs for migrations or schema checks.
 
 ### Files (`file`)
 
-File formats are grouped as in `FileFormat`. Support is marked as:
+Recognized file formats are listed in the tables below. Support for reading to or writing from a recognized file format is marked as:
 
 - **Y**: implemented (may require optional dependencies)
 - **N**: stubbed or not yet implemented
 
 #### Stubbed / Placeholder
 
-| Format | Supported | Description |
-| --- | --- | --- |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
 | `stub` | N | Placeholder format for tests and future connectors. |
 
 #### Tabular & Delimited Text
 
-| Format | Supported | Description |
-| --- | --- | --- |
-| `csv` | Y | Comma-Separated Values |
-| `fwf` | N | Fixed-Width Fields |
-| `dat` | N | Generic data file, often delimited or fixed-width |
-| `psv` | N | Pipe-Separated Values |
-| `tab` | N | Often synonymous with TSV |
-| `tsv` | Y | Tab-Separated Values |
-| `txt` | Y | Plain text, often delimited or fixed-width |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `csv` | Y | Y | Comma-Separated Values |
+| `dat` | N | N | Generic data file, often delimited or fixed-width |
+| `fwf` | N | N | Fixed-Width Fields |
+| `psv` | N | N | Pipe-Separated Values |
+| `tab` | N | N | Often synonymous with TSV |
+| `tsv` | Y | Y | Tab-Separated Values |
+| `txt` | Y | Y | Plain text, often delimited or fixed-width |
 
 #### Semi-Structured Text
 
-| Format | Supported | Description |
-| --- | --- | --- |
-| `json` | Y | JavaScript Object Notation |
-| `ndjson` | Y | Newline-Delimited JSON |
-| `xml` | Y | Extensible Markup Language |
-| `yaml` | Y | YAML Ain't Markup Language |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `cfg` | N | N | Config-style key-value pairs |
+| `conf` | N | N | Config-style key-value pairs |
+| `ini` | N | N | Config-style key-value pairs |
+| `json` | Y | Y | JavaScript Object Notation |
+| `ndjson` | Y | Y | Newline-Delimited JSON |
+| `properties` | N | N | Java-style key-value pairs |
+| `toml` | N | N | Tom's Obvious Minimal Language |
+| `xml` | Y | Y | Extensible Markup Language |
+| `yaml` | Y | Y | YAML Ain't Markup Language |
 
 #### Columnar / Analytics-Friendly
 
-| Format | Supported | Description |
-| --- | --- | --- |
-| `feather` | Y | Apache Arrow Feather |
-| `orc` | Y | Optimized Row Columnar; common in Hadoop |
-| `parquet` | Y | Apache Parquet; common in Big Data |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `arrow` | N | N | Apache Arrow IPC |
+| `feather` | Y | Y | Apache Arrow Feather |
+| `orc` | Y | Y | Optimized Row Columnar; common in Hadoop |
+| `parquet` | Y | Y | Apache Parquet; common in Big Data |
 
 #### Binary Serialization and Interchange
 
-| Format | Supported | Description |
-| --- | --- | --- |
-| `avro` | Y | Apache Avro |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `avro` | Y | Y | Apache Avro |
+| `bson` | N | N | Binary JSON; common with MongoDB exports/dumps |
+| `cbor` | N | N | Concise Binary Object Representation |
+| `ion` | N | N | Amazon Ion |
+| `msgpack` | N | N | MessagePack |
+| `pb` | N | N | Protocol Buffers (Google Protobuf) |
+| `pbf` | N | N | Protocolbuffer Binary Format; often for GIS data |
+| `proto` | N | N | Protocol Buffers schema; often in .pb / .bin |
+
+#### Databases and Embedded Storage
+
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `accdb` | N | N | Microsoft Access (newer format) |
+| `duckdb` | N | N | DuckDB |
+| `mdb` | N | N | Microsoft Access (older format) |
+| `sqlite` | N | N | SQLite |
 
 #### Spreadsheets
 
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `numbers` | N | N | Apple Numbers |
+| `ods` | N | N | OpenDocument |
+| `wks` | N | N | Lotus 1-2-3  |
+| `xls` | Y | Y | Microsoft Excel (BIFF) |
+| `xlsm` | N | N | Microsoft Excel Macro-Enabled (Open XML) |
+| `xlsx` | Y | Y | Microsoft Excel (Open XML) |
+
+#### Statistical / Scientific / Numeric Computing
+
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `dta` | N | N | Stata |
+| `hdf5` | N | N | Hierarchical Data Format |
+| `mat` | N | N | MATLAB |
+| `nc` | N | N | NetCDF |
+| `rda` | N | N | RData workspace/object |
+| `rds` | N | N | R data |
+| `sas7bdat` | N | N | SAS data |
+| `sav` | N | N | SPSS data |
+| `sylk` | N | N | Symbolic Link |
+| `xpt` | N | N | SAS Transport |
+| `zsav` | N | N | Compressed SPSS data |
+
+#### Logs and Event Streams
+
 | Format | Supported | Description |
 | --- | --- | --- |
-| `xls` | Y | Microsoft Excel (BIFF); read-only |
-| `xlsx` | Y | Microsoft Excel (Open XML) |
+| `log` | N | N | Generic log file |
 
 #### Data Archives
 
-| Format | Supported | Description |
-| --- | --- | --- |
-| `gz` | Y | Gzip-compressed file |
-| `zip` | Y | ZIP archive |
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `gz` | Y | Y | Gzip-compressed file |
+| `zip` | Y | Y | ZIP archive |
+
+#### Templates
+
+| Format | Read | Write | Description |
+| --- | --- | --- | --- |
+| `hbs` | N | N | Handlebars |
+| `jinja2` | N | N | Jinja2 |
+| `mustache` | N | N | Mustache |
+| `vm` | N | N | Apache Velocity |
 
 ## Usage
 

{etlplus-0.12.9 → etlplus-0.13.0}/docs/pipeline-guide.md

@@ -390,10 +390,20 @@ target:
 ```yaml
 jobs:
   - name: file_to_file_customers
+    depends_on: [seed_customers]
     extract: { source: customers_csv }
     validate: { ruleset: customers_basic }
     transform: { pipeline: clean_customers }
     load: { target: customers_json_out }
+  - name: seed_customers
+    extract: { source: seed_customers_csv }
+    load: { target: customers_db_out }
+```
+
+Notes:
+
+- `depends_on` is optional and can be a string or list of job names.
+- Jobs without dependencies run first when ordered as a DAG.
 
 ## Running pipelines (CLI and Python)
 

{etlplus-0.12.9 → etlplus-0.13.0}/etlplus/config/jobs.py

@@ -34,10 +34,7 @@ __all__ = [
 ]
 
 
-# SECTION: TYPE ALIASES ===================================================== #
-
-
-# SECTION: CLASSES ========================================================== #
+# SECTION: DATA CLASSES ===================================================== #
 
 
 @dataclass(kw_only=True, slots=True)
@@ -100,6 +97,8 @@ class JobConfig:
         Unique job name.
     description : str | None
         Optional human-friendly description.
+    depends_on : list[str]
+        Optional job dependency list. Dependencies must refer to other jobs.
     extract : ExtractRef | None
         Extraction reference.
     validate : ValidationRef | None
@@ -114,6 +113,7 @@ class JobConfig:
 
     name: str
     description: str | None = None
+    depends_on: list[str] = field(default_factory=list)
     extract: ExtractRef | None = None
     validate: ValidationRef | None = None
     transform: TransformRef | None = None
@@ -149,9 +149,19 @@ class JobConfig:
         if description is not None and not isinstance(description, str):
             description = str(description)
 
+        depends_raw = data.get('depends_on')
+        depends_on: list[str] = []
+        if isinstance(depends_raw, str):
+            depends_on = [depends_raw]
+        elif isinstance(depends_raw, list):
+            for entry in depends_raw:
+                if isinstance(entry, str):
+                    depends_on.append(entry)
+
         return cls(
             name=name,
             description=description,
+            depends_on=depends_on,
             extract=ExtractRef.from_obj(data.get('extract')),
             validate=ValidationRef.from_obj(data.get('validate')),
             transform=TransformRef.from_obj(data.get('transform')),

etlplus-0.13.0/etlplus/dag.py

@@ -0,0 +1,103 @@
+"""
+:mod:`etlplus.dag` module.
+
+Lightweight directed acyclic graph (DAG) helpers for ordering jobs based on
+``depends_on``.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass
+
+from .config.jobs import JobConfig
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'DagError',
+    'topological_sort_jobs',
+]
+
+
+# SECTION: ERRORS =========================================================== #
+
+
+@dataclass(slots=True)
+class DagError(ValueError):
+    """
+    Raised when the job dependency graph is invalid.
+
+    Attributes
+    ----------
+    message : str
+        Error message.
+    """
+
+    # -- Attributes -- #
+
+    message: str
+
+    # -- Magic Methods (Object Representation) -- #
+
+    def __str__(self) -> str:
+        return self.message
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def topological_sort_jobs(
+    jobs: list[JobConfig],
+) -> list[JobConfig]:
+    """
+    Return jobs in topological order based on ``depends_on``.
+
+    Parameters
+    ----------
+    jobs : list[JobConfig]
+        List of job configurations to sort.
+
+    Returns
+    -------
+    list[JobConfig]
+        Jobs sorted in topological order.
+
+    Raises
+    ------
+    DagError
+        If a dependency is missing, self-referential, or when a cycle is
+        detected.
+    """
+    index = {job.name: job for job in jobs}
+    edges: dict[str, set[str]] = {name: set() for name in index}
+    indegree: dict[str, int] = {name: 0 for name in index}
+
+    for job in jobs:
+        for dep in job.depends_on:
+            if dep not in index:
+                raise DagError(
+                    f'Unknown dependency "{dep}" in job "{job.name}"',
+                )
+            if dep == job.name:
+                raise DagError(f'Job "{job.name}" depends on itself')
+            if job.name not in edges[dep]:
+                edges[dep].add(job.name)
+                indegree[job.name] += 1
+
+    queue = deque(sorted(name for name, deg in indegree.items() if deg == 0))
+    ordered: list[str] = []
+
+    while queue:
+        name = queue.popleft()
+        ordered.append(name)
+        for child in sorted(edges[name]):
+            indegree[child] -= 1
+            if indegree[child] == 0:
+                queue.append(child)
+
+    if len(ordered) != len(jobs):
+        raise DagError('Dependency cycle detected')
+
+    return [index[name] for name in ordered]

etlplus-0.13.0/etlplus/file/accdb.py

@@ -0,0 +1,78 @@
+"""
+:mod:`etlplus.file.accdb` module.
+
+Helpers for reading/writing newer Microsoft Access database (ACCDB) files.
+
+Notes
+-----
+- An ACCDB file is a proprietary database file format used by Microsoft Access
+    2007 and later.
+- Common cases:
+    - Storing relational data for small to medium-sized applications.
+    - Desktop database applications.
+    - Data management for non-enterprise solutions.
+- Rule of thumb:
+    - If the file follows the ACCDB specification, use this module for reading
+        and writing.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ..types import JSONData
+from ..types import JSONList
+from . import stub
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
+    """
+    Read ACCDB content from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the ACCDB file on disk.
+
+    Returns
+    -------
+    JSONList
+        The list of dictionaries read from the ACCDB file.
+    """
+    return stub.read(path, format_name='ACCDB')
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write ``data`` to ACCDB at ``path`` and return record count.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the ACCDB file on disk.
+    data : JSONData
+        Data to write as ACCDB. Should be a list of dictionaries or a single
+        dictionary.
+
+    Returns
+    -------
+    int
+        The number of rows written to the ACCDB file.
+    """
+    return stub.write(path, data, format_name='ACCDB')

etlplus-0.13.0/etlplus/file/arrow.py

@@ -0,0 +1,78 @@
+"""
+:mod:`etlplus.file.arrow` module.
+
+Helpers for reading/writing Apache Arrow (ARROW) files.
+
+Notes
+-----
+- An ARROW file is a binary file format designed for efficient
+    columnar data storage and processing.
+- Common cases:
+    - High-performance data analytics.
+    - Interoperability between different data processing systems.
+    - In-memory data representation for fast computations.
+- Rule of thumb:
+    - If the file follows the Apache Arrow specification, use this module for
+        reading and writing.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ..types import JSONData
+from ..types import JSONList
+from . import stub
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'read',
+    'write',
+]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def read(
+    path: Path,
+) -> JSONList:
+    """
+    Read ARROW content from ``path``.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the Apache Arrow file on disk.
+
+    Returns
+    -------
+    JSONList
+        The list of dictionaries read from the Apache Arrow file.
+    """
+    return stub.read(path, format_name='ARROW')
+
+
+def write(
+    path: Path,
+    data: JSONData,
+) -> int:
+    """
+    Write ``data`` to ARROW at ``path`` and return record count.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the ARROW file on disk.
+    data : JSONData
+        Data to write as ARROW. Should be a list of dictionaries or a
+        single dictionary.
+
+    Returns
+    -------
+    int
+        The number of rows written to the ARROW file.
+    """
+    return stub.write(path, data, format_name='ARROW')