etlplus 0.16.10__py3-none-any.whl → 0.17.3__py3-none-any.whl

etlplus/file/README.md

@@ -9,6 +9,12 @@ and writing data files.
   types
 - Exposes a `File` class with instance methods for reading and writing data
 
+Some formats require optional dependencies. Install with:
+
+```bash
+pip install -e ".[file]"
+```
+
 Back to project overview: see the top-level [README](../../README.md).
 
 - [`etlplus.file` Subpackage](#etlplusfile-subpackage)
@@ -29,21 +35,48 @@ matrix across all `FileFormat` values, see the top-level [README](../../README.m
 | Format    | Description                                 |
 |-----------|---------------------------------------------|
 | avro      | Apache Avro binary serialization            |
+| arrow     | Apache Arrow IPC                            |
+| bson      | Binary JSON (BSON)                          |
+| cbor      | Concise Binary Object Representation        |
 | csv       | Comma-separated values text files           |
+| dat       | Generic data files (delimited)              |
+| dta       | Stata datasets                              |
+| duckdb    | DuckDB database file                        |
 | feather   | Apache Arrow Feather columnar format        |
+| fwf       | Fixed-width formatted text files            |
 | gz        | Gzip-compressed files (see Compression)     |
+| hdf5      | Hierarchical Data Format                    |
+| ini       | INI config files                            |
 | json      | Standard JSON files                         |
+| msgpack   | MessagePack binary serialization            |
+| nc        | NetCDF datasets                             |
 | ndjson    | Newline-delimited JSON (JSON Lines)         |
+| ods       | OpenDocument spreadsheets                   |
 | orc       | Apache ORC columnar format                  |
 | parquet   | Apache Parquet columnar format              |
+| pb        | Protocol Buffers binary                     |
+| properties | Java-style properties                     |
+| proto     | Protocol Buffers schema                     |
+| psv       | Pipe-separated values text files            |
+| rda       | RData workspace bundles                     |
+| rds       | RDS datasets                                |
+| sas7bdat  | SAS datasets                                |
+| sav       | SPSS datasets                               |
+| sqlite    | SQLite database file                        |
+| tab       | Tab-delimited text files                    |
+| toml      | TOML config files                           |
 | tsv       | Tab-separated values text files             |
 | txt       | Plain text files                            |
 | xls       | Microsoft Excel (legacy .xls; read-only)    |
+| xlsm      | Microsoft Excel Macro-Enabled (XLSM)        |
 | xlsx      | Microsoft Excel (modern .xlsx)              |
+| xpt       | SAS transport files                         |
 | zip       | ZIP-compressed files (see Compression)      |
 | xml       | XML files                                   |
 | yaml      | YAML files                                  |
 
+Note: HDF5 support is read-only; writing is currently disabled.
+
 Compression formats (gz, zip) are also supported as wrappers for other formats. Formats not listed
 here are currently stubbed and will raise `NotImplementedError` on read/write.
 

etlplus/file/_imports.py

@@ -22,6 +22,7 @@ _MODULE_CACHE: dict[str, Any] = {}
 def _error_message(
     module_name: str,
     format_name: str,
+    pip_name: str | None = None,
 ) -> str:
     """
     Build an import error message for an optional dependency.
@@ -32,16 +33,19 @@ def _error_message(
         Module name to look up.
     format_name : str
         Human-readable format name for templated messages.
+    pip_name : str | None, optional
+        Package name to suggest for installation. Defaults to *module_name*.
 
     Returns
     -------
     str
         Formatted error message.
     """
+    install_name = pip_name or module_name
     return (
         f'{format_name} support requires '
-        f'optional dependency "{module_name}".\n'
-        f'Install with: pip install {module_name}'
+        f'optional dependency "{install_name}".\n'
+        f'Install with: pip install {install_name}'
     )
 
 
@@ -84,19 +88,36 @@ def get_optional_module(
     return module
 
 
-def get_fastavro() -> Any:
+def get_dependency(
+    module_name: str,
+    *,
+    format_name: str,
+    pip_name: str | None = None,
+) -> Any:
     """
-    Return the fastavro module, importing it on first use.
+    Return an optional dependency module with a standardized error message.
 
-    Raises an informative ImportError if the optional dependency is missing.
+    Parameters
+    ----------
+    module_name : str
+        Name of the module to import.
+    format_name : str
+        Human-readable format name for error messages.
+    pip_name : str | None, optional
+        Package name to suggest for installation (defaults to *module_name*).
 
-    Notes
-    -----
-    Prefer :func:`get_optional_module` for new call sites.
+    Returns
+    -------
+    Any
+        The imported module.
     """
     return get_optional_module(
-        'fastavro',
-        error_message=_error_message('fastavro', format_name='AVRO'),
+        module_name,
+        error_message=_error_message(
+            module_name,
+            format_name=format_name,
+            pip_name=pip_name,
+        ),
     )
 
 
@@ -118,12 +139,9 @@ def get_pandas(
 
     Notes
     -----
-    Prefer :func:`get_optional_module` for new call sites.
+    Prefer :func:`get_dependency` for new call sites.
     """
-    return get_optional_module(
-        'pandas',
-        error_message=_error_message('pandas', format_name=format_name),
-    )
+    return get_dependency('pandas', format_name=format_name)
 
 
 def get_yaml() -> Any:
@@ -134,9 +152,6 @@ def get_yaml() -> Any:
 
     Notes
     -----
-    Prefer :func:`get_optional_module` for new call sites.
+    Prefer :func:`get_dependency` for new call sites.
     """
-    return get_optional_module(
-        'yaml',
-        error_message=_error_message('PyYAML', format_name='YAML'),
-    )
+    return get_dependency('yaml', format_name='YAML', pip_name='PyYAML')

etlplus/file/_io.py

@@ -14,10 +14,30 @@ from typing import cast
 from ..types import JSONData
 from ..types import JSONDict
 from ..types import JSONList
+from ..types import StrPath
 
 # SECTION: FUNCTIONS ======================================================== #
 
 
+def coerce_path(
+    path: StrPath,
+) -> Path:
+    """
+    Coerce path-like inputs into :class:`~pathlib.Path`.
+
+    Parameters
+    ----------
+    path : StrPath
+        Path-like input to normalize.
+
+    Returns
+    -------
+    Path
+        Normalized :class:`~pathlib.Path` instance.
+    """
+    return path if isinstance(path, Path) else Path(path)
+
+
 def coerce_record_payload(
     payload: Any,
     *,
@@ -56,6 +76,21 @@ def coerce_record_payload(
     )
 
 
+def ensure_parent_dir(
+    path: StrPath,
+) -> None:
+    """
+    Ensure the parent directory for *path* exists.
+
+    Parameters
+    ----------
+    path : StrPath
+        Target path to ensure the parent directory for.
+    """
+    path = coerce_path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+
 def normalize_records(
     data: JSONData,
     format_name: str,
@@ -78,7 +113,7 @@ def normalize_records(
     Raises
     ------
     TypeError
-        If a list payload contains non-dict items.
+        If the payload is not a dict or a list of dicts.
     """
     if isinstance(data, list):
         if not all(isinstance(item, dict) for item in data):
@@ -86,11 +121,15 @@ def normalize_records(
                 f'{format_name} payloads must contain only objects (dicts)',
             )
         return cast(JSONList, data)
-    return [cast(JSONDict, data)]
+    if isinstance(data, dict):
+        return [cast(JSONDict, data)]
+    raise TypeError(
+        f'{format_name} payloads must be an object or an array of objects',
+    )
 
 
 def read_delimited(
-    path: Path,
+    path: StrPath,
     *,
     delimiter: str,
 ) -> JSONList:
@@ -99,7 +138,7 @@ def read_delimited(
 
     Parameters
     ----------
-    path : Path
+    path : StrPath
         Path to the delimited file on disk.
     delimiter : str
         Delimiter character for parsing.
@@ -109,6 +148,7 @@ def read_delimited(
     JSONList
         The list of dictionaries read from the delimited file.
     """
+    path = coerce_path(path)
     with path.open('r', encoding='utf-8', newline='') as handle:
         reader: csv.DictReader[str] = csv.DictReader(
             handle,
@@ -122,40 +162,123 @@ def read_delimited(
     return rows
 
 
+def require_dict_payload(
+    data: JSONData,
+    *,
+    format_name: str,
+) -> JSONDict:
+    """
+    Validate that *data* is a dictionary payload.
+
+    Parameters
+    ----------
+    data : JSONData
+        Input payload to validate.
+    format_name : str
+        Human-readable format name for error messages.
+
+    Returns
+    -------
+    JSONDict
+        Validated dictionary payload.
+
+    Raises
+    ------
+    TypeError
+        If the payload is not a dictionary.
+    """
+    if isinstance(data, list) or not isinstance(data, dict):
+        raise TypeError(f'{format_name} payloads must be a dict')
+    return cast(JSONDict, data)
+
+
+def require_str_key(
+    payload: JSONDict,
+    *,
+    format_name: str,
+    key: str,
+) -> str:
+    """
+    Require a string value for *key* in *payload*.
+
+    Parameters
+    ----------
+    payload : JSONDict
+        Dictionary payload to inspect.
+    format_name : str
+        Human-readable format name for error messages.
+    key : str
+        Key to extract.
+
+    Returns
+    -------
+    str
+        The string value for *key*.
+
+    Raises
+    ------
+    TypeError
+        If the key is missing or not a string.
+    """
+    value = payload.get(key)
+    if not isinstance(value, str):
+        raise TypeError(
+            f'{format_name} payloads must include a "{key}" string',
+        )
+    return value
+
+
+def stringify_value(value: Any) -> str:
+    """
+    Normalize configuration-like values into strings.
+
+    Parameters
+    ----------
+    value : Any
+        Value to normalize.
+
+    Returns
+    -------
+    str
+        Stringified value (``''`` for ``None``).
+    """
+    if value is None:
+        return ''
+    return str(value)
+
+
 def write_delimited(
-    path: Path,
+    path: StrPath,
     data: JSONData,
     *,
     delimiter: str,
+    format_name: str = 'Delimited',
 ) -> int:
     """
     Write *data* to a delimited file and return record count.
 
     Parameters
     ----------
-    path : Path
+    path : StrPath
         Path to the delimited file on disk.
     data : JSONData
         Data to write as delimited rows.
     delimiter : str
         Delimiter character for writing.
+    format_name : str, optional
+        Human-readable format name for error messages. Defaults to
+        ``'Delimited'``.
 
     Returns
     -------
     int
         The number of rows written.
     """
-    rows: list[JSONDict]
-    if isinstance(data, list):
-        rows = [row for row in data if isinstance(row, dict)]
-    else:
-        rows = [data]
-
-    if not rows:
-        return 0
+    path = coerce_path(path)
+    rows = normalize_records(data, format_name)
 
     fieldnames = sorted({key for row in rows for key in row})
-    path.parent.mkdir(parents=True, exist_ok=True)
+    ensure_parent_dir(path)
     with path.open('w', encoding='utf-8', newline='') as handle:
         writer = csv.DictWriter(
             handle,

etlplus/file/_r.py

@@ -0,0 +1,48 @@
+"""
+:mod:`etlplus.file._r` module.
+
+Shared helpers for R-related file formats.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..types import JSONData
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    'coerce_r_object',
+]
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def coerce_r_object(value: Any, pandas: Any) -> JSONData:
+    """
+    Normalize a pyreadr object into JSON-friendly data.
+
+    Parameters
+    ----------
+    value : Any
+        Object returned by ``pyreadr``.
+    pandas : Any
+        pandas module used for DataFrame checks.
+
+    Returns
+    -------
+    JSONData
+        Normalized JSON-like payload.
+    """
+    if isinstance(value, pandas.DataFrame):
+        return value.to_dict(orient='records')
+    if isinstance(value, dict):
+        return value
+    if isinstance(value, list) and all(
+        isinstance(item, dict) for item in value
+    ):
+        return value
+    return {'value': value}

etlplus/file/_sql.py

@@ -0,0 +1,224 @@
+"""
+:mod:`etlplus.file._sql` module.
+
+Shared helpers for lightweight SQL-backed file formats.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any
+
+from ..types import JSONList
+
+# SECTION: EXPORTS ========================================================== #
+
+
+__all__ = [
+    # Constants
+    'DEFAULT_TABLE',
+    'DUCKDB_DIALECT',
+    'SQLITE_DIALECT',
+    # Data Classes
+    'SqlDialect',
+    # Functions
+    'collect_column_values',
+    'coerce_sql_value',
+    'infer_column_type',
+    'quote_identifier',
+    'resolve_table',
+]
+
+
+# SECTION: DATA CLASSES ===================================================== #
+
+
+@dataclass(frozen=True, slots=True)
+class SqlDialect:
+    """
+    Simple SQL type mapping for inferred column types.
+
+    Attributes
+    ----------
+    text : str
+        Textual column type name.
+    integer : str
+        Integer column type name.
+    floating : str
+        Floating-point column type name.
+    boolean : str
+        Boolean column type name.
+    """
+
+    text: str
+    integer: str
+    floating: str
+    boolean: str
+
+
+# SECTION: CONSTANTS ======================================================== #
+
+
+DEFAULT_TABLE = 'data'
+
+SQLITE_DIALECT = SqlDialect(
+    text='TEXT',
+    integer='INTEGER',
+    floating='REAL',
+    boolean='INTEGER',
+)
+
+DUCKDB_DIALECT = SqlDialect(
+    text='VARCHAR',
+    integer='BIGINT',
+    floating='DOUBLE',
+    boolean='BOOLEAN',
+)
+
+
+# SECTION: FUNCTIONS ======================================================== #
+
+
+def coerce_sql_value(value: Any) -> Any:
+    """
+    Normalize values into SQL-compatible scalar types.
+
+    Parameters
+    ----------
+    value : Any
+        Value to normalize.
+
+    Returns
+    -------
+    Any
+        Scalar value or serialized JSON string for complex objects.
+    """
+    if value is None or isinstance(value, (str, int, float, bool)):
+        return value
+    return json.dumps(value, ensure_ascii=True)
+
+
+def collect_column_values(
+    records: JSONList,
+) -> tuple[list[str], dict[str, list[Any]]]:
+    """
+    Collect column names and values from record payloads.
+
+    Parameters
+    ----------
+    records : JSONList
+        Record payloads to scan.
+
+    Returns
+    -------
+    tuple[list[str], dict[str, list[Any]]]
+        Sorted column names and mapped column values.
+    """
+    columns = sorted({key for row in records for key in row})
+    column_values: dict[str, list[Any]] = {col: [] for col in columns}
+    for row in records:
+        for column in columns:
+            column_values[column].append(row.get(column))
+    return columns, column_values
+
+
+def infer_column_type(values: list[Any], dialect: SqlDialect) -> str:
+    """
+    Infer a SQL column type for the provided values.
+
+    Parameters
+    ----------
+    values : list[Any]
+        Sample values for a column.
+    dialect : SqlDialect
+        Dialect mapping for type names.
+
+    Returns
+    -------
+    str
+        Dialect-specific type name.
+    """
+    seen_bool = False
+    seen_int = False
+    seen_float = False
+    seen_other = False
+    for value in values:
+        if value is None:
+            continue
+        if isinstance(value, bool):
+            seen_bool = True
+        elif isinstance(value, int):
+            seen_int = True
+        elif isinstance(value, float):
+            seen_float = True
+        else:
+            seen_other = True
+            break
+    if seen_other:
+        return dialect.text
+    if seen_float:
+        return dialect.floating
+    if seen_int:
+        return dialect.integer
+    if seen_bool:
+        return dialect.boolean
+    return dialect.text
+
+
+def quote_identifier(value: str) -> str:
+    """
+    Return a safely quoted SQL identifier.
+
+    Parameters
+    ----------
+    value : str
+        Identifier to quote.
+
+    Returns
+    -------
+    str
+        Quoted identifier.
+    """
+    escaped = value.replace('"', '""')
+    return f'"{escaped}"'
+
+
+def resolve_table(
+    tables: list[str],
+    *,
+    engine_name: str,
+    default_table: str = DEFAULT_TABLE,
+) -> str | None:
+    """
+    Pick a table name for read operations.
+
+    Parameters
+    ----------
+    tables : list[str]
+        Table names available in the database.
+    engine_name : str
+        Engine name used in error messages.
+    default_table : str, optional
+        Preferred table name to look for first.
+
+    Returns
+    -------
+    str | None
+        Selected table name or ``None`` when no tables exist.
+
+    Raises
+    ------
+    ValueError
+        If multiple candidate tables exist.
+    """
+    if not tables:
+        return None
+    if default_table in tables:
+        return default_table
+    if len(tables) == 1:
+        return tables[0]
+    raise ValueError(
+        f'Multiple tables found in {engine_name} file; expected '
+        f'"{default_table}" or a single table',
+    )