nexolith 0.2.0__py3-none-any.whl

nexolith/README.md

@@ -0,0 +1,10 @@
+# Nexolith package
+
+This directory contains the application package. Configuration, connectors, transformations,
+execution state, CLI presentation, and domain exceptions remain separated by subpackage.
+
+New pipeline capabilities should depend on the shared row types and domain interfaces rather than
+CLI code. See the [project README](../../README.md) for installation and usage.
+
+`release_validation.py` is maintainer tooling used by the tagged-release workflow to enforce the
+version, changelog, and distribution contract before publishing.

nexolith/__init__.py

@@ -0,0 +1,3 @@
+"""Nexolith public package."""
+
+__version__ = "0.2.0"

nexolith/cli/README.md

@@ -0,0 +1,16 @@
+# CLI
+
+The Typer application exposes `nexolith validate`, `nexolith run`, `nexolith diagnostics`, and
+`nexolith --version`.
+Presentation belongs here; pipeline execution and I/O belong in their dedicated packages.
+
+`diagnostics` prints deterministic, issue-friendly environment information. It reports Nexolith,
+Python, platform, dependency, and optional-feature versions without inspecting environment
+variables or reporting usernames, hostnames, filesystem paths, or connection details.
+
+Expected errors render as `Error [category]: message` without a traceback. Configuration failures
+exit with code `2`; execution, connector, and transformation failures exit with code `3`.
+Unexpected programming errors are allowed to propagate for debugging.
+
+Add CLI tests under `tests/unit/` whenever diagnostics or exit codes change. Tests must use
+recognizable sentinel secrets and generic assertions that never echo those values on failure.

nexolith/cli/__init__.py

@@ -0,0 +1,3 @@
+from nexolith.cli.app import app
+
+__all__ = ["app"]

nexolith/cli/app.py

@@ -0,0 +1,105 @@
+import logging
+from enum import IntEnum
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from nexolith import __version__
+from nexolith.cli.diagnostics import (
+    collect_environment_diagnostics,
+    render_environment_diagnostics,
+)
+from nexolith.config import load_pipeline
+from nexolith.exceptions import (
+    ConfigurationError,
+    ConnectorError,
+    ExecutionError,
+    NexolithError,
+    TransformationError,
+)
+from nexolith.execution import DefaultPipelineRunner
+from nexolith.models import ExecutionResult
+
+app = typer.Typer(help="Build data flows that last.", no_args_is_help=True)
+
+
+class ExitCode(IntEnum):
+    CONFIGURATION_ERROR = 2
+    EXECUTION_ERROR = 3
+
+
+def _error_category(error: NexolithError) -> str:
+    cause = error.__cause__ if isinstance(error, ExecutionError) else error
+    if isinstance(cause, ConfigurationError):
+        return "configuration"
+    if isinstance(cause, ConnectorError):
+        return "connector"
+    if isinstance(cause, TransformationError):
+        return "transformation"
+    return "execution"
+
+
+def _show_error(error: NexolithError) -> None:
+    typer.echo(f"Error [{_error_category(error)}]: {error}", err=True)
+
+
+def version_callback(value: bool) -> None:
+    if value:
+        typer.echo(f"Nexolith {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def main(
+    version: bool = typer.Option(
+        False, "--version", callback=version_callback, is_eager=True, help="Show version."
+    ),
+) -> None:
+    """Nexolith pipeline CLI."""
+
+
+def _show_result(result: ExecutionResult) -> None:
+    duration = result.duration_seconds or 0.0
+    typer.echo(f"Pipeline: {result.pipeline_name}")
+    typer.echo(f"Status: {result.status.value}")
+    typer.echo(f"Rows read: {result.rows_read}")
+    typer.echo(f"Rows written: {result.rows_written}")
+    typer.echo(f"Duration: {duration:.3f}s")
+    if result.error:
+        typer.echo(f"Error: {result.error}", err=True)
+
+
+@app.command()
+def diagnostics() -> None:
+    """Print secret-safe environment details for troubleshooting."""
+    report = collect_environment_diagnostics()
+    typer.echo(render_environment_diagnostics(report))
+
+
+@app.command()
+def validate(path: Annotated[Path, typer.Argument(exists=False, readable=True)]) -> None:
+    """Validate a pipeline YAML file without running it."""
+    try:
+        config = load_pipeline(path)
+    except ConfigurationError as exc:
+        _show_error(exc)
+        raise typer.Exit(code=ExitCode.CONFIGURATION_ERROR) from exc
+    typer.echo(f"Pipeline '{config.name}' is valid.")
+
+
+@app.command()
+def run(path: Annotated[Path, typer.Argument(exists=False, readable=True)]) -> None:
+    """Run a pipeline YAML file."""
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
+    try:
+        config = load_pipeline(path)
+    except ConfigurationError as exc:
+        _show_error(exc)
+        raise typer.Exit(code=ExitCode.CONFIGURATION_ERROR) from exc
+    try:
+        result = DefaultPipelineRunner().run(config, raise_on_error=True)
+    except ExecutionError as exc:
+        _show_error(exc)
+        raise typer.Exit(code=ExitCode.EXECUTION_ERROR) from exc
+    _show_result(result)

nexolith/cli/diagnostics.py

@@ -0,0 +1,125 @@
+"""Collect and render shareable, secret-safe environment diagnostics."""
+
+import json
+import platform
+import sqlite3
+from dataclasses import dataclass
+from importlib import metadata
+
+from nexolith import __version__
+
+
+@dataclass(frozen=True)
+class DiagnosticItem:
+    """A named diagnostic value rendered in a stable order."""
+
+    name: str
+    value: str
+
+
+@dataclass(frozen=True)
+class EnvironmentDiagnostics:
+    """Environment details that are safe to include in a public issue."""
+
+    nexolith_version: str
+    python_version: str
+    python_implementation: str
+    operating_system: str
+    operating_system_release: str
+    machine: str
+    installation: str
+    dependencies: tuple[DiagnosticItem, ...]
+    features: tuple[DiagnosticItem, ...]
+
+
+_CORE_DISTRIBUTIONS = (
+    ("Pydantic", "pydantic"),
+    ("PyYAML", "PyYAML"),
+    ("SQLAlchemy", "SQLAlchemy"),
+    ("Typer", "typer"),
+)
+
+
+def _known(value: str) -> str:
+    return value or "unknown"
+
+
+def _distribution_status(distribution_name: str) -> str:
+    try:
+        installed_version = metadata.version(distribution_name)
+    except metadata.PackageNotFoundError:
+        return "not installed"
+    return f"available ({installed_version})"
+
+
+def _installation_status() -> str:
+    try:
+        distribution = metadata.distribution("nexolith")
+    except metadata.PackageNotFoundError:
+        return "package metadata unavailable"
+    direct_url = distribution.read_text("direct_url.json")
+    if direct_url is None:
+        return "installed package"
+    try:
+        direct_url_data = json.loads(direct_url)
+    except (json.JSONDecodeError, TypeError):
+        return "installed package"
+    if not isinstance(direct_url_data, dict):
+        return "installed package"
+    directory_info = direct_url_data.get("dir_info")
+    if isinstance(directory_info, dict) and directory_info.get("editable") is True:
+        return "editable package"
+    return "installed package"
+
+
+def collect_environment_diagnostics() -> EnvironmentDiagnostics:
+    """Collect bounded diagnostics without reading environment values or reporting local paths."""
+
+    dependencies = tuple(
+        DiagnosticItem(label, _distribution_status(distribution_name))
+        for label, distribution_name in _CORE_DISTRIBUTIONS
+    )
+    features = (
+        DiagnosticItem("SQLite", f"available (SQLite {sqlite3.sqlite_version})"),
+        DiagnosticItem("PostgreSQL", _distribution_status("psycopg")),
+    )
+    return EnvironmentDiagnostics(
+        nexolith_version=__version__,
+        python_version=_known(platform.python_version()),
+        python_implementation=_known(platform.python_implementation()),
+        operating_system=_known(platform.system()),
+        operating_system_release=_known(platform.release()),
+        machine=_known(platform.machine()),
+        installation=_installation_status(),
+        dependencies=dependencies,
+        features=features,
+    )
+
+
+def render_environment_diagnostics(report: EnvironmentDiagnostics) -> str:
+    """Render diagnostics as deterministic plain text suitable for an issue report."""
+
+    lines = [
+        "Nexolith environment diagnostics",
+        "================================",
+        f"Nexolith: {report.nexolith_version}",
+        f"Python: {report.python_version} ({report.python_implementation})",
+        (
+            "Platform: "
+            f"{report.operating_system} {report.operating_system_release} ({report.machine})"
+        ),
+        f"Installation: {report.installation}",
+        "",
+        "Core dependencies:",
+    ]
+    lines.extend(f"  {item.name}: {item.value}" for item in report.dependencies)
+    lines.extend(("", "Optional features:"))
+    lines.extend(f"  {item.name}: {item.value}" for item in report.features)
+    lines.extend(
+        (
+            "",
+            "Privacy: environment variables, usernames, hostnames, and filesystem paths "
+            "are not reported.",
+        )
+    )
+    return "\n".join(lines)

nexolith/config/README.md

@@ -0,0 +1,9 @@
+# Configuration
+
+This package defines Pydantic pipeline models and loads YAML with `${VARIABLE_NAME}` substitution.
+Unknown fields and missing environment variables must produce concise errors without exposing
+environment contents.
+
+Configuration failures raise `ConfigurationError`; parser, validation, and file I/O causes are
+chained internally while public messages omit raw input. Add a discriminated model when
+introducing a new built-in component type.

nexolith/config/__init__.py

@@ -0,0 +1,4 @@
+from nexolith.config.loader import load_pipeline
+from nexolith.config.models import PipelineConfig
+
+__all__ = ["PipelineConfig", "load_pipeline"]

nexolith/config/loader.py

@@ -0,0 +1,67 @@
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import ValidationError
+
+from nexolith.config.models import PipelineConfig
+from nexolith.exceptions import ConfigurationError
+
+ENV_PATTERN = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
+
+
+def _resolve_string(value: str) -> str:
+    missing: set[str] = set()
+
+    def replace(match: re.Match[str]) -> str:
+        name = match.group(1)
+        resolved = os.getenv(name)
+        if resolved is None:
+            missing.add(name)
+            return match.group(0)
+        return resolved
+
+    result = ENV_PATTERN.sub(replace, value)
+    if missing:
+        names = ", ".join(sorted(missing))
+        raise ConfigurationError(f"Missing environment variable(s): {names}")
+    return result
+
+
+def _resolve_environment(value: Any) -> Any:
+    if isinstance(value, str):
+        return _resolve_string(value)
+    if isinstance(value, list):
+        return [_resolve_environment(item) for item in value]
+    if isinstance(value, dict):
+        return {key: _resolve_environment(item) for key, item in value.items()}
+    return value
+
+
+def load_pipeline(path: Path) -> PipelineConfig:
+    if not path.is_file():
+        raise ConfigurationError(f"Pipeline file not found: {path}. Check the path and try again.")
+    try:
+        content = path.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise ConfigurationError(
+            f"Could not read pipeline file: {path}. Check file permissions."
+        ) from exc
+    try:
+        document = yaml.safe_load(content)
+    except yaml.YAMLError as exc:
+        mark = getattr(exc, "problem_mark", None)
+        location = f" at line {mark.line + 1}, column {mark.column + 1}" if mark else ""
+        raise ConfigurationError(f"Invalid YAML in {path}{location}.") from exc
+    if not isinstance(document, dict):
+        raise ConfigurationError("Pipeline YAML must contain a mapping at its root")
+    try:
+        return PipelineConfig.validate_document(_resolve_environment(document))
+    except ValidationError as exc:
+        errors = []
+        for error in exc.errors(include_url=False):
+            location = ".".join(str(part) for part in error["loc"])
+            errors.append(f"{location}: {error['msg']}")
+        raise ConfigurationError("Invalid pipeline configuration:\n" + "\n".join(errors)) from exc

nexolith/config/models.py

@@ -0,0 +1,97 @@
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from nexolith.types import Scalar
+
+
+class ComponentConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    type: str
+
+
+class CsvSourceConfig(ComponentConfig):
+    type: Literal["csv"]
+    path: str
+    encoding: str = "utf-8"
+
+
+class SqlSourceConfig(ComponentConfig):
+    type: Literal["sqlite", "postgresql"]
+    connection_url: str
+    query: str | None = None
+    table: str | None = None
+
+    @model_validator(mode="after")
+    def require_query_or_table(self) -> "SqlSourceConfig":
+        if not self.query and not self.table:
+            raise ValueError("either 'query' or 'table' is required")
+        return self
+
+
+class CsvDestinationConfig(ComponentConfig):
+    type: Literal["csv"]
+    path: str
+    encoding: str = "utf-8"
+
+
+class SqlDestinationConfig(ComponentConfig):
+    type: Literal["sqlite", "postgresql"]
+    connection_url: str
+    table: str
+    mode: Literal["append", "replace", "fail"] = "fail"
+
+
+class SelectConfig(ComponentConfig):
+    type: Literal["select"]
+    columns: list[str] = Field(min_length=1)
+
+
+class RenameConfig(ComponentConfig):
+    type: Literal["rename"]
+    columns: dict[str, str] = Field(min_length=1)
+
+
+class DropNullsConfig(ComponentConfig):
+    type: Literal["drop_nulls"]
+    columns: list[str] | None = None
+
+
+class FilterConfig(ComponentConfig):
+    type: Literal["filter"]
+    column: str
+    operator: Literal[
+        "equals",
+        "not_equals",
+        "greater_than",
+        "greater_than_or_equal",
+        "less_than",
+        "less_than_or_equal",
+        "contains",
+        "is_null",
+        "is_not_null",
+    ]
+    value: Scalar = None
+
+    @model_validator(mode="after")
+    def require_value(self) -> "FilterConfig":
+        if self.operator not in {"is_null", "is_not_null"} and self.value is None:
+            raise ValueError(f"'value' is required for operator '{self.operator}'")
+        return self
+
+
+SourceConfig = CsvSourceConfig | SqlSourceConfig
+DestinationConfig = CsvDestinationConfig | SqlDestinationConfig
+TransformationConfig = SelectConfig | RenameConfig | DropNullsConfig | FilterConfig
+
+
+class PipelineConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    name: str = Field(min_length=1)
+    source: SourceConfig = Field(discriminator="type")
+    transformations: list[TransformationConfig] = Field(default_factory=list)
+    destination: DestinationConfig = Field(discriminator="type")
+
+    @classmethod
+    def validate_document(cls, document: Any) -> "PipelineConfig":
+        return cls.model_validate(document)

nexolith/connectors/README.md

@@ -0,0 +1,9 @@
+# Connectors
+
+Connectors translate external data to and from Nexolith's `Rows` representation. CSV and SQL
+implementations are constructed through the connector registry.
+
+New connectors should implement the source or destination protocol, wrap expected I/O or driver
+failures in `ConnectorError` with exception chaining, and close resources deterministically.
+Public errors and logs must never include usernames, credentials, complete connection URLs, or raw
+driver diagnostics. Real PostgreSQL coverage lives in `tests/integration/test_postgresql.py`.

nexolith/connectors/__init__.py

@@ -0,0 +1,3 @@
+from nexolith.connectors.registry import ConnectorRegistry, default_connector_registry
+
+__all__ = ["ConnectorRegistry", "default_connector_registry"]

nexolith/connectors/base.py

@@ -0,0 +1,11 @@
+from typing import Protocol
+
+from nexolith.types import Rows
+
+
+class SourceConnector(Protocol):
+    def read(self) -> Rows: ...
+
+
+class DestinationConnector(Protocol):
+    def write(self, rows: Rows) -> int: ...

nexolith/connectors/csv.py

@@ -0,0 +1,42 @@
+import csv
+from pathlib import Path
+
+from nexolith.exceptions import ConnectorError
+from nexolith.types import Rows
+
+
+class CsvSource:
+    def __init__(self, path: str, encoding: str = "utf-8") -> None:
+        self.path = Path(path)
+        self.encoding = encoding
+
+    def read(self) -> Rows:
+        try:
+            with self.path.open(newline="", encoding=self.encoding) as handle:
+                return [dict(row) for row in csv.DictReader(handle)]
+        except (OSError, csv.Error) as exc:
+            raise ConnectorError(
+                f"Could not read CSV file '{self.path}'. Check the path and permissions."
+            ) from exc
+
+
+class CsvDestination:
+    def __init__(self, path: str, encoding: str = "utf-8") -> None:
+        self.path = Path(path)
+        self.encoding = encoding
+
+    def write(self, rows: Rows) -> int:
+        if not rows:
+            raise ConnectorError("Cannot write an empty dataset to CSV")
+        try:
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            columns = list(rows[0])
+            with self.path.open("w", newline="", encoding=self.encoding) as handle:
+                writer = csv.DictWriter(handle, fieldnames=columns)
+                writer.writeheader()
+                writer.writerows(rows)
+        except (OSError, csv.Error) as exc:
+            raise ConnectorError(
+                f"Could not write CSV file '{self.path}'. Check the path and permissions."
+            ) from exc
+        return len(rows)

nexolith/connectors/registry.py

@@ -0,0 +1,69 @@
+from collections.abc import Callable
+
+from nexolith.config.models import (
+    CsvDestinationConfig,
+    CsvSourceConfig,
+    DestinationConfig,
+    SourceConfig,
+    SqlDestinationConfig,
+    SqlSourceConfig,
+)
+from nexolith.connectors.base import DestinationConnector, SourceConnector
+from nexolith.connectors.csv import CsvDestination, CsvSource
+from nexolith.connectors.sql import SqlDestination, SqlSource
+from nexolith.exceptions import ConnectorError
+
+SourceFactory = Callable[[SourceConfig], SourceConnector]
+DestinationFactory = Callable[[DestinationConfig], DestinationConnector]
+
+
+class ConnectorRegistry:
+    def __init__(self) -> None:
+        self._sources: dict[str, SourceFactory] = {}
+        self._destinations: dict[str, DestinationFactory] = {}
+
+    def register_source(self, name: str, factory: SourceFactory) -> None:
+        self._sources[name] = factory
+
+    def register_destination(self, name: str, factory: DestinationFactory) -> None:
+        self._destinations[name] = factory
+
+    def create_source(self, config: SourceConfig) -> SourceConnector:
+        try:
+            return self._sources[config.type](config)
+        except KeyError as exc:
+            raise ConnectorError(f"Unknown source connector: {config.type}") from exc
+
+    def create_destination(self, config: DestinationConfig) -> DestinationConnector:
+        try:
+            return self._destinations[config.type](config)
+        except KeyError as exc:
+            raise ConnectorError(f"Unknown destination connector: {config.type}") from exc
+
+
+def default_connector_registry() -> ConnectorRegistry:
+    registry = ConnectorRegistry()
+
+    def csv_source(config: SourceConfig) -> SourceConnector:
+        assert isinstance(config, CsvSourceConfig)
+        return CsvSource(config.path, config.encoding)
+
+    def sql_source(config: SourceConfig) -> SourceConnector:
+        assert isinstance(config, SqlSourceConfig)
+        return SqlSource(config.connection_url, config.query, config.table)
+
+    def csv_destination(config: DestinationConfig) -> DestinationConnector:
+        assert isinstance(config, CsvDestinationConfig)
+        return CsvDestination(config.path, config.encoding)
+
+    def sql_destination(config: DestinationConfig) -> DestinationConnector:
+        assert isinstance(config, SqlDestinationConfig)
+        return SqlDestination(config.connection_url, config.table, config.mode)
+
+    registry.register_source("csv", csv_source)
+    registry.register_source("sqlite", sql_source)
+    registry.register_source("postgresql", sql_source)
+    registry.register_destination("csv", csv_destination)
+    registry.register_destination("sqlite", sql_destination)
+    registry.register_destination("postgresql", sql_destination)
+    return registry

nexolith/connectors/sql.py

@@ -0,0 +1,90 @@
+from typing import Any
+
+from sqlalchemy import MetaData, Table, create_engine, inspect, text
+from sqlalchemy.engine import Engine
+from sqlalchemy.exc import SQLAlchemyError
+
+from nexolith.exceptions import ConnectorError
+from nexolith.types import Rows
+
+
+class SqlSource:
+    def __init__(self, connection_url: str, query: str | None, table: str | None) -> None:
+        self.engine = _create_sql_engine(connection_url)
+        self.query = query
+        self.table = table
+
+    def read(self) -> Rows:
+        statement = self.query or f'SELECT * FROM "{self.table}"'
+        try:
+            with self.engine.connect() as connection:
+                return [dict(row) for row in connection.execute(text(statement)).mappings()]
+        except SQLAlchemyError as exc:
+            raise ConnectorError(
+                "Could not read from SQL source. Check the connection, table, or query."
+            ) from exc
+        finally:
+            self.engine.dispose()
+
+
+class SqlDestination:
+    def __init__(self, connection_url: str, table: str, mode: str) -> None:
+        self.engine = _create_sql_engine(connection_url)
+        self.table = table
+        self.mode = mode
+
+    def _prepare_table(self, engine: Engine, rows: Rows) -> Table:
+        metadata = MetaData()
+        exists = inspect(engine).has_table(self.table)
+        if exists and self.mode == "fail":
+            raise ConnectorError(f"Destination table '{self.table}' already exists")
+        if exists and self.mode == "replace":
+            Table(self.table, metadata, autoload_with=engine).drop(engine)
+            metadata.clear()
+            exists = False
+        if not exists:
+            from sqlalchemy import Column, Float, Integer, String
+
+            def column_type(value: Any) -> Any:
+                if isinstance(value, bool):
+                    return Integer()
+                if isinstance(value, int):
+                    return Integer()
+                if isinstance(value, float):
+                    return Float()
+                return String()
+
+            table = Table(
+                self.table,
+                metadata,
+                *(Column(name, column_type(value)) for name, value in rows[0].items()),
+            )
+            metadata.create_all(engine)
+            return table
+        return Table(self.table, metadata, autoload_with=engine)
+
+    def write(self, rows: Rows) -> int:
+        if not rows:
+            raise ConnectorError("Cannot write an empty dataset to SQL")
+        try:
+            table = self._prepare_table(self.engine, rows)
+            with self.engine.begin() as connection:
+                connection.execute(table.insert(), rows)
+            return len(rows)
+        except ConnectorError:
+            raise
+        except SQLAlchemyError as exc:
+            raise ConnectorError(
+                "Could not write to SQL destination. Check the connection, table, and write mode."
+            ) from exc
+        finally:
+            self.engine.dispose()
+
+
+def _create_sql_engine(connection_url: str) -> Engine:
+    try:
+        return create_engine(connection_url)
+    except (ImportError, SQLAlchemyError) as exc:
+        raise ConnectorError(
+            "Could not configure SQL connector. Check the connection URL and database driver."
+        ) from exc