phlo-sling 0.1.0__tar.gz

phlo_sling-0.1.0/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.4
+Name: phlo-sling
+Version: 0.1.0
+Summary: Sling replication ingestion provider for Phlo
+Author-email: Phlo Team <team@phlo.dev>
+License: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/plain
+Requires-Dist: phlo>=0.1.0
+Requires-Dist: sling>=1.0.0
+Requires-Dist: pyarrow>=21.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+
+Sling-based database replication ingestion provider for Phlo.

phlo_sling-0.1.0/README.md

@@ -0,0 +1,111 @@
+# phlo-sling
+
+Sling-based database replication ingestion provider for Phlo.
+
+## Overview
+
+`phlo-sling` wraps [Sling](https://slingdata.io/) as a complementary ingestion engine alongside `phlo-dlt`. Sling is a data movement CLI (DB→DB, File→DB, DB→File) with 30+ connectors, optimized for high-speed database replication.
+
+Where DLT excels at API-based ingestion with schema evolution and normalisation, Sling excels at raw-speed database replication with wildcard stream selection (`my_schema.*`), incremental modes, and direct DB-to-DB transfers.
+
+## Installation
+
+```bash
+pip install phlo-sling
+```
+
+## Usage
+
+### Decorator-Based Replication
+
+```python
+import phlo
+from phlo_sling import phlo_sling_replication
+
+
+@phlo_sling_replication(
+    stream_name="public.users",
+    table_name="users",
+    source_conn="PHLO_POSTGRES",
+    target_conn="WAREHOUSE",
+    group="crm",
+    mode="incremental",
+    primary_key="id",
+    update_key="updated_at",
+    cron="0 */2 * * *",
+    owner="data-team",
+)
+def replicate_users(context):
+    """Replicate users table from Postgres into raw.users on WAREHOUSE."""
+    return None
+```
+
+### Python-First File Discovery
+
+Use `phlo_sling_assets` when you want Python logic to discover folders/files
+first and register one Sling-backed asset per result.
+
+```python
+from pathlib import Path
+
+from phlo_sling import SlingReplication, phlo_sling_assets
+
+
+@phlo_sling_assets(group="finance")
+def discover_workbooks():
+    root = Path("/mnt/finance")
+
+    for workbook in root.rglob("*.xlsx"):
+        table_name = workbook.stem.replace("-", "_").lower()
+        yield SlingReplication(
+            stream_name=f"file://{workbook}",
+            table_name=table_name,
+            source_conn="LOCAL",
+            target_conn="WAREHOUSE",
+            object=f"raw.{table_name}",
+            mode="full-refresh",
+            source_options={"sheet": "Sheet1!A:F"},
+            description=f"Ingest workbook {workbook.name}",
+            metadata={"workbook_path": str(workbook)},
+            tags={"format": "xlsx"},
+        )
+```
+
+Use the original `phlo_sling_replication` decorator when you want one stable
+asset whose function may return runtime Sling overrides such as a dynamic
+`src_stream` or `where` clause.
+
+### CLI Commands
+
+```bash
+# Run replication from YAML
+phlo sling run --replication replications/pg_to_lake.yaml
+
+# Run ad-hoc replication
+phlo sling run --source PHLO_POSTGRES --stream public.users --target PHLO_S3 --object raw/users.parquet
+
+# Override the inferred destination object when needed
+phlo sling run --source PHLO_POSTGRES --stream public.users --target WAREHOUSE --object raw.users
+
+# List connections
+phlo sling conns
+
+# Discover available streams
+phlo sling discover PHLO_POSTGRES
+phlo sling discover PHLO_POSTGRES --schema public --format json
+```
+
+## Configuration
+
+The following environment variables can be used to configure Sling:
+
+- `SLING_DEFAULT_NAMESPACE` - Default namespace for generated replication table names (default: "raw")
+- `SLING_DEFAULT_MODE` - Default replication mode (default: "incremental")
+- `SLING_AUTO_CONNECTIONS` - Auto-generate Sling connections from Phlo capability metadata (default: true)
+- `PHLO_OBJECT_STORE` - Select the active `object_store` capability when more than one is installed
+
+Notes:
+
+- Decorator-backed replications need a real Sling destination. When `target_conn` is set and `object` is omitted, `phlo-sling` targets `<namespace>.<table_name>` automatically.
+- If you set `SLING_AUTO_CONNECTIONS=false`, `phlo-sling` stops injecting `PHLO_POSTGRES` / `PHLO_S3` connection definitions into the environment.
+- `PHLO_S3` now resolves from the active `object_store` capability instead of importing `phlo-minio` / `phlo-rustfs` directly. If both are installed, set `PHLO_OBJECT_STORE=minio` or `PHLO_OBJECT_STORE=rustfs`.

phlo_sling-0.1.0/pyproject.toml

@@ -0,0 +1,44 @@
+[build-system]
+requires = ["setuptools>=45", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "phlo-sling"
+version = "0.1.0"
+description = "Sling replication ingestion provider for Phlo"
+readme = {text = "Sling-based database replication ingestion provider for Phlo.", content-type = "text/plain"}
+requires-python = ">=3.11"
+authors = [
+    {name = "Phlo Team", email = "team@phlo.dev"},
+]
+license = {text = "MIT"}
+dependencies = [
+    "phlo>=0.1.0",
+    "sling>=1.0.0",
+    "pyarrow>=21.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "ruff>=0.1.0",
+]
+
+[project.entry-points."phlo.plugins.assets"]
+sling = "phlo_sling.plugin:SlingAssetProvider"
+
+[project.entry-points."phlo.plugins.ingestion_providers"]
+sling = "phlo_sling.plugin:SlingIngestionProvider"
+
+[project.entry-points."phlo.plugins.cli"]
+sling = "phlo_sling.cli_plugin:SlingCliPlugin"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"

phlo_sling-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

phlo_sling-0.1.0/src/phlo_sling/__init__.py

@@ -0,0 +1,28 @@
+from collections.abc import Callable
+from typing import Any
+
+from phlo_sling.registry import SlingReplication
+
+
+def phlo_sling_assets(*args: Any, **kwargs: Any) -> Callable[..., Any]:
+    """Lazily resolve and forward to the Sling asset discovery decorator."""
+    from phlo_sling.decorator import phlo_sling_assets as _phlo_sling_assets
+
+    return _phlo_sling_assets(*args, **kwargs)
+
+
+def phlo_sling_replication(*args: Any, **kwargs: Any) -> Callable[..., Any]:
+    """Lazily resolve and forward to the sling replication decorator factory."""
+    from phlo_sling.decorator import phlo_sling_replication as _phlo_sling_replication
+
+    return _phlo_sling_replication(*args, **kwargs)
+
+
+def get_sling_assets() -> list[Any]:
+    """Lazily resolve and return registered sling replication assets."""
+    from phlo_sling.decorator import get_sling_assets as _get_sling_assets
+
+    return _get_sling_assets()
+
+
+__all__ = ["SlingReplication", "get_sling_assets", "phlo_sling_assets", "phlo_sling_replication"]

phlo_sling-0.1.0/src/phlo_sling/cli_commands.py

@@ -0,0 +1,187 @@
+"""CLI commands for Sling replication management."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+
+import click
+
+from phlo.logging import get_logger
+from phlo_sling.connections import apply_sling_connection_env
+from phlo_sling.settings import get_settings
+
+logger = get_logger(__name__)
+
+
+@click.group("sling")
+def sling_group() -> None:
+    """Sling replication commands."""
+
+
+@sling_group.command("run")
+@click.option("--replication", "-r", type=click.Path(exists=True), help="Sling replication YAML.")
+@click.option("--source", "-s", help="Source connection name.")
+@click.option("--target", "-t", help="Target connection name.")
+@click.option("--stream", help="Source stream (e.g., 'public.users').")
+@click.option("--object", "target_object", help="Target object/table name.")
+@click.option(
+    "--mode",
+    default=None,
+    help="Replication mode. Defaults to SLING_DEFAULT_MODE when omitted.",
+)
+def run_command(
+    replication: str | None,
+    source: str | None,
+    target: str | None,
+    stream: str | None,
+    target_object: str | None,
+    mode: str | None,
+) -> None:
+    """Run a Sling replication.
+
+    Either provide --replication YAML or --source/--stream/--target for ad-hoc runs.
+    """
+    from sling import Replication, Sling
+
+    apply_sling_connection_env()
+    resolved_mode = mode or get_settings().sling_default_mode
+
+    if replication:
+        click.echo(f"Running replication from {replication}")
+        repl = Replication(file_path=replication)
+        repl.run()
+    elif source and stream:
+        if not target:
+            raise click.UsageError("Provide --target for ad-hoc runs.")
+        resolved_target_object = _resolve_target_object(stream=stream, target_object=target_object)
+        click.echo(f"Replicating {stream} from {source}")
+        config = Sling(
+            src_conn=source,
+            src_stream=stream,
+            tgt_conn=target,
+            tgt_object=resolved_target_object,
+            mode=resolved_mode,
+        )
+        config.run()
+    else:
+        raise click.UsageError("Provide --replication YAML or --source/--stream.")
+
+
+@sling_group.command("conns")
+@click.option("--auto/--no-auto", default=True, help="Include auto-discovered connections.")
+def conns_command(auto: bool) -> None:
+    """List available Sling connections.
+
+    Shows auto-discovered connections from Phlo capability metadata and any
+    connections from explicit env.yaml files.
+    """
+    if auto:
+        from phlo_sling.connections import resolve_phlo_connections
+
+        connections = resolve_phlo_connections()
+        if connections:
+            click.echo("Auto-discovered connections:")
+            for name, config in connections.items():
+                conn_type = config.get("type", "unknown")
+                host = config.get("host") or config.get("endpoint", "")
+                click.echo(f"  {name}: {conn_type} ({host})")
+        else:
+            click.echo("No auto-discovered connections found.")
+
+    click.echo("\nSling native connections:")
+    try:
+        result = _run_sling_cli_command(["conns", "list"])
+        click.echo(result.stdout, nl=False)
+    except Exception as exc:
+        click.echo(f"  Could not list native connections: {exc}")
+
+
+@sling_group.command("discover")
+@click.argument("connection")
+@click.option("--schema", help="Filter by schema name.")
+@click.option(
+    "--format",
+    "output_format",
+    type=click.Choice(["table", "json"]),
+    default="table",
+    show_default=True,
+    help="Output format.",
+)
+def discover_command(connection: str, schema: str | None, output_format: str) -> None:
+    """Discover available streams from a Sling connection.
+
+    Lists tables/views available in the source connection for use as
+    stream_name in @phlo_sling_replication decorators.
+    """
+    apply_sling_connection_env()
+
+    click.echo(f"Discovering streams from {connection}...")
+    try:
+        command = ["conns", "discover", connection]
+        if schema:
+            command.extend(["--pattern", f"{schema}.*"])
+
+        result = _run_sling_cli_command(command)
+        if output_format == "json":
+            click.echo(json.dumps(_parse_discovery_output(result.stdout), indent=2))
+            return
+
+        click.echo(result.stdout, nl=False)
+    except Exception as exc:
+        raise click.ClickException(f"Discovery failed: {exc}") from exc
+
+
+def _resolve_target_object(stream: str, target_object: str | None) -> str:
+    """Resolve the destination object for an ad-hoc Sling run."""
+    if target_object:
+        return target_object
+    if "*" in stream:
+        raise click.UsageError("Provide --object when --stream uses a wildcard.")
+    return stream
+
+
+def _get_sling_binary() -> str:
+    """Return the Sling binary path, honoring package settings."""
+    settings = get_settings()
+    if settings.sling_binary_path:
+        return settings.sling_binary_path
+
+    from sling.bin import SLING_BIN
+
+    return SLING_BIN
+
+
+def _run_sling_cli_command(args: list[str]) -> subprocess.CompletedProcess[str]:
+    """Execute the Sling CLI and return captured output."""
+    return subprocess.run(
+        [_get_sling_binary(), *args],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+
+
+def _parse_discovery_output(output: str) -> list[dict[str, str]]:
+    """Parse Sling's ASCII discovery table into JSON-serializable rows."""
+    lines = [line.rstrip() for line in output.splitlines() if line.strip()]
+    table_lines = [line for line in lines if "|" in line]
+    if len(table_lines) < 2:
+        return []
+
+    headers = [_normalize_column_name(part) for part in table_lines[0].split("|")]
+    rows: list[dict[str, str]] = []
+    for line in table_lines[1:]:
+        values = [part.strip() for part in line.split("|")]
+        if len(values) != len(headers):
+            continue
+        if all(set(value) <= {"-"} for value in values):
+            continue
+        rows.append(dict(zip(headers, values, strict=True)))
+
+    return rows
+
+
+def _normalize_column_name(value: str) -> str:
+    """Normalize discovery table headers for JSON output."""
+    return value.strip().lower().replace(" ", "_")

phlo_sling-0.1.0/src/phlo_sling/cli_plugin.py

@@ -0,0 +1,25 @@
+"""CLI plugin for Sling commands."""
+
+from __future__ import annotations
+
+import click
+
+from phlo.plugins.base import CliCommandPlugin, PluginMetadata
+from phlo_sling.cli_commands import sling_group
+
+
+class SlingCliPlugin(CliCommandPlugin):
+    """Expose Sling CLI command groups to the Phlo plugin system."""
+
+    @property
+    def metadata(self) -> PluginMetadata:
+        """Return plugin metadata for CLI command discovery."""
+        return PluginMetadata(
+            name="sling",
+            version="0.1.0",
+            description="Sling replication CLI commands for Phlo",
+        )
+
+    def get_cli_commands(self) -> list[click.Command]:
+        """Return CLI commands contributed by this plugin."""
+        return [sling_group]

phlo_sling-0.1.0/src/phlo_sling/connections.py

@@ -0,0 +1,186 @@
+"""Auto-generate Sling connections from Phlo capability metadata."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import MutableMapping
+from typing import Any
+
+from phlo.capabilities import list_capabilities, resolve_capability
+from phlo.infrastructure.config import load_project_config
+from phlo.logging import get_logger
+from phlo_sling.settings import get_settings
+
+logger = get_logger(__name__)
+
+
+def resolve_phlo_connections() -> dict[str, dict[str, Any]]:
+    """Build Sling connection definitions from installed Phlo package settings.
+
+    Inspects known Phlo capability providers (phlo-postgres, phlo-minio, etc.)
+    and generates Sling-compatible connection dicts.
+
+    Returns:
+        Dict mapping connection name to Sling connection config.
+    """
+    if not get_settings().sling_auto_connections:
+        logger.debug("sling_auto_connections_disabled")
+        return {}
+
+    connections: dict[str, dict[str, Any]] = {}
+
+    connections.update(_resolve_postgres_connection())
+    connections.update(_resolve_iceberg_connection())
+    connections.update(_resolve_s3_connection())
+
+    return connections
+
+
+def _project_env_value(name: str) -> str | None:
+    """Read a non-secret default from phlo.yaml env: when host os.environ lacks it."""
+    try:
+        project_config = load_project_config()
+    except Exception as exc:
+        logger.debug("project_env_lookup_failed", name=name, error=str(exc))
+        return None
+
+    env_config = project_config.get("env", {})
+    if not isinstance(env_config, dict):
+        return None
+
+    value = env_config.get(name)
+    return value if isinstance(value, str) and value else None
+
+
+def _ensure_capabilities_discovered(*kinds: str) -> None:
+    """Populate the capability registry only when the requested kinds are absent."""
+    if any(list_capabilities(kind) for kind in kinds):
+        return
+
+    from phlo.capabilities.discovery import discover_capabilities
+
+    discover_capabilities()
+
+
+def _get_iceberg_settings():
+    """Import phlo-iceberg settings lazily for optional package installs."""
+    from phlo_iceberg.settings import get_settings as get_iceberg_settings
+
+    return get_iceberg_settings()
+
+
+def _resolve_postgres_connection() -> dict[str, dict[str, Any]]:
+    """Resolve Postgres connection from phlo-postgres settings."""
+    try:
+        from phlo_postgres.settings import get_settings as get_pg_settings
+
+        pg = get_pg_settings()
+        return {
+            "PHLO_POSTGRES": {
+                "type": "postgres",
+                "host": pg.postgres_host,
+                "port": pg.postgres_port,
+                "database": pg.postgres_db,
+                "user": pg.postgres_user,
+                "password": pg.postgres_password,
+                "schema": getattr(pg, "postgres_schema", "public"),
+            }
+        }
+    except (ImportError, Exception) as exc:
+        logger.debug("postgres_connection_skipped", error=str(exc))
+        return {}
+
+
+def _resolve_iceberg_connection() -> dict[str, dict[str, Any]]:
+    """Resolve an Iceberg REST catalog connection from phlo-iceberg settings."""
+    try:
+        settings = _get_iceberg_settings()
+        ref = settings.iceberg_default_ref
+        config = settings.get_pyiceberg_catalog_config(ref)
+        return {
+            "PHLO_ICEBERG": {
+                "type": "iceberg",
+                "catalog_type": "rest",
+                "rest_uri": config["uri"],
+                "rest_warehouse": config["warehouse"],
+                "s3_endpoint": config["s3.endpoint"],
+                "s3_access_key_id": config["s3.access-key-id"],
+                "s3_secret_access_key": config["s3.secret-access-key"],
+                "s3_region": config["s3.region"],
+                "schema": settings.iceberg_default_namespace,
+            }
+        }
+    except (ImportError, Exception) as exc:
+        logger.debug("iceberg_connection_skipped", error=str(exc))
+        return {}
+
+
+def _resolve_s3_connection() -> dict[str, dict[str, Any]]:
+    """Resolve S3 connection from the active object-store capability."""
+    _ensure_capabilities_discovered("object_store")
+    requested_name = os.environ.get("PHLO_OBJECT_STORE") or _project_env_value("PHLO_OBJECT_STORE")
+    resolution = resolve_capability("object_store", requested_name)
+    if resolution is None:
+        available = list_capabilities("object_store")
+        logger.debug(
+            "object_store_connection_skipped",
+            requested_name=requested_name,
+            available=available,
+        )
+        return {}
+
+    provider = resolution.provider
+    if hasattr(provider, "to_sling_connection"):
+        config = provider.to_sling_connection()
+    else:
+        config = {
+            key: value
+            for key, value in resolution.metadata.items()
+            if key in {"type", "endpoint", "access_key_id", "secret_access_key", "region"}
+        }
+
+    if not config:
+        logger.debug(
+            "object_store_connection_missing_config",
+            capability_name=resolution.name,
+        )
+        return {}
+
+    return {"PHLO_S3": config}
+
+
+def export_sling_env(connections: dict[str, dict[str, Any]]) -> dict[str, str]:
+    """Convert connection dicts to Sling environment variable format.
+
+    Sling expects connections as environment variables with JSON values.
+
+    Args:
+        connections: Dict of connection name → connection config.
+
+    Returns:
+        Dict of environment variable name → JSON string value.
+    """
+    import json
+
+    env_vars: dict[str, str] = {}
+    for name, config in connections.items():
+        env_vars[name] = json.dumps(config)
+    return env_vars
+
+
+def apply_sling_connection_env(environ: MutableMapping[str, str] | None = None) -> dict[str, str]:
+    """Inject resolved Sling connections into an environment mapping.
+
+    Existing variables win over auto-generated values.
+
+    Args:
+        environ: Environment mapping to mutate. Defaults to ``os.environ``.
+
+    Returns:
+        Dict of injected environment variables.
+    """
+    target_env = os.environ if environ is None else environ
+    env_vars = export_sling_env(resolve_phlo_connections())
+    for name, value in env_vars.items():
+        target_env.setdefault(name, value)
+    return env_vars