aptdata 0.0.2__py3-none-any.whl

aptdata/__init__.py

@@ -0,0 +1,3 @@
+"""aptdata: A framework for smart data pipelines."""
+
+__version__ = "0.0.2"

aptdata/cli/__init__.py

@@ -0,0 +1,5 @@
+"""Static CLI entry-point for aptdata (Typer-based)."""
+
+from aptdata.cli.app import app
+
+__all__ = ["app"]

aptdata/cli/app.py

@@ -0,0 +1,247 @@
+"""Typer-based static CLI for aptdata.
+
+Design goals
+------------
+* Machine / AI-readable: every outcome is emitted as a single JSON line on
+  stdout (success) or stderr (error).
+* Exit codes: 0 = success, 1 = error.
+* Self-documenting: Typer generates --help automatically from the docstrings
+  and type annotations.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import time
+from pathlib import Path
+
+import typer
+from opentelemetry import trace
+
+from aptdata.cli.scaffold import scaffold
+from aptdata.config.schema import write_domain_schema
+
+app = typer.Typer(
+    name="aptdata",
+    help="Smart Data – declarative data-pipeline framework.",
+    add_completion=False,
+)
+schema_app = typer.Typer(help="Schema utilities for declarative configuration.")
+
+
+def _emit(payload: dict, *, error: bool = False) -> None:
+    """Emit *payload* as a single JSON line to stdout or stderr."""
+    event = dict(payload)
+    span_context = trace.get_current_span().get_span_context()
+    event["trace_id"] = (
+        f"{span_context.trace_id:032x}" if span_context.is_valid else None
+    )
+    line = json.dumps(event, default=str)
+    if error:
+        print(line, file=sys.stderr, flush=True)
+    else:
+        print(line, flush=True)
+
+
+@app.command()
+def run(
+    pipeline: str = typer.Argument(..., help="Pipeline name / identifier to run."),
+    env: str = typer.Option("dev", "--env", "-e", help="Target execution environment."),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        help="Validate and compile the pipeline without executing it.",
+    ),
+) -> None:
+    """Run a registered data pipeline.
+
+    Emits structured JSON logs and returns exit code 0 on success or 1 on
+    failure so that orchestrators and AI agents can parse the outcome.
+
+    Examples
+    --------
+    aptdata run pipeline_x --env prod
+    aptdata run pipeline_x --env staging --dry-run
+    """
+    started_at = time.time()
+    _emit(
+        {
+            "event": "pipeline.started",
+            "pipeline": pipeline,
+            "env": env,
+            "dry_run": dry_run,
+        }
+    )
+
+    try:
+        # Plugin registry look-up (stub – real implementations are in plugins/)
+        from aptdata.plugins import registry  # noqa: PLC0415
+
+        pipeline_cls = registry.get(pipeline)
+        if pipeline_cls is None:
+            raise LookupError(f"Pipeline '{pipeline}' not found in registry.")
+
+        instance = pipeline_cls(system_id=pipeline)
+
+        if not dry_run:
+            instance.run()
+
+        elapsed = round(time.time() - started_at, 3)
+        _emit(
+            {
+                "event": "pipeline.completed",
+                "pipeline": pipeline,
+                "env": env,
+                "dry_run": dry_run,
+                "elapsed_seconds": elapsed,
+            }
+        )
+        raise SystemExit(0)
+
+    except LookupError as exc:
+        elapsed = round(time.time() - started_at, 3)
+        _emit(
+            {
+                "event": "pipeline.error",
+                "pipeline": pipeline,
+                "env": env,
+                "error": str(exc),
+                "elapsed_seconds": elapsed,
+            },
+            error=True,
+        )
+        raise SystemExit(1) from exc
+
+    except Exception as exc:  # noqa: BLE001
+        elapsed = round(time.time() - started_at, 3)
+        _emit(
+            {
+                "event": "pipeline.error",
+                "pipeline": pipeline,
+                "env": env,
+                "error": str(exc),
+                "elapsed_seconds": elapsed,
+            },
+            error=True,
+        )
+        raise SystemExit(1) from exc
+
+
+@app.command()
+def monitor(
+    refresh: float = typer.Option(
+        1.0,
+        "--refresh",
+        "-r",
+        help="Dashboard refresh interval in seconds.",
+    ),
+) -> None:
+    """Open the interactive TUI monitoring dashboard.
+
+    Displays the pipeline DAG, memory usage and task status in real time.
+    Press **q** or **Ctrl+C** to exit.
+
+    Examples
+    --------
+    aptdata monitor
+    aptdata monitor --refresh 0.5
+    """
+    from aptdata.tui.monitor import MonitorApp  # noqa: PLC0415
+
+    app_instance = MonitorApp(refresh_interval=refresh)
+    app_instance.run()
+
+
+@app.command()
+def mcp_start(
+    transport: str = typer.Option(
+        "stdio",
+        "--transport",
+        "-t",
+        help="MCP transport to use (stdio or sse).",
+    ),
+) -> None:
+    """Start the MCP (Model Context Protocol) server.
+
+    This exposes aptdata tools and resources so that AI agents
+    (Claude Desktop, Copilot, Devin, …) can discover and run pipelines.
+
+    Examples
+    --------
+    aptdata mcp-start
+    aptdata mcp-start --transport sse
+    """
+    _emit({"event": "mcp.server.starting", "transport": transport})
+    try:
+        from aptdata.mcp.server import mcp as mcp_server  # noqa: PLC0415
+
+        mcp_server.run(transport=transport)
+    except Exception as exc:  # noqa: BLE001
+        _emit(
+            {"event": "mcp.server.error", "error": str(exc)},
+            error=True,
+        )
+        raise SystemExit(1) from exc
+
+
+app.command()(scaffold)
+app.add_typer(schema_app, name="schema")
+
+from aptdata.cli.commands import (  # noqa: E402
+    config_app,
+    mesh_app,
+    plugin_app,
+    system_app,
+    telemetry_app,
+)
+from aptdata.cli.interactive import interactive_command  # noqa: E402
+
+app.add_typer(system_app, name="system")
+app.add_typer(plugin_app, name="plugin")
+app.add_typer(config_app, name="config")
+app.add_typer(telemetry_app, name="telemetry")
+app.add_typer(mesh_app, name="mesh")
+
+
+@app.command("interactive")
+def interactive() -> None:
+    """Launch the interactive wizard mode."""
+    interactive_command()
+
+
+@schema_app.command("export")
+def schema_export(
+    output: Path = typer.Option(
+        ...,
+        "--output",
+        "-o",
+        help="Output path for the generated JSON Schema.",
+    ),
+) -> None:
+    """Export JSON Schema for declarative YAML configs."""
+    started_at = time.time()
+    _emit({"event": "schema.export.started", "output": str(output)})
+    try:
+        write_domain_schema(output)
+        elapsed = round(time.time() - started_at, 3)
+        _emit(
+            {
+                "event": "schema.export.completed",
+                "output": str(output),
+                "elapsed_seconds": elapsed,
+            }
+        )
+        raise SystemExit(0)
+    except Exception as exc:  # noqa: BLE001
+        elapsed = round(time.time() - started_at, 3)
+        _emit(
+            {
+                "event": "schema.export.error",
+                "output": str(output),
+                "error": str(exc),
+                "elapsed_seconds": elapsed,
+            },
+            error=True,
+        )
+        raise SystemExit(1) from exc

aptdata/cli/commands/__init__.py

@@ -0,0 +1,9 @@
+"""CLI sub-command packages for aptdata."""
+
+from aptdata.cli.commands.config_cmd import config_app
+from aptdata.cli.commands.mesh_cmd import mesh_app
+from aptdata.cli.commands.plugin_cmd import plugin_app
+from aptdata.cli.commands.system_cmd import system_app
+from aptdata.cli.commands.telemetry_cmd import telemetry_app
+
+__all__ = ["system_app", "plugin_app", "config_app", "telemetry_app", "mesh_app"]

aptdata/cli/commands/config_cmd.py

@@ -0,0 +1,128 @@
+"""CLI sub-commands for YAML configuration management."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+
+from aptdata.cli.rendering.console import SmartConsole
+from aptdata.cli.rendering.panels import yaml_preview
+from aptdata.cli.rendering.tables import config_summary_table
+
+config_app = typer.Typer(name="config", help="Manage declarative YAML configurations.")
+
+_STARTER_YAML = """\
+# aptdata declarative pipeline configuration
+metadata:
+  name: my_pipeline
+  version: "1.0"
+  description: A sample pipeline
+
+system:
+  system_id: my_pipeline
+  flows:
+    - flow_id: main_flow
+      components:
+        - component_id: extract
+          metadata:
+            kind: EXTRACT
+            description: Extract raw data
+        - component_id: transform
+          metadata:
+            kind: TRANSFORM
+            description: Transform data
+        - component_id: load
+          metadata:
+            kind: LOAD
+            description: Load to destination
+      edges:
+        - source_id: extract
+          target_id: transform
+        - source_id: transform
+          target_id: load
+"""
+
+
+@config_app.command("validate")
+def config_validate(
+    path: Path = typer.Argument(..., help="Path to YAML config file.", exists=True),
+) -> None:
+    """Parse and validate a YAML config file."""
+    from aptdata.config.parser import YamlConfigParser  # noqa: PLC0415
+
+    console = SmartConsole(json_mode=False)
+
+    try:
+        with console.spinner(f"Validating '{path}'..."):
+            parser = YamlConfigParser()
+            parsed = parser.parse_file(path)
+        console.success(f"Config valid: system_id='{parsed.system.system_id}'")
+    except Exception as exc:  # noqa: BLE001
+        console.error(f"Validation failed: {exc}")
+        raise typer.Exit(1) from exc
+
+
+@config_app.command("init")
+def config_init(
+    output: Path = typer.Option(
+        Path("pipeline.yaml"),
+        "--output",
+        "-o",
+        help="Output file path.",
+    ),
+    template: bool = typer.Option(
+        False, "--template", help="Use the starter template."
+    ),
+) -> None:
+    """Generate a starter YAML configuration file."""
+    console = SmartConsole(json_mode=False)
+
+    if output.exists():
+        console.error(f"File '{output}' already exists.")
+        raise typer.Exit(1)
+
+    output.write_text(_STARTER_YAML, encoding="utf-8")
+    console.success(f"Config template written to '{output}'.")
+    console.render(yaml_preview(_STARTER_YAML))
+
+
+@config_app.command("show")
+def config_show(
+    path: Path = typer.Argument(..., help="Path to YAML config file.", exists=True),
+) -> None:
+    """Pretty-print a YAML config file with syntax highlighting."""
+    console = SmartConsole(json_mode=False)
+    content = path.read_text(encoding="utf-8")
+    console.render(yaml_preview(content))
+
+
+@config_app.command("run")
+def config_run(
+    path: Path = typer.Argument(..., help="Path to YAML config file.", exists=True),
+    env: str = typer.Option("dev", "--env", "-e", help="Target environment."),
+) -> None:
+    """Parse a YAML config, register the system, and execute it."""
+    from aptdata.config.parser import YamlConfigParser  # noqa: PLC0415
+    from aptdata.plugins import registry  # noqa: PLC0415
+
+    console = SmartConsole(json_mode=False)
+
+    try:
+        with console.spinner(f"Parsing '{path}'..."):
+            parser = YamlConfigParser()
+            parsed = parser.parse_file(path)
+
+        system_id = parsed.system.system_id
+        system_cls = type(parsed.system)
+        registry.register(system_id, system_cls)
+
+        console.render(config_summary_table(parsed))
+
+        with console.spinner(f"Running '{system_id}' [{env}]..."):
+            parsed.system.run()
+
+        console.success(f"System '{system_id}' executed successfully.")
+    except Exception as exc:  # noqa: BLE001
+        console.error(f"Execution failed: {exc}")
+        raise typer.Exit(1) from exc