lumilake-cli 0.1.2__tar.gz → 0.1.4__tar.gz

{lumilake_cli-0.1.2/src/lumilake_cli.egg-info → lumilake_cli-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lumilake-cli
-Version: 0.1.2
+Version: 0.1.4
 Summary: Lumilake command-line interface
 Author: Lumilake contributors
 License-Expression: Apache-2.0
@@ -14,12 +14,12 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: <3.13,>=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: lumilake-sdk==0.1.2
+Requires-Dist: lumilake-sdk==0.1.4
 Requires-Dist: flowmesh-sdk==0.1.1
 Requires-Dist: requests>=2.32.3
 Requires-Dist: typer>=0.12.5
 Provides-Extra: deploy
-Requires-Dist: lumilake-deploy==0.1.2; extra == "deploy"
+Requires-Dist: lumilake-deploy==0.1.4; extra == "deploy"
 Dynamic: license-file
 
 # `lumilake-cli`

{lumilake_cli-0.1.2 → lumilake_cli-0.1.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lumilake-cli"
-version = "0.1.2"
+version = "0.1.4"
 description = "Lumilake command-line interface"
 readme = "README.md"
 requires-python = "<3.13,>=3.12"
@@ -10,7 +10,7 @@ authors = [
     {name = "Lumilake contributors"},
 ]
 dependencies = [
-    "lumilake-sdk==0.1.2",
+    "lumilake-sdk==0.1.4",
     # Renders ``lumilake trace`` ProfileSummary payloads.
     "flowmesh-sdk==0.1.1",
     "requests>=2.32.3",
@@ -26,7 +26,7 @@ classifiers = [
 
 [project.optional-dependencies]
 # Thin ``pip install lumilake-cli`` (no extra) skips Docker/FlowMesh.
-deploy = ["lumilake-deploy==0.1.2"]
+deploy = ["lumilake-deploy==0.1.4"]
 
 [project.urls]
 Homepage = "https://github.com/mlsys-io/lumilake_OSS"

{lumilake_cli-0.1.2 → lumilake_cli-0.1.4}/src/lumilake_cli/commands/deploy.py

@@ -340,12 +340,13 @@ def down(
 ) -> None:
     """Stop the stack but keep data volumes.
 
-    Safe to run between sessions: the archive bucket (job records, run
-    artifacts) and the compute postgres/minio volumes survive, so
-    ``deploy up`` resumes against the same state. ``--wipe-archive`` also
-    removes the compute Postgres and FlowMesh runtime-state volumes, while
-    preserving MinIO corpus data. Use ``deploy reset`` (destructive) to
-    wipe every volume instead.
+    Safe to run between sessions: the archive layer (job records, runtime
+    artifacts) lives in lumid-data-app under ``S3_ARCHIVE_PREFIX`` and is
+    not managed by ``lumilake deploy``, so ``deploy up`` resumes against
+    the same lumid-data-app state. ``--wipe-archive`` wipes the FlowMesh
+    runtime-state volumes that ``lumilake deploy`` does own; remote
+    lumid-data-app state is untouched. Use ``deploy reset`` (destructive)
+    to wipe every locally-managed volume instead.
     """
     try:
         stop_mod.run_stop(
@@ -493,11 +494,13 @@ def reset(
         help="Skip the destructive-action confirmation prompt.",
     ),
 ) -> None:
-    """Wipe the archive and every volume, then start the stack fresh.
+    """Wipe every locally-managed volume, then start the stack fresh.
 
-    Destructive: removes the compute postgres / minio volumes (job
-    records, run artifacts, demo data) and the FlowMesh runtime state.
-    Use ``deploy down`` if you want to stop the stack but keep its data.
+    Destructive: removes the FlowMesh runtime-state volumes managed by
+    ``lumilake deploy``. The archive layer (job records, runtime
+    artifacts) lives in lumid-data-app under ``S3_ARCHIVE_PREFIX`` and
+    is **not** wiped by this command. Use ``deploy down`` if you want
+    to stop the stack but keep its local state.
     """
     if not yes and not typer.confirm(
         "deploy reset deletes every Lumilake volume (archive + compute "

{lumilake_cli-0.1.2 → lumilake_cli-0.1.4}/src/lumilake_cli/commands/job.py

@@ -1,11 +1,15 @@
 """Job submission, monitoring, and result retrieval commands."""
 
 import json
+import tarfile
 import time
 from pathlib import Path
 from typing import Any
 
+import requests
 import typer
+from rich.console import Console
+from rich.table import Table
 
 from ..core import logging
 from ..core.http import HttpError, client_from_config
@@ -23,6 +27,55 @@ def _unwrap(response_json: dict[str, Any]) -> dict[str, Any]:
     return response_json.get("data", response_json)
 
 
+_HARDWARE_FIELDS = ("cpu", "memory", "gpu", "gpu_memory")
+
+
+def _build_hardware_payload(
+    cpu: int | None,
+    memory: str | None,
+    gpu: int | None,
+    gpu_memory: str | None,
+    hardware_json: Path | None,
+) -> dict[str, Any] | None:
+    """Merge ``--hardware-json`` with individual ``--cpu`` etc. flags.
+
+    Individual flags win on conflict so users can layer a one-off override on a
+    saved profile. Returns ``None`` when nothing is set so the server keeps the
+    env defaults.
+    """
+    base: dict[str, Any] = {}
+    if hardware_json is not None:
+        if not hardware_json.is_file():
+            logging.error(f"--hardware-json file not found: {hardware_json}")
+            raise typer.Exit(code=1)
+        try:
+            loaded = json.loads(hardware_json.read_text())
+        except json.JSONDecodeError as exc:
+            logging.error(f"--hardware-json is not valid JSON: {exc}")
+            raise typer.Exit(code=1) from exc
+        if not isinstance(loaded, dict):
+            logging.error("--hardware-json must contain a JSON object")
+            raise typer.Exit(code=1)
+        unknown = set(loaded) - set(_HARDWARE_FIELDS)
+        if unknown:
+            logging.error(
+                "--hardware-json has unknown fields "
+                f"{sorted(unknown)}; allowed: {list(_HARDWARE_FIELDS)}"
+            )
+            raise typer.Exit(code=1)
+        base.update(loaded)
+    overrides: dict[str, Any] = {
+        "cpu": cpu,
+        "memory": memory,
+        "gpu": gpu,
+        "gpu_memory": gpu_memory,
+    }
+    for key, value in overrides.items():
+        if value is not None:
+            base[key] = value
+    return base or None
+
+
 def _build_inputs(
     input_values: list[str] | None,
     input_files: list[str] | None,
@@ -118,11 +171,6 @@ def submit(
         "--output-prefix",
         help="S3 output prefix (required when --output-type s3)",
     ),
-    output_connection_string: str | None = typer.Option(
-        None,
-        "--output-connection-string",
-        help="S3 connection string (s3://user:pass@endpoint/bucket)",
-    ),
     output_table: str | None = typer.Option(
         None, "--output-table", help="DB output table (required when --output-type db)"
     ),
@@ -160,6 +208,53 @@ def submit(
             '`{"query": ["NVDA", "TSLA"], "year": ["2024"]}`.'
         ),
     ),
+    optimizer: str | None = typer.Option(
+        None,
+        "--optimizer",
+        help="Override the server default optimizer (must be in /optimizer).",
+    ),
+    cpu: int | None = typer.Option(
+        None,
+        "--cpu",
+        help=(
+            "Per-job CPU cores override. Unset falls back to "
+            "HARDWARE_CPU_REQUIREMENT."
+        ),
+    ),
+    memory: str | None = typer.Option(
+        None,
+        "--memory",
+        help=(
+            "Per-job RAM override (e.g. `32Gi`). Unset falls back to "
+            "HARDWARE_MEMORY_REQUIREMENT."
+        ),
+    ),
+    gpu: int | None = typer.Option(
+        None,
+        "--gpu",
+        help=(
+            "Per-job GPU count override. `0` disables GPU; the server rejects "
+            "workflows with GPU ops when `--gpu 0`. Unset falls back to "
+            "HARDWARE_GPU_REQUIREMENT."
+        ),
+    ),
+    gpu_memory: str | None = typer.Option(
+        None,
+        "--gpu-memory",
+        help=(
+            "Per-job GPU memory override (e.g. `24Gi`). Unset falls back to "
+            "HARDWARE_GPU_MEMORY_REQUIREMENT."
+        ),
+    ),
+    hardware_json: Path | None = typer.Option(
+        None,
+        "--hardware-json",
+        help=(
+            "JSON file with the full hardware override (keys: cpu, memory, gpu, "
+            "gpu_memory). Merged with `--cpu`/`--memory`/`--gpu`/`--gpu-memory`; "
+            "flag values win on conflict."
+        ),
+    ),
 ) -> None:
     """Submit a workflow for optimization and execution."""
     if not workflow.exists():
@@ -184,8 +279,6 @@ def submit(
             logging.error("--output-prefix is required when --output-type is s3")
             raise typer.Exit(code=1)
         output_location: dict[str, Any] = {"type": "s3", "prefix": output_prefix}
-        if output_connection_string:
-            output_location["connection_string"] = output_connection_string
     elif output_type == "db":
         if not output_table or not output_column:
             logging.error("--output-table and --output-column required for db output")
@@ -212,6 +305,13 @@ def submit(
         "data": [item],
         "priority": priority,
     }
+    if optimizer is not None:
+        payload["optimizer"] = optimizer
+    hardware_payload = _build_hardware_payload(
+        cpu, memory, gpu, gpu_memory, hardware_json
+    )
+    if hardware_payload is not None:
+        payload["hardware"] = hardware_payload
 
     client = client_from_config()
     try:
@@ -624,6 +724,27 @@ def preview(
     input_json: Path | None = typer.Option(
         None, "--input-json", help="JSON file with full inputs object"
     ),
+    optimizer: str | None = typer.Option(
+        None,
+        "--optimizer",
+        help="Override the server default optimizer (must be in /optimizer).",
+    ),
+    cpu: int | None = typer.Option(None, "--cpu", help="Per-job CPU cores override."),
+    memory: str | None = typer.Option(
+        None, "--memory", help="Per-job RAM override (e.g. `32Gi`)."
+    ),
+    gpu: int | None = typer.Option(None, "--gpu", help="Per-job GPU count override."),
+    gpu_memory: str | None = typer.Option(
+        None, "--gpu-memory", help="Per-job GPU memory override (e.g. `24Gi`)."
+    ),
+    hardware_json: Path | None = typer.Option(
+        None,
+        "--hardware-json",
+        help=(
+            "JSON file with the full hardware override; merged with `--cpu` / "
+            "`--memory` / `--gpu` / `--gpu-memory` (flags win on conflict)."
+        ),
+    ),
 ) -> None:
     """Preview the optimization schedule for a workflow without executing it.
 
@@ -646,6 +767,13 @@ def preview(
     payload: dict[str, Any] = {
         "data": [{"workflow": workflow_text, "inputs": inputs}],
     }
+    if optimizer is not None:
+        payload["optimizer"] = optimizer
+    hardware_payload = _build_hardware_payload(
+        cpu, memory, gpu, gpu_memory, hardware_json
+    )
+    if hardware_payload is not None:
+        payload["hardware"] = hardware_payload
 
     client = client_from_config()
     try:
@@ -659,3 +787,238 @@ def preview(
         logging.error(str(exc))
         raise typer.Exit(code=1)
     logging.log(json.dumps(response.json(), indent=2))
+
+
+def _render_workflows_table(job_id: str, workflows: list[dict[str, Any]]) -> Any:
+    table = Table(
+        title=f"Job {job_id} - FlowMesh Workflows",
+        show_header=True,
+        header_style="bold",
+    )
+    table.add_column("Workflow ID", style="cyan", overflow="fold")
+    table.add_column("Status", style="magenta")
+    table.add_column("Submitted At", overflow="fold")
+    table.add_column("Tasks", justify="right")
+    table.add_column("Succeeded", justify="right")
+    table.add_column("Failed", justify="right")
+    for wf in workflows:
+        table.add_row(
+            str(wf.get("workflow_id", "")),
+            str(wf.get("status", "")),
+            str(wf.get("submitted_at") or ""),
+            str(wf.get("task_count") if wf.get("task_count") is not None else ""),
+            str(
+                wf.get("succeeded_count")
+                if wf.get("succeeded_count") is not None
+                else ""
+            ),
+            str(wf.get("failed_count") if wf.get("failed_count") is not None else ""),
+        )
+    return table
+
+
+@app.command("workflows")
+def workflows(
+    job_id: str = typer.Argument(..., help="Job identifier"),
+    as_json: bool = typer.Option(
+        False, "--json", help="Emit raw JSON instead of a table"
+    ),
+) -> None:
+    """List FlowMesh workflows associated with a job."""
+    client = client_from_config()
+    try:
+        response = client.get(f"/jobs/{job_id}/workflows", version_prefix=True)
+    except HttpError as exc:
+        logging.error(str(exc))
+        raise typer.Exit(code=1)
+    payload = _unwrap(response.json())
+    workflow_list = payload.get("workflows", []) if isinstance(payload, dict) else []
+    if as_json:
+        logging.log(json.dumps(payload, indent=2))
+        return
+    Console().print(_render_workflows_table(job_id, workflow_list))
+
+
+def _format_log_line(entry: dict[str, Any]) -> str:
+    event = entry.get("event", {}) if isinstance(entry, dict) else {}
+    if not isinstance(event, dict):
+        event = {}
+    ts = event.get("ts") or ""
+    level = event.get("level") or ""
+    stream = event.get("stream") or ""
+    message = event.get("message") or ""
+    head = f"{ts} {level:<5}".strip()
+    if stream:
+        head = f"{head} [{stream}]"
+    return f"{head} {message}".rstrip()
+
+
+def _print_log_entries(entries: list[dict[str, Any]], as_json: bool) -> None:
+    for entry in entries:
+        if as_json:
+            logging.log(json.dumps(entry))
+        else:
+            logging.log(_format_log_line(entry))
+
+
+def _parse_sse_entries(raw: str) -> list[dict[str, Any]]:
+    entries: list[dict[str, Any]] = []
+    for block in raw.split("\n\n"):
+        current_event: str | None = None
+        for line in block.splitlines():
+            if line.startswith("event:"):
+                current_event = line[len("event:") :].strip()
+            elif line.startswith("data:"):
+                data = line[len("data:") :].strip()
+                if data:
+                    try:
+                        parsed = json.loads(data)
+                        if current_event == "error" or (
+                            isinstance(parsed, dict)
+                            and parsed.get("kind") == "stream_error"
+                        ):
+                            parsed["_sse_error"] = True
+                        entries.append(parsed)
+                    except json.JSONDecodeError:
+                        pass
+    return entries
+
+
+logs_app = get_typer(help="Fetch, stream, or download logs for a FlowMesh workflow.")
+
+
+@logs_app.command("show")
+def logs_show(
+    job_id: str = typer.Argument(..., help="Job identifier"),
+    workflow_id: str = typer.Argument(..., help="FlowMesh workflow identifier"),
+    limit: int = typer.Option(200, "--limit", help="Maximum entries per page (1-1000)"),
+    before: str | None = typer.Option(
+        None, "--before", help="Cursor to fetch entries older than this point"
+    ),
+    after: str | None = typer.Option(
+        None, "--after", help="Cursor to fetch entries newer than this point"
+    ),
+    as_json: bool = typer.Option(False, "--json", help="Emit raw JSON for each entry"),
+) -> None:
+    """Fetch one page of paginated logs for a FlowMesh workflow."""
+    if limit < 1 or limit > 1000:
+        logging.error("--limit must be between 1 and 1000")
+        raise typer.Exit(code=1)
+    client = client_from_config()
+    url = f"/jobs/{job_id}/workflows/{workflow_id}/logs"
+    params: list[tuple[str, str]] = [("limit", str(limit))]
+    if before is not None:
+        params.append(("before", before))
+    if after is not None:
+        params.append(("after", after))
+    try:
+        resp = client.get(url, version_prefix=True, params=params)
+    except HttpError as exc:
+        logging.error(str(exc))
+        raise typer.Exit(code=1)
+    payload = _unwrap(resp.json())
+    entries = payload.get("entries", []) if isinstance(payload, dict) else []
+    _print_log_entries(entries, as_json)
+
+
+@logs_app.command("stream")
+def logs_stream(
+    job_id: str = typer.Argument(..., help="Job identifier"),
+    workflow_id: str = typer.Argument(..., help="FlowMesh workflow identifier"),
+    cursor: str | None = typer.Option(
+        None, "--cursor", help="Resume streaming from this cursor position"
+    ),
+) -> None:
+    """Stream live log entries for a FlowMesh workflow via SSE."""
+    client = client_from_config()
+    stream_url = f"/jobs/{job_id}/workflows/{workflow_id}/logs/stream"
+    stream_params: list[tuple[str, str]] = []
+    if cursor is not None:
+        stream_params.append(("cursor", cursor))
+    try:
+        base = client.base_url.rstrip("/") + "/api/v1"
+        headers = {"Accept": "text/event-stream"}
+        if client.api_key:
+            headers["Authorization"] = f"Bearer {client.api_key}"
+        with requests.get(
+            base + stream_url,
+            headers=headers,
+            params=stream_params or None,
+            stream=True,
+            timeout=client.timeout,
+        ) as resp:
+            if resp.status_code >= 400:
+                logging.error(f"Stream error: {resp.status_code} {resp.text}")
+                raise typer.Exit(code=1)
+            buffer = ""
+            stream_error: dict[str, Any] | None = None
+            for chunk in resp.iter_content(chunk_size=None, decode_unicode=True):
+                if chunk:
+                    buffer += chunk
+                    while "\n\n" in buffer:
+                        block, buffer = buffer.split("\n\n", 1)
+                        for entry in _parse_sse_entries(block + "\n\n"):
+                            if entry.get("_sse_error"):
+                                stream_error = entry
+                                break
+                            _print_log_entries([entry], False)
+                        if stream_error is not None:
+                            break
+                if stream_error is not None:
+                    break
+            if stream_error is not None:
+                code = stream_error.get("code", "APIError")
+                message = stream_error.get("message", "Upstream stream error.")
+                logging.error(f"log stream error [{code}]: {message}")
+                raise typer.Exit(code=1)
+    except KeyboardInterrupt:
+        raise typer.Exit(code=0)
+
+
+@logs_app.command("download")
+def logs_download(
+    job_id: str = typer.Argument(..., help="Job identifier"),
+    workflow_id: str = typer.Argument(..., help="FlowMesh workflow identifier"),
+    output: Path = typer.Option(
+        ..., "--output", "-o", help="Directory to extract logs into"
+    ),
+) -> None:
+    """Download per-task archived logs for a FlowMesh workflow."""
+    output.mkdir(parents=True, exist_ok=True)
+    client = client_from_config()
+    try:
+        client.download(
+            f"/jobs/{job_id}/workflows/{workflow_id}/logs/download",
+            output / f"{workflow_id}-logs.tar",
+            version_prefix=True,
+        )
+    except HttpError as exc:
+        logging.error(str(exc))
+        raise typer.Exit(code=1)
+    except OSError as exc:
+        logging.error(f"Failed to write archive: {exc}")
+        raise typer.Exit(code=1)
+
+    archive_path = output / f"{workflow_id}-logs.tar"
+    extracted: list[Path] = []
+    try:
+        with tarfile.open(archive_path, "r") as tf:
+            members = tf.getmembers()
+            if not members:
+                logging.log(f"No logs downloaded for workflow {workflow_id}.")
+                archive_path.unlink(missing_ok=True)
+                return
+            tf.extractall(output, filter="data")
+            for member in members:
+                extracted.append(output / member.name)
+    except tarfile.TarError as exc:
+        logging.error(f"Failed to extract log archive: {exc}")
+        archive_path.unlink(missing_ok=True)
+        raise typer.Exit(code=1)
+
+    archive_path.unlink(missing_ok=True)
+    for path in extracted:
+        logging.log(str(path))
+
+
+app.add_typer(logs_app, name="logs")

{lumilake_cli-0.1.2 → lumilake_cli-0.1.4/src/lumilake_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lumilake-cli
-Version: 0.1.2
+Version: 0.1.4
 Summary: Lumilake command-line interface
 Author: Lumilake contributors
 License-Expression: Apache-2.0
@@ -14,12 +14,12 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: <3.13,>=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: lumilake-sdk==0.1.2
+Requires-Dist: lumilake-sdk==0.1.4
 Requires-Dist: flowmesh-sdk==0.1.1
 Requires-Dist: requests>=2.32.3
 Requires-Dist: typer>=0.12.5
 Provides-Extra: deploy
-Requires-Dist: lumilake-deploy==0.1.2; extra == "deploy"
+Requires-Dist: lumilake-deploy==0.1.4; extra == "deploy"
 Dynamic: license-file
 
 # `lumilake-cli`

{lumilake_cli-0.1.2 → lumilake_cli-0.1.4}/src/lumilake_cli.egg-info/requires.txt

@@ -1,7 +1,7 @@
-lumilake-sdk==0.1.2
+lumilake-sdk==0.1.4
 flowmesh-sdk==0.1.1
 requests>=2.32.3
 typer>=0.12.5
 
 [deploy]
-lumilake-deploy==0.1.2
+lumilake-deploy==0.1.4