lql-cli 0.2.0__py3-none-any.whl

lql/commands/datasets.py

@@ -0,0 +1,398 @@
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+
+from .._opts import ApiUrlOpt, JsonOpt, ProfileOpt
+from ..api import ApiClient
+from ..output import print_error, print_json, print_table
+from ..util import q
+
+app = typer.Typer(help="Manage datasets")
+
+
+def _truncate(v: object, n: int = 80) -> str:
+    s = json.dumps(v) if isinstance(v, (dict, list)) else ("" if v is None else str(v))
+    return s[: n - 3] + "..." if len(s) > n else s
+
+
+@app.command("list")
+def list_datasets(
+    workspace: Annotated[Optional[str], typer.Option("--workspace", help="Filter by workspace ID")] = None,
+    json_out: JsonOpt = False,
+    profile: ProfileOpt = None,
+    api_url: ApiUrlOpt = None,
+) -> None:
+    """List datasets."""
+    client = ApiClient(profile=profile, api_url=api_url)
+    params = {}
+    if workspace:
+        params["workspace_id"] = workspace
+    items = client.get("/v1/datasets", params=params).json()
+    print_table(
+        ["ID", "Name", "Workspace", "Rows"],
+        [
+            [
+                d.get("id") or "",
+                d.get("display_name") or d.get("name") or "",
+                d.get("workspace_id") or "",
+                d.get("row_count") if d.get("row_count") is not None else "",
+            ]
+            for d in items
+        ],
+        json_out,
+        items,
+    )
+
+
+@app.command("show")
+def show(
+    id: Annotated[str, typer.Argument(help="Dataset ID")],
+    json_out: JsonOpt = False,
+    profile: ProfileOpt = None,
+    api_url: ApiUrlOpt = None,
+) -> None:
+    """Show dataset details."""
+    client = ApiClient(profile=profile, api_url=api_url)
+    d = client.get(f"/v1/datasets/{q(id)}").json()
+    if json_out:
+        print_json(d)
+    else:
+        print_table(
+            ["Field", "Value"],
+            [
+                ["ID", d.get("id") or ""],
+                ["Name", d.get("display_name") or d.get("name") or ""],
+                ["Workspace", d.get("workspace_id") or ""],
+                ["HF Repo", d.get("hf_repo_id") or ""],
+                ["Split", d.get("hf_split") or ""],
+                ["Rows", d.get("row_count") if d.get("row_count") is not None else ""],
+                ["Status", d.get("sync_status") or ""],
+            ],
+            False,
+            [d],
+        )
+
+
+@app.command("create")
+def create(
+    workspace: Annotated[str, typer.Option("--workspace", help="Workspace ID")],
+    hf_repo: Annotated[Optional[str], typer.Option("--hf-repo", help="HuggingFace repo (e.g. org/dataset)")] = None,
+    hf_bucket: Annotated[Optional[str], typer.Option("--hf-bucket", help="HuggingFace storage bucket")] = None,
+    key: Annotated[Optional[str], typer.Option("--key", help="Path or glob within the bucket (with --hf-bucket)")] = None,
+    name: Annotated[Optional[str], typer.Option("--name", help="Display name")] = None,
+    split: Annotated[str, typer.Option("--split", help="Dataset split (HF repo only)")] = "train",
+    json_out: JsonOpt = False,
+    profile: ProfileOpt = None,
+    api_url: ApiUrlOpt = None,
+) -> None:
+    """Create a dataset from a HuggingFace repo or storage bucket."""
+    if not hf_repo and not hf_bucket:
+        print_error("Either --hf-repo or --hf-bucket is required.", "missing_source")
+        raise typer.Exit(1)
+    if hf_repo and hf_bucket:
+        print_error("--hf-repo and --hf-bucket are mutually exclusive.", "ambiguous_source")
+        raise typer.Exit(1)
+    if hf_bucket and not key:
+        print_error("--key is required with --hf-bucket (path or glob, e.g. data/*.parquet).", "missing_key")
+        raise typer.Exit(1)
+
+    client = ApiClient(profile=profile, api_url=api_url)
+    if hf_bucket:
+        body = {
+            "workspace_id": workspace,
+            "source_type": "hf_bucket",
+            "hf_bucket": hf_bucket,
+            "hf_bucket_key": key,
+            "display_name": name or (key.split("/")[-1] if key else None) or hf_bucket,
+        }
+    else:
+        body = {
+            "workspace_id": workspace,
+            "hf_repo_id": hf_repo,
+            "hf_split": split or "train",
+            "display_name": name or hf_repo,
+        }
+    data = client.post("/v1/datasets", json=body).json()
+    if json_out:
+        print_json(data)
+    else:
+        status = f" ({data.get('sync_status')})" if data.get("sync_status") else ""
+        sys.stdout.write(f"Created dataset: {data.get('id')}{status}\n")
+
+
+@app.command("sync")
+def sync(
+    id: Annotated[str, typer.Argument(help="Dataset ID")],
+    json_out: JsonOpt = False,
+    profile: ProfileOpt = None,
+    api_url: ApiUrlOpt = None,
+) -> None:
+    """Trigger a dataset sync."""
+    client = ApiClient(profile=profile, api_url=api_url)
+    data = client.post(f"/v1/datasets/{q(id)}/sync").json()
+    if json_out:
+        print_json(data)
+    else:
+        sys.stdout.write(f"Sync started for dataset: {id}\n")
+
+
+@app.command("schema")
+def schema(
+    id: Annotated[str, typer.Argument(help="Dataset ID")],
+    json_out: JsonOpt = False,
+    profile: ProfileOpt = None,
+    api_url: ApiUrlOpt = None,
+) -> None:
+    """Get dataset schema."""
+    client = ApiClient(profile=profile, api_url=api_url)
+    data = client.get(f"/v1/datasets/{q(id)}/schema").json()
+    if json_out:
+        print_json(data)
+    else:
+        columns = (data or {}).get("columns") if isinstance(data, dict) else data
+        columns = columns or []
+        print_table(
+            ["Column", "Type"],
+            [[c.get("name") or "", c.get("type") or ""] for c in columns],
+            False,
+            columns,
+        )
+
+
+def _fmt_null_pct(nr: float) -> str:
+    # Avoid rounding a not-quite-0/100 null rate to a misleading "0%"/"100%".
+    pct = nr * 100
+    if 0 < pct < 0.5:
+        return "<1%"
+    if 99.5 < pct < 100:
+        return ">99%"
+    return f"{round(pct)}%"
+
+
+@app.command("profile")
+def profile_cmd(
+    id: Annotated[str, typer.Argument(help="Dataset ID")],
+    full_content: Annotated[bool, typer.Option("--full-content", help="Scan every row for content stats (slow)")] = False,
+    skip_content: Annotated[bool, typer.Option("--skip-content", help="Skip content/token stats entirely")] = False,
+    json_out: JsonOpt = False,
+    profile: ProfileOpt = None,
+    api_url: ApiUrlOpt = None,
+) -> None:
+    """Profile a dataset: per-column nulls/cardinality/numeric stats/top values + content token stats."""
+    client = ApiClient(profile=profile, api_url=api_url)
+    params = {}
+    if full_content:
+        params["full_content"] = "true"
+    if skip_content:
+        params["skip_content"] = "true"
+    p = client.get(f"/v1/datasets/{q(id)}/profile", params=params).json()
+    if json_out:
+        print_json(p)
+        return
+    if p.get("skip_reason"):
+        sys.stdout.write(f"Profile unavailable: {p['skip_reason']}\n")
+        return
+    sys.stdout.write(f"Rows: {p.get('total_rows', '?')}\n")
+    cols = p.get("columns") or []
+
+    def col_row(c):
+        num = c.get("numeric_stats")
+        num_str = f"{num['min']}/{num['p50']}/{num['p95']}/{num['max']}" if num else ""
+        top = c.get("top_values")
+        note = f"[{c['skip_reason']}]" if c.get("skip_reason") else ""
+        if top:
+            note = ", ".join(f"{t.get('value')}:{t.get('count')}" for t in top[:3])
+        nr = c.get("null_rate")
+        return [
+            c.get("name") or "",
+            str(c.get("type") or "")[:28],
+            "" if nr is None else _fmt_null_pct(float(nr)),
+            "" if c.get("approx_distinct") is None else str(c.get("approx_distinct")),
+            num_str,
+            note[:45] + "..." if len(note) > 48 else note,
+        ]
+
+    print_table(
+        ["Column", "Type", "Null %", "~Distinct", "Numeric min/p50/p95/max", "Top / note"],
+        [col_row(c) for c in cols],
+        False,
+        cols,
+    )
+
+    cs = p.get("content_stats")
+    if cs:
+        if cs.get("sampled"):
+            sys.stdout.write(
+                f"\n⚠ content stats sampled from the first shard "
+                f"({cs.get('sample_rows')} / {cs.get('total_rows')} rows). "
+                "max_chars is sample-bound — use --full-content for an exact scan.\n"
+            )
+        else:
+            sys.stdout.write(f"\nContent stats (full scan, {cs.get('sample_rows')} rows):\n")
+        ccols = cs.get("columns") or []
+        print_table(
+            ["Column", "Field", "P50 ch/~tok", "P95 ch/~tok", "Max ch", "Turns avg/max"],
+            [
+                [
+                    c.get("column") or "",
+                    c.get("text_field") or "",
+                    f"{c.get('p50_chars', '')} / ~{c.get('p50_tokens_est', '')}",
+                    f"{c.get('p95_chars', '')} / ~{c.get('p95_tokens_est', '')}",
+                    c.get("max_chars") if c.get("max_chars") is not None else "",
+                    "" if c.get("avg_turns") is None else f"{c.get('avg_turns')} / {c.get('max_turns')}",
+                ]
+                for c in ccols
+            ],
+            False,
+            ccols,
+        )
+
+
+@app.command("rows")
+def rows(
+    id: Annotated[str, typer.Argument(help="Dataset ID")],
+    limit: Annotated[str, typer.Option("--limit", help="Number of rows")] = "20",
+    offset: Annotated[str, typer.Option("--offset", help="Row offset")] = "0",
+    json_out: JsonOpt = False,
+    profile: ProfileOpt = None,
+    api_url: ApiUrlOpt = None,
+) -> None:
+    """Get dataset rows."""
+    client = ApiClient(profile=profile, api_url=api_url)
+    data = client.get(f"/v1/datasets/{q(id)}/rows", params={"limit": limit, "offset": offset}).json()
+    if json_out:
+        print_json(data)
+        return
+    rows_data = (data or {}).get("rows") if isinstance(data, dict) else data
+    rows_data = rows_data or []
+    if not rows_data:
+        sys.stdout.write("No rows.\n")
+        return
+    headers = list(rows_data[0].keys())
+    print_table(
+        headers,
+        [[_truncate(r.get(h)) for h in headers] for r in rows_data],
+        False,
+        rows_data,
+    )
+
+
+@app.command("delete")
+def delete(
+    id: Annotated[str, typer.Argument(help="Dataset ID")],
+    profile: ProfileOpt = None,
+    api_url: ApiUrlOpt = None,
+) -> None:
+    """Delete a dataset."""
+    client = ApiClient(profile=profile, api_url=api_url)
+    client.delete(f"/v1/datasets/{q(id)}")
+    sys.stdout.write(f"Deleted dataset: {id}\n")
+
+
+@app.command("push")
+def push(
+    id: Annotated[str, typer.Argument(help="Dataset ID")],
+    json_out: JsonOpt = False,
+    profile: ProfileOpt = None,
+    api_url: ApiUrlOpt = None,
+) -> None:
+    """Push dataset to HuggingFace."""
+    client = ApiClient(profile=profile, api_url=api_url)
+    data = client.post(f"/v1/datasets/{q(id)}/push").json()
+    if json_out:
+        print_json(data)
+    else:
+        sys.stdout.write(f"Push started. Job ID: {data.get('job_id', 'unknown')}\n")
+
+
+@app.command("push-status")
+def push_status(
+    id: Annotated[str, typer.Argument(help="Dataset ID")],
+    job: Annotated[Optional[str], typer.Option("--job", help="Specific job ID")] = None,
+    json_out: JsonOpt = False,
+    profile: ProfileOpt = None,
+    api_url: ApiUrlOpt = None,
+) -> None:
+    """Get push job status for a dataset."""
+    client = ApiClient(profile=profile, api_url=api_url)
+    url = f"/v1/datasets/{q(id)}/push-jobs/{q(job)}" if job else f"/v1/datasets/{q(id)}/push-jobs"
+    data = client.get(url).json()
+    if json_out:
+        print_json(data)
+    else:
+        jobs = data if isinstance(data, list) else [data]
+        print_table(
+            ["Job ID", "Status", "Created"],
+            [[j.get("id") or "", j.get("status") or "", j.get("created_at") or ""] for j in jobs],
+            False,
+            jobs,
+        )
+
+
+@app.command("upload")
+def upload(
+    file: Annotated[str, typer.Argument(help="Local file to upload")],
+    workspace: Annotated[str, typer.Option("--workspace", help="Workspace ID")],
+    name: Annotated[str, typer.Option("--name", help="Repo name (will upload to liquid-ai/<name>)")],
+    split: Annotated[str, typer.Option("--split", help="Dataset split")] = "train",
+    json_out: JsonOpt = False,
+    profile: ProfileOpt = None,
+    api_url: ApiUrlOpt = None,
+) -> None:
+    """Upload a local file to HuggingFace and create a dataset."""
+    hf_token = os.environ.get("LQL_HF_TOKEN")
+    if not hf_token:
+        print_error("LQL_HF_TOKEN environment variable is required for upload.", "missing_hf_token")
+        raise typer.Exit(1)
+
+    resolved = Path(file).resolve()
+    if not resolved.exists():
+        print_error(f"File not found: {resolved}", "file_not_found")
+        raise typer.Exit(1)
+
+    # Validate LQL auth + API URL *before* the external HF side effect, so a
+    # missing/expired token or bad API URL can't leave an orphaned HF upload.
+    client = ApiClient(profile=profile, api_url=api_url)
+
+    repo_name = f"liquid-ai/{name}"
+    sys.stdout.write(f"Uploading to HuggingFace: {repo_name}...\n")
+
+    try:
+        from huggingface_hub import HfApi  # lazy import: keeps startup fast
+
+        hf = HfApi(token=hf_token)
+        hf.create_repo(repo_id=repo_name, repo_type="dataset", exist_ok=True)
+        hf.upload_file(
+            path_or_fileobj=str(resolved),
+            path_in_repo=f"data/{split or 'train'}/{resolved.name}",
+            repo_id=repo_name,
+            repo_type="dataset",
+        )
+        sys.stdout.write("Uploaded to HuggingFace.\n")
+    except Exception as e:
+        print_error(f"HuggingFace upload failed: {e or 'unknown error'}", "hf_upload_failed")
+        raise typer.Exit(1)
+
+    create_res = client.post(
+        "/v1/datasets",
+        json={
+            "workspace_id": workspace,
+            "hf_id": repo_name,
+            "display_name": name,
+            "split": split or "train",
+        },
+    ).json()
+    dataset_id = create_res.get("id")
+    sys.stdout.write(f"Created dataset: {dataset_id}\n")
+
+    client.post(f"/v1/datasets/{q(dataset_id)}/sync")
+    sys.stdout.write("Sync started.\n")
+
+    if json_out:
+        print_json({"id": dataset_id, "hf_repo": repo_name, "status": "syncing"})
+    else:
+        sys.stdout.write(f"Dataset ID: {dataset_id}\nStatus: syncing\n")

lql/commands/edits.py

@@ -0,0 +1,95 @@
+import json
+import sys
+from typing import Annotated, Optional
+
+import typer
+
+from .._opts import ApiUrlOpt, JsonOpt, ProfileOpt
+from ..api import ApiClient
+from ..output import print_error, print_json, print_table
+from ..util import q
+
+app = typer.Typer(help="Manage row edits")
+
+
+@app.command("list")
+def list_edits(
+    dataset_id: Annotated[str, typer.Argument(help="Dataset ID")],
+    limit: Annotated[str, typer.Option("--limit", help="Number of edits")] = "50",
+    json_out: JsonOpt = False,
+    profile: ProfileOpt = None,
+    api_url: ApiUrlOpt = None,
+) -> None:
+    """List edits for a dataset."""
+    client = ApiClient(profile=profile, api_url=api_url)
+    items = client.get(f"/v1/datasets/{q(dataset_id)}/edits", params={"limit": limit}).json()
+    print_table(
+        ["ID", "Row", "Column", "Value"],
+        [
+            [
+                e.get("id") or "",
+                e.get("row_external_id") or "",
+                e.get("column") or "",
+                json.dumps(e.get("new_value"))[:60],
+            ]
+            for e in items
+        ],
+        json_out,
+        items,
+    )
+
+
+@app.command("count")
+def count(
+    dataset_id: Annotated[str, typer.Argument(help="Dataset ID")],
+    json_out: JsonOpt = False,
+    profile: ProfileOpt = None,
+    api_url: ApiUrlOpt = None,
+) -> None:
+    """Count edits for a dataset."""
+    client = ApiClient(profile=profile, api_url=api_url)
+    data = client.get(f"/v1/datasets/{q(dataset_id)}/edits/count").json()
+    if json_out:
+        print_json(data)
+    else:
+        sys.stdout.write(f"Pending edits: {data.get('pending', 0)}\n")
+
+
+@app.command("add")
+def add(
+    dataset_id: Annotated[str, typer.Argument(help="Dataset ID")],
+    row: Annotated[str, typer.Option("--row", help="Row external ID")],
+    column: Annotated[str, typer.Option("--column", help="Column name")],
+    value: Annotated[str, typer.Option("--value", help="New value as JSON")],
+    json_out: JsonOpt = False,
+    profile: ProfileOpt = None,
+    api_url: ApiUrlOpt = None,
+) -> None:
+    """Add an edit to a row."""
+    try:
+        parsed = json.loads(value)
+    except ValueError:
+        print_error(f"Invalid JSON for --value: {value}", "invalid_json")
+        raise typer.Exit(1)
+    client = ApiClient(profile=profile, api_url=api_url)
+    data = client.post(
+        f"/v1/datasets/{q(dataset_id)}/edits",
+        json={"row_external_id": row, "column": column, "new_value": parsed},
+    ).json()
+    if json_out:
+        print_json(data)
+    else:
+        sys.stdout.write(f"Created edit: {data.get('id', 'ok')}\n")
+
+
+@app.command("delete")
+def delete(
+    dataset_id: Annotated[str, typer.Argument(help="Dataset ID")],
+    edit_id: Annotated[str, typer.Argument(help="Edit ID")],
+    profile: ProfileOpt = None,
+    api_url: ApiUrlOpt = None,
+) -> None:
+    """Delete an edit."""
+    client = ApiClient(profile=profile, api_url=api_url)
+    client.delete(f"/v1/datasets/{q(dataset_id)}/edits/{q(edit_id)}")
+    sys.stdout.write(f"Deleted edit: {edit_id}\n")