lql-cli 0.3.0__tar.gz → 0.4.0__tar.gz

{lql_cli-0.3.0 → lql_cli-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lql-cli
-Version: 0.3.0
+Version: 0.4.0
 Summary: lql — CLI for the Liquid DataViewer platform
 Project-URL: Homepage, https://github.com/Liquid4All/lql
 Author: Liquid AI
@@ -149,18 +149,30 @@ multimodal content (text/image/audio), ShareGPT `{from, value}`, native OpenAI
 `tool_calls`, plus `<think>` reasoning blocks, `<|tool_call_start|>…<|tool_call_end|>`
 / Python / XML / JSON tool calls, tool results, tool-definition tables, and code.
 
-Works on a local `.jsonl`/`.json` file or a platform dataset ID. No browser, and
-nothing to forward over SSH — it's just the terminal.
+Works on a local `.jsonl`/`.json` file, a platform dataset ID, or — with `--hf`
+— a HuggingFace repo. No browser, and nothing to forward over SSH — it's just
+the terminal.
 
 ```
 lql preview <file.jsonl|file.json>     Local file: each line/object is a row
 lql preview <dataset-id>               Platform dataset (fetched & paged lazily)
+lql preview <org/name> --hf            HuggingFace repo: sync to DataViewer, then view
 lql preview <src> -c <field>           Force field(s) as conversations (repeatable)
 lql preview <src> -n <N>               Page size when paging a platform dataset
 lql preview <src> --offset N           Start at row index N
 lql preview <src> --title "<title>"    Title shown in the viewer header
 ```
 
+**HuggingFace datasets (`--hf`).** `lql preview org/name --hf` syncs the repo
+into a DataViewer workspace, then opens it. You pick the target workspace from
+an interactive list (or pass `--workspace <id>`; `--split` defaults to `train`).
+Already-synced repos are reused — no duplicate, instant re-open.
+
+```
+lql preview tatsu-lab/alpaca --hf
+lql preview org/name --hf --split validation --workspace <id>
+```
+
 **Navigation** — two modes, toggle with `m` / `Tab`:
 
 - **pager** (default): one sample at a time · `←/→` or `n`/`b` switch samples · `↑/↓`/`j`/`k`/PgUp-Dn scroll

{lql_cli-0.3.0 → lql_cli-0.4.0}/README.md

@@ -134,18 +134,30 @@ multimodal content (text/image/audio), ShareGPT `{from, value}`, native OpenAI
 `tool_calls`, plus `<think>` reasoning blocks, `<|tool_call_start|>…<|tool_call_end|>`
 / Python / XML / JSON tool calls, tool results, tool-definition tables, and code.
 
-Works on a local `.jsonl`/`.json` file or a platform dataset ID. No browser, and
-nothing to forward over SSH — it's just the terminal.
+Works on a local `.jsonl`/`.json` file, a platform dataset ID, or — with `--hf`
+— a HuggingFace repo. No browser, and nothing to forward over SSH — it's just
+the terminal.
 
 ```
 lql preview <file.jsonl|file.json>     Local file: each line/object is a row
 lql preview <dataset-id>               Platform dataset (fetched & paged lazily)
+lql preview <org/name> --hf            HuggingFace repo: sync to DataViewer, then view
 lql preview <src> -c <field>           Force field(s) as conversations (repeatable)
 lql preview <src> -n <N>               Page size when paging a platform dataset
 lql preview <src> --offset N           Start at row index N
 lql preview <src> --title "<title>"    Title shown in the viewer header
 ```
 
+**HuggingFace datasets (`--hf`).** `lql preview org/name --hf` syncs the repo
+into a DataViewer workspace, then opens it. You pick the target workspace from
+an interactive list (or pass `--workspace <id>`; `--split` defaults to `train`).
+Already-synced repos are reused — no duplicate, instant re-open.
+
+```
+lql preview tatsu-lab/alpaca --hf
+lql preview org/name --hf --split validation --workspace <id>
+```
+
 **Navigation** — two modes, toggle with `m` / `Tab`:
 
 - **pager** (default): one sample at a time · `←/→` or `n`/`b` switch samples · `↑/↓`/`j`/`k`/PgUp-Dn scroll

{lql_cli-0.3.0 → lql_cli-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "lql-cli"
-version = "0.3.0"
+version = "0.4.0"
 description = "lql — CLI for the Liquid DataViewer platform"
 readme = "README.md"
 requires-python = ">=3.9"

lql_cli-0.4.0/src/lql/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.4.0"

{lql_cli-0.3.0 → lql_cli-0.4.0}/src/lql/commands/instructions.py

@@ -83,11 +83,16 @@ Non-conversation fields render as labeled metadata.
 
   lql preview <file.jsonl|file.json>     # local file: each line/object is a row
   lql preview <dataset-id>               # platform dataset, fetched & paged lazily
+  lql preview <org/name> --hf            # HuggingFace repo: sync to DataViewer, then view
   lql preview data.jsonl -c messages     # force field(s) as conversations
 
+With --hf, SOURCE is a HuggingFace repo (org/name): it's synced into a
+DataViewer workspace (you pick one interactively, or pass --workspace <id>;
+--split defaults to train) and reused on later previews (dedup by repo+split).
+
 Options: -c/--column (field(s) to treat as conversations; default auto-detect,
 repeatable), -n/--limit (page size when paging a platform dataset), --offset
-(start row index), --title, --profile, --api-url.
+(start row index), --title, --hf, --split, --workspace, --profile, --api-url.
 
 Navigation: two modes toggled with m/Tab — pager (one sample at a time; ←/→ or
 n/b switch samples, ↑/↓/j/k scroll) and scroll (all samples; n/b jump between

{lql_cli-0.3.0 → lql_cli-0.4.0}/src/lql/commands/preview.py

@@ -12,6 +12,7 @@ these and renders to the terminal via Rich + Textual.
 
 import json
 import re
+import sys
 from pathlib import Path
 from typing import Annotated, List, Optional
 
@@ -585,13 +586,128 @@ def _normalize_loaded(data: object) -> List[object]:
     return [data]
 
 
+# --------------------------------------------------------------------------
+# HuggingFace auto-sync (for `--hf`)
+# --------------------------------------------------------------------------
+
+# Default landing workspace for HF datasets synced on the fly (there is no
+# "personal workspace" concept in the API, so we find-or-create this one).
+_HF_WORKSPACE_NAME = "lql-preview"
+# Terminal sync states. Accept several "done" spellings defensively in case the
+# API's vocabulary varies (verified to return "ready" today).
+_SYNC_READY = {"ready", "completed", "synced", "success", "done"}
+_SYNC_FAILED = {"failed", "error", "cancelled", "canceled"}
+
+
+def _find_or_create_workspace(client: ApiClient, name: str) -> str:
+    """Return the id of the workspace named `name`, creating it if absent."""
+    for w in client.get("/v1/workspaces").json():
+        if (w.get("display_name") or w.get("name") or "").strip().lower() == name.lower():
+            return w["id"]
+    created = client.post("/v1/workspaces", json={"name": name}).json()
+    return created["id"]
+
+
+def _find_dataset_for_repo(client: ApiClient, workspace_id: str, repo: str, split: str) -> Optional[dict]:
+    """Return an existing dataset in the workspace for this HF repo+split, if any."""
+    items = client.get("/v1/datasets", params={"workspace_id": workspace_id}).json()
+    for d in items or []:
+        if d.get("hf_repo_id") == repo and (d.get("hf_split") or "train") == split:
+            return d
+    return None
+
+
+def _wait_until_ready(client: ApiClient, dataset_id: str, timeout: float = 180.0) -> None:
+    """Poll the dataset's sync_status until it's ready (or fails / times out),
+    printing progress dots so a long sync doesn't look hung."""
+    import time
+
+    start = time.time()
+    dots = 0
+    while True:
+        d = client.get(f"/v1/datasets/{q(dataset_id)}").json()
+        status = (d.get("sync_status") or "").lower()
+        if status in _SYNC_READY:
+            sys.stdout.write("\n")
+            return
+        if status in _SYNC_FAILED:
+            print_error(f"Sync failed for {dataset_id} (status: {status or 'unknown'})", "sync_failed")
+            raise typer.Exit(5)
+        if time.time() - start > timeout:
+            sys.stdout.write("\n")
+            print_error(
+                f"Timed out waiting for sync (status: {status or 'unknown'}). "
+                f"It may still finish — re-run the same command to resume.",
+                "sync_timeout",
+            )
+            raise typer.Exit(1)
+        dots = (dots % 3) + 1
+        sys.stdout.write("\r  syncing" + "." * dots + "   ")
+        sys.stdout.flush()
+        time.sleep(2.0)
+
+
+def _resolve_hf_dataset(client: ApiClient, repo: str, split: str, ws_id: str) -> str:
+    """Reuse an already-synced dataset for this HF repo in the given workspace if
+    present, otherwise create + sync one. Returns the dataset id."""
+    existing = _find_dataset_for_repo(client, ws_id, repo, split)
+    if existing is not None:
+        dataset_id = existing["id"]
+        if (existing.get("sync_status") or "").lower() not in _SYNC_READY:
+            sys.stdout.write(f"Resuming sync of {repo} …\n")
+            _wait_until_ready(client, dataset_id)
+        return dataset_id
+    sys.stdout.write(f"Syncing {repo} ({split}) into workspace '{_HF_WORKSPACE_NAME}' …\n")
+    created = client.post(
+        "/v1/datasets",
+        json={"workspace_id": ws_id, "hf_repo_id": repo, "hf_split": split, "display_name": repo},
+    ).json()
+    dataset_id = created.get("id")
+    if not dataset_id:
+        print_error("Dataset creation did not return an id.", "create_failed")
+        raise typer.Exit(5)
+    if (created.get("sync_status") or "").lower() not in _SYNC_READY:
+        _wait_until_ready(client, dataset_id)
+    return dataset_id
+
+
+def _resolve_workspace_arg(client: ApiClient, value: str) -> str:
+    """Resolve a --workspace value that may be an id OR a name. Returns a
+    workspace id, creating the workspace if the value is an unknown name."""
+    workspaces = client.get("/v1/workspaces").json() or []
+    for w in workspaces:
+        if w.get("id") == value:
+            return value
+    for w in workspaces:
+        if (w.get("display_name") or w.get("name") or "").strip().lower() == value.strip().lower():
+            return w["id"]
+    return _find_or_create_workspace(client, value)
+
+
+_CREATE_SENTINEL = "__create_lql_preview__"
+
+
+def _choose_workspace(client: ApiClient, tui_mod) -> Optional[str]:
+    """Interactive workspace picker (arrow keys + Enter). Returns the chosen
+    workspace id, or None if the user cancels."""
+    workspaces = client.get("/v1/workspaces").json() or []
+    options = [((w.get("display_name") or w.get("name") or w["id"]), w["id"]) for w in workspaces]
+    options.append((f"➕  Create new workspace '{_HF_WORKSPACE_NAME}'", _CREATE_SENTINEL))
+    choice = tui_mod.select_workspace(options, title="Select a workspace for this HF dataset")
+    if choice is None:
+        return None
+    if choice == _CREATE_SENTINEL:
+        return _find_or_create_workspace(client, _HF_WORKSPACE_NAME)
+    return choice
+
+
 # --------------------------------------------------------------------------
 # Command
 # --------------------------------------------------------------------------
 
 
 def preview(
-    source: Annotated[str, typer.Argument(help="Local .jsonl/.json file, or a dataset ID")],
+    source: Annotated[str, typer.Argument(help="Local .jsonl/.json file, a dataset ID, or an HF repo with --hf")],
     column: Annotated[
         Optional[List[str]],
         typer.Option("--column", "-c", help="Field(s) to render as conversations (default: auto-detect)"),
@@ -599,15 +715,24 @@ def preview(
     limit: Annotated[int, typer.Option("--limit", "-n", help="Page size when paging a platform dataset")] = 25,
     offset: Annotated[int, typer.Option("--offset", help="Start at this row index")] = 0,
     title: Annotated[Optional[str], typer.Option("--title", help="Title shown in the viewer header")] = None,
+    hf: Annotated[
+        bool, typer.Option("--hf", help="Treat SOURCE as a HuggingFace repo (org/name); sync it to DataViewer, then view")
+    ] = False,
+    split: Annotated[str, typer.Option("--split", help="HF split (with --hf)")] = "train",
+    workspace: Annotated[
+        Optional[str],
+        typer.Option("--workspace", help="Workspace name or ID to sync into (with --hf); default: pick one interactively"),
+    ] = None,
     profile: ProfileOpt = None,
     api_url: ApiUrlOpt = None,
 ) -> None:
     """View dataset samples in the terminal (a Textual TUI).
 
-    SOURCE may be a local .jsonl/.json file or a platform dataset ID. Local
-    files are loaded whole; platform datasets are fetched and paged lazily as
-    you navigate (--limit is the page size). Works over plain SSH — no browser
-    or port-forward needed.
+    SOURCE may be a local .jsonl/.json file, a platform dataset ID, or — with
+    --hf — a HuggingFace repo (org/name). Local files are loaded whole; platform
+    datasets are fetched and paged lazily (--limit is the page size). With --hf,
+    the repo is synced into a DataViewer workspace (reused if already synced),
+    then opened. Works over plain SSH — no browser or port-forward needed.
 
     Navigation: ←/→ or n/b switch samples · ↑/↓/j/k scroll · m toggles
     pager/scroll mode · q quits.
@@ -619,35 +744,69 @@ def preview(
         raise typer.Exit(1)
 
     local_path = Path(source)
-    is_local = local_path.exists() and local_path.is_file()
+    is_local = (not hf) and local_path.exists() and local_path.is_file()
 
+    # Local file → load whole, view immediately.
     if is_local:
-        row_source = tui_mod.RowSource(initial=_load_local(local_path))
-        start_idx = max(0, offset)
-        index_base = 0  # local rows are absolute; the cursor just starts at offset
-        view_title = title or local_path.name
+        tui_mod.run(
+            tui_mod.RowSource(initial=_load_local(local_path)),
+            title or local_path.name,
+            forced_cols=column or None,
+            start_idx=max(0, offset),
+            index_base=0,  # local rows are absolute; the cursor just starts at offset
+        )
+        return
+
+    client = ApiClient(profile=profile, api_url=api_url)
+
+    # --hf: resolve (sync/reuse) the repo into a workspace and get its dataset id.
+    if hf:
+        if workspace:
+            ws_id = _resolve_workspace_arg(client, workspace)
+        elif sys.stdin.isatty() and sys.stdout.isatty():
+            ws_id = _choose_workspace(client, tui_mod)
+            if ws_id is None:
+                raise typer.Exit(0)  # user cancelled the picker
+        else:
+            ws_id = _find_or_create_workspace(client, _HF_WORKSPACE_NAME)
+        dataset_id = _resolve_hf_dataset(client, source, split, ws_id)
+        view_title = title or f"{source} ({split})"
     else:
-        client = ApiClient(profile=profile, api_url=api_url)
-        page_size = limit if limit and limit > 0 else 25
-
-        def _fetch_page(off: int, lim: int) -> List[object]:
-            data = client.get(
-                f"/v1/datasets/{q(source)}/rows", params={"limit": str(lim), "offset": str(offset + off)}
-            ).json()
-            return _normalize_loaded(data)
-
-        # Fetch the first page up front (with feedback) so the viewer opens
-        # already populated instead of launching into a blank alternate-screen
-        # while the network call blocks. API/auth errors surface here as normal
-        # CLI errors too, rather than vanishing behind the TUI.
-        print(f"Loading dataset {source} …", flush=True)
-        first = _fetch_page(0, page_size)
-        if not first:
-            print_error("Dataset returned no rows.", "empty_dataset")
-            raise typer.Exit(3)
-        row_source = tui_mod.RowSource(initial=first, fetch_page=_fetch_page, page_size=page_size)
-        start_idx = 0
-        index_base = offset  # fetched window starts at --offset; show absolute indexes
+        dataset_id = source
         view_title = title or f"dataset {source}"
 
-    tui_mod.run(row_source, view_title, forced_cols=column or None, start_idx=start_idx, index_base=index_base)
+    page_size = limit if limit and limit > 0 else 25
+
+    def _fetch_page(off: int, lim: int) -> List[object]:
+        data = client.get(
+            f"/v1/datasets/{q(dataset_id)}/rows", params={"limit": str(lim), "offset": str(offset + off)}
+        ).json()
+        return _normalize_loaded(data)
+
+    # Fetch the first page up front (with feedback) so the viewer opens already
+    # populated instead of launching into a blank alternate-screen while the
+    # network call blocks. API/auth errors surface here as normal CLI errors too.
+    print(f"Loading dataset {dataset_id} …", flush=True)
+    first = _fetch_page(0, page_size)
+    if not first and hf:
+        # Rows can lag a freshly-reported-ready sync (indexing/eventual
+        # consistency); give it a few short retries before giving up.
+        import time
+
+        for _ in range(3):
+            time.sleep(2.0)
+            first = _fetch_page(0, page_size)
+            if first:
+                break
+    if not first:
+        msg = f"Dataset returned no rows for split '{split}'." if hf else "Dataset returned no rows."
+        print_error(msg, "empty_dataset")
+        raise typer.Exit(3)
+    row_source = tui_mod.RowSource(initial=first, fetch_page=_fetch_page, page_size=page_size)
+    tui_mod.run(
+        row_source,
+        view_title,
+        forced_cols=column or None,
+        start_idx=0,
+        index_base=offset,  # fetched window starts at --offset; show absolute indexes
+    )

{lql_cli-0.3.0 → lql_cli-0.4.0}/src/lql/commands/tui.py

@@ -338,6 +338,44 @@ class RowSource:
 # --------------------------------------------------------------------------
 
 
+def select_workspace(options: List[tuple], title: str = "Select a workspace") -> Optional[object]:
+    """Interactive arrow-key picker. `options` is a list of (label, value);
+    returns the chosen value, or None if the user cancels (q / esc)."""
+    from textual.app import App, ComposeResult
+    from textual.binding import Binding
+    from textual.widgets import Footer, Header, OptionList
+    from textual.widgets.option_list import Option
+
+    class Picker(App):
+        CSS = "OptionList { height: 1fr; padding: 1 2; }"
+        BINDINGS = [Binding("escape,q", "cancel", "Cancel")]
+
+        def __init__(self) -> None:
+            super().__init__()
+            self.selected: Optional[object] = None
+
+        def compose(self) -> "ComposeResult":
+            yield Header()
+            yield OptionList(*[Option(label, id=str(i)) for i, (label, _v) in enumerate(options)])
+            yield Footer()
+
+        def on_mount(self) -> None:
+            self.title = title
+            self.sub_title = "↑/↓ move · Enter select · q cancel"
+            self.query_one(OptionList).focus()
+
+        def on_option_list_option_selected(self, event) -> None:
+            self.selected = options[int(event.option.id)][1]
+            self.exit()
+
+        def action_cancel(self) -> None:
+            self.exit()
+
+    app = Picker()
+    app.run()
+    return app.selected
+
+
 def run(
     source: "RowSource",
     title: str,

{lql_cli-0.3.0 → lql_cli-0.4.0}/uv.lock

@@ -315,7 +315,7 @@ wheels = [
 
 [[package]]
 name = "lql-cli"
-version = "0.2.1"
+version = "0.4.0"
 source = { editable = "." }
 dependencies = [
     { name = "httpx" },

lql_cli-0.3.0/src/lql/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.0"