teich 0.2.3__tar.gz → 0.2.4__tar.gz

{teich-0.2.3 → teich-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: teich
-Version: 0.2.3
+Version: 0.2.4
 Summary: Turn coding agent traces into auditable supervised fine-tuning data
 License: Apache-2.0
 License-File: LICENSE

{teich-0.2.3 → teich-0.2.4}/docs/studio.md

@@ -43,6 +43,7 @@ Studio gives you browser controls for the same files and runners used by the CLI
 - edit `config.yaml`
 - add, edit, import, and validate `prompts.jsonl`
 - start and resume batch generation
+- extract existing local agent sessions from browser-selected source paths
 - watch generation progress and provider/model usage
 - inspect generated traces
 - open interactive agent sessions
@@ -81,6 +82,42 @@ teich generate -c config.yaml
 
 Use it to start a batch, monitor status, and resume a run without leaving the browser.
 
+## Browser Extraction
+
+The Extract tab is a UI wrapper over:
+
+```bash
+teich extract PROVIDER --sessions-dir PATH --out ./output
+```
+
+Use it to stage existing local sessions from `claude`, `codex`, `pi`, or `hermes` without leaving the browser. The source box accepts either the provider home folder or the provider's direct data path:
+
+- Claude: `.claude` or `.claude/projects`
+- Codex: `.codex` or `.codex/sessions`
+- Pi: `.pi`, `.pi/agent/sessions`, or `.pi/sessions`
+- Hermes: `.hermes` or `.hermes/state.db`
+
+Studio can fill in detected default paths for the selected provider. Extraction writes into the configured output folder, generates a dataset `README.md`, and anonymizes staged traces by default. Check **Skip anonymization** only when you are intentionally keeping raw local values.
+
+## Dataset Preview
+
+The Dataset Preview tab lets you inspect the configured output folder before upload. It shows:
+
+- JSONL files and row counts
+- Hugging Face-style features for the converted training rows
+- searchable row previews
+- row JSON details
+- local session/trace previews with messages, reasoning, tool calls, and tool results
+- the generated dataset card preview when `README.md` is present
+
+If `publish.repo_id` is set in `config.yaml`, Studio also exposes the official Hugging Face embedded Dataset Viewer URL:
+
+```text
+https://huggingface.co/datasets/<owner>/<dataset>/embed/viewer
+```
+
+That official embed works for datasets already available on the Hub. For unpublished local output, Studio uses Teich's local converter to approximate the parts of the viewer that matter before upload. The full Hugging Face viewer backend is hosted by Hugging Face and adds Parquet-backed row serving, search, filtering, SQL, and statistics after the dataset is uploaded and processed.
+
 ## Requirements
 
 For agent providers:

{teich-0.2.3 → teich-0.2.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "teich"
-version = "0.2.3"
+version = "0.2.4"
 description = "Turn coding agent traces into auditable supervised fine-tuning data"
 readme = "README.md"
 license = {text = "Apache-2.0"}

{teich-0.2.3 → teich-0.2.4}/src/teich/__init__.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-__version__ = "0.2.3"
+__version__ = "0.2.4"
 
 from .audit import SFTAuditReport, audit_sft_dataset
 from .config import Config, load_config

{teich-0.2.3 → teich-0.2.4}/src/teich/formatter.py

@@ -1,10 +1,9 @@
 from __future__ import annotations
 
-from copy import deepcopy
-from dataclasses import dataclass, field
-import difflib
-import json
-import re
+from copy import deepcopy
+from dataclasses import dataclass, field
+import json
+import re
 from collections.abc import Mapping, Sequence
 from typing import Any
 
@@ -623,11 +622,11 @@ def _wrap_with_markers(value: Any, start_marker: str, end_marker: str) -> tuple[
     return updated_value, True
 
 
-def _mark_supervised_messages(
-    messages: list[dict[str, Any]],
-    *,
-    include_context_spans: bool = True,
-) -> tuple[list[dict[str, Any]], list[dict[str, str]]]:
+def _mark_supervised_messages(
+    messages: list[dict[str, Any]],
+    *,
+    include_context_spans: bool = True,
+) -> tuple[list[dict[str, Any]], list[dict[str, str]]]:
     marked_messages = deepcopy(messages)
     markers: list[dict[str, str]] = []
     marker_index = 0
@@ -692,24 +691,31 @@ def _mark_supervised_messages(
         updated_content, changed = mark_value(content, kind=kind, role=role)
         if changed:
             message["content"] = updated_content
-    return marked_messages, markers
-
-
-def _strip_markers_and_collect_spans(text: str, markers: list[dict[str, str]]) -> tuple[str, list[dict[str, Any]]] | None:
-    if not markers:
-        return text, []
-    marker_lookup: dict[str, tuple[str, int]] = {}
-    pattern_parts: list[str] = []
-    for index, marker in enumerate(markers):
-        start_marker = marker["start_marker"]
-        end_marker = marker["end_marker"]
-        marker_lookup[start_marker] = ("start", index)
-        marker_lookup[end_marker] = ("end", index)
-        pattern_parts.append(re.escape(start_marker))
-        pattern_parts.append(re.escape(end_marker))
+    return marked_messages, markers
+
+
+def _marker_text_variants(marker: str) -> tuple[str, ...]:
+    escaped = json.dumps(marker)[1:-1]
+    return (marker, escaped) if escaped != marker else (marker,)
+
+
+def _strip_markers_and_collect_spans(text: str, markers: list[dict[str, str]]) -> tuple[str, list[dict[str, Any]]] | None:
+    if not markers:
+        return text, []
+    marker_lookup: dict[str, tuple[str, int]] = {}
+    pattern_parts: list[str] = []
+    for index, marker in enumerate(markers):
+        start_marker = marker["start_marker"]
+        end_marker = marker["end_marker"]
+        for marker_variant in _marker_text_variants(start_marker):
+            marker_lookup[marker_variant] = ("start", index)
+            pattern_parts.append(re.escape(marker_variant))
+        for marker_variant in _marker_text_variants(end_marker):
+            marker_lookup[marker_variant] = ("end", index)
+            pattern_parts.append(re.escape(marker_variant))
     pattern = re.compile("|".join(pattern_parts))
     cleaned_parts: list[str] = []
-    active_starts: dict[int, int] = {}
+    active_starts: dict[int, list[int]] = {}
     spans: list[dict[str, Any]] = []
     cursor = 0
     cleaned_length = 0
@@ -720,15 +726,19 @@ def _strip_markers_and_collect_spans(text: str, markers: list[dict[str, str]]) -
             cleaned_length += len(chunk)
         marker = match.group(0)
         kind, index = marker_lookup[marker]
-        if kind == "start":
-            active_starts[index] = cleaned_length
-        else:
-            start = active_starts.pop(index, None)
-            if start is None:
-                return None
-            if start < cleaned_length:
-                marker = markers[index]
-                spans.append(
+        if kind == "start":
+            active_starts.setdefault(index, []).append(cleaned_length)
+        else:
+            starts = active_starts.get(index)
+            if not starts:
+                cursor = match.end()
+                continue
+            start = starts.pop()
+            if not starts:
+                active_starts.pop(index, None)
+            if start < cleaned_length:
+                marker = markers[index]
+                spans.append(
                     {
                         "start": start,
                         "end": cleaned_length,
@@ -742,21 +752,19 @@ def _strip_markers_and_collect_spans(text: str, markers: list[dict[str, str]]) -
     tail = text[cursor:]
     if tail:
         cleaned_parts.append(tail)
-    if active_starts:
-        return None
-    cleaned_text = "".join(cleaned_parts)
-    return cleaned_text, spans
+    cleaned_text = "".join(cleaned_parts)
+    return cleaned_text, spans
 
 
 def _range_touches_span_boundary(start: int, end: int, spans: list[dict[str, Any]]) -> bool:
     for span in spans:
-        span_start = span["start"]
-        span_end = span["end"]
-        if start == end and (start == span_start or start == span_end):
-            return True
-        if span_start <= start and end <= span_end and (start == span_start or end == span_end):
-            return True
-    return False
+        span_start = span["start"]
+        span_end = span["end"]
+        if start == end and (start == span_start or start == span_end):
+            return True
+        if span_start <= start and end <= span_end:
+            return True
+    return False
 
 
 def _shift_span_for_whitespace_edit(
@@ -806,33 +814,49 @@ def _shift_span_for_whitespace_edit(
     return updated
 
 
-def _reconcile_marker_boundary_whitespace(
-    text: str,
-    spans: list[dict[str, Any]],
-    target_text: str,
-) -> tuple[str, list[dict[str, Any]]] | None:
-    if text == target_text:
-        return text, spans
-    matcher = difflib.SequenceMatcher(a=text, b=target_text, autojunk=False)
-    edits: list[tuple[int, int, str]] = []
-    for tag, start_a, end_a, start_b, end_b in matcher.get_opcodes():
-        if tag == "equal":
-            continue
-        if tag not in {"delete", "insert", "replace"}:
-            return None
-        removed = text[start_a:end_a]
-        inserted = target_text[start_b:end_b]
-        if removed and not removed.isspace():
-            return None
-        if inserted and not inserted.isspace():
-            return None
-        if not removed and not inserted:
-            return None
-        if not _range_touches_span_boundary(start_a, end_a, spans):
-            return None
-        edits.append((start_a, end_a, inserted))
-    if not edits:
-        return None
+def _reconcile_marker_boundary_whitespace(
+    text: str,
+    spans: list[dict[str, Any]],
+    target_text: str,
+) -> tuple[str, list[dict[str, Any]]] | None:
+    if text == target_text:
+        return text, spans
+    edits: list[tuple[int, int, str]] = []
+
+    text_index = 0
+    target_index = 0
+    while text_index < len(text) or target_index < len(target_text):
+        text_space_start = text_index
+        while text_index < len(text) and text[text_index].isspace():
+            text_index += 1
+        target_space_start = target_index
+        while target_index < len(target_text) and target_text[target_index].isspace():
+            target_index += 1
+
+        removed = text[text_space_start:text_index]
+        inserted = target_text[target_space_start:target_index]
+        if removed != inserted:
+            if not _range_touches_span_boundary(text_space_start, text_index, spans):
+                return None
+            edits.append((text_space_start, text_index, inserted))
+
+        while (
+            text_index < len(text)
+            and target_index < len(target_text)
+            and not text[text_index].isspace()
+            and not target_text[target_index].isspace()
+        ):
+            if text[text_index] != target_text[target_index]:
+                return None
+            text_index += 1
+            target_index += 1
+
+        if text_index >= len(text) and target_index < len(target_text) and not target_text[target_index].isspace():
+            return None
+        if target_index >= len(target_text) and text_index < len(text) and not text[text_index].isspace():
+            return None
+    if not edits:
+        return None
     adjusted_text = text
     adjusted_spans = [dict(span) for span in spans]
     for start, end, inserted in reversed(edits):
@@ -1345,11 +1369,14 @@ def _supervised_text_and_spans(
         )
         if inferred_spans:
             return formatted_text, _span_dicts(inferred_spans)
-    gemma_spans = _gemma_like_supervised_spans(formatted_text)
-    if gemma_spans:
-        supervised_spans.extend(
-            {
-                "start": start,
+    gemma_spans = _gemma_like_supervised_spans(formatted_text)
+    if gemma_spans:
+        gemma_spans = _subtract_spans(gemma_spans, _tool_call_spans(formatted_text))
+        gemma_spans = _subtract_spans(gemma_spans, _tool_response_spans(formatted_text))
+        gemma_spans = _subtract_spans(gemma_spans, _reasoning_spans(formatted_text))
+        supervised_spans.extend(
+            {
+                "start": start,
                 "end": end,
                 "source_start": start,
                 "source_end": end,

teich-0.2.4/src/teich/studio/dataset_preview.py

@@ -0,0 +1,274 @@
+"""Local Hugging Face-style dataset previews for Teich Studio."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from ..converter import convert_traces_to_training_data
+from ..loader import trace_is_complete
+from .events import summarize_chat_row, summarize_trace_events
+
+MAX_README_CHARS = 24_000
+MAX_TRACE_FILES = 8
+MAX_TRACE_EVENTS = 5_000
+MAX_TRACE_DISPLAY = 80
+
+
+def _jsonl_files(root: Path) -> list[Path]:
+    if root.is_file() and root.suffix.lower() == ".jsonl":
+        return [root]
+    if not root.is_dir():
+        return []
+    files: list[Path] = []
+    for path in sorted(root.rglob("*.jsonl")):
+        if not path.is_file():
+            continue
+        relative_parts = path.relative_to(root).parts
+        if any(part in {"failures", "partials"} for part in relative_parts):
+            continue
+        files.append(path)
+    return files
+
+
+def _line_count(path: Path) -> int:
+    try:
+        with path.open("r", encoding="utf-8") as handle:
+            return sum(1 for line in handle if line.strip())
+    except OSError:
+        return 0
+
+
+def _feature_type(value: Any) -> dict[str, Any]:
+    if isinstance(value, str):
+        return {"dtype": "string", "_type": "Value"}
+    if isinstance(value, bool):
+        return {"dtype": "bool", "_type": "Value"}
+    if isinstance(value, int):
+        return {"dtype": "int64", "_type": "Value"}
+    if isinstance(value, float):
+        return {"dtype": "float64", "_type": "Value"}
+    if isinstance(value, list):
+        nested = _feature_type(value[0]) if value else {"dtype": "null", "_type": "Value"}
+        return {"feature": nested, "_type": "Sequence"}
+    if isinstance(value, dict):
+        return {key: _feature_type(item) for key, item in sorted(value.items())}
+    if value is None:
+        return {"dtype": "null", "_type": "Value"}
+    return {"dtype": type(value).__name__, "_type": "Value"}
+
+
+def _features(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    columns: dict[str, Any] = {}
+    for row in rows:
+        for key, value in row.items():
+            columns.setdefault(key, value)
+    return [
+        {"feature_idx": index, "name": name, "type": _feature_type(value)}
+        for index, (name, value) in enumerate(sorted(columns.items()))
+    ]
+
+
+def _stringify_for_search(row: dict[str, Any]) -> str:
+    return json.dumps(row, ensure_ascii=False, sort_keys=True).casefold()
+
+
+def _row_preview(row: dict[str, Any]) -> dict[str, Any]:
+    messages = row.get("messages") if isinstance(row.get("messages"), list) else []
+    tools = row.get("tools") if isinstance(row.get("tools"), list) else []
+    metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
+    return {
+        "prompt": row.get("prompt") or _first_message_text(messages, "user"),
+        "response": row.get("response") or _last_message_text(messages, "assistant"),
+        "model": row.get("model") or metadata.get("model"),
+        "message_count": len(messages),
+        "tool_count": len(tools),
+        "trace_type": metadata.get("trace_type"),
+        "complete": trace_is_complete(row),
+    }
+
+
+def _first_message_text(messages: list[Any], role: str) -> str | None:
+    for message in messages:
+        if isinstance(message, dict) and message.get("role") == role:
+            content = message.get("content")
+            return content if isinstance(content, str) else None
+    return None
+
+
+def _last_message_text(messages: list[Any], role: str) -> str | None:
+    for message in reversed(messages):
+        if isinstance(message, dict) and message.get("role") == role:
+            content = message.get("content")
+            return content if isinstance(content, str) else None
+    return None
+
+
+def _column_statistics(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    columns = sorted({key for row in rows for key in row.keys()})
+    stats: list[dict[str, Any]] = []
+    for column in columns:
+        values = [row.get(column) for row in rows if column in row]
+        scalar_counts: dict[str, int] = {}
+        for value in values:
+            if isinstance(value, str | int | float | bool) or value is None:
+                label = "null" if value is None else str(value)
+                scalar_counts[label] = scalar_counts.get(label, 0) + 1
+        top_values = sorted(scalar_counts.items(), key=lambda item: (-item[1], item[0]))[:5]
+        stats.append(
+            {
+                "name": column,
+                "present": len(values),
+                "missing": max(len(rows) - len(values), 0),
+                "type": _feature_type(values[0]) if values else {"dtype": "null", "_type": "Value"},
+                "top_values": [{"value": value, "count": count} for value, count in top_values],
+            }
+        )
+    return stats
+
+
+def _detect_trace_provider(events: list[dict[str, Any]]) -> str:
+    for event in events[:5]:
+        event_type = event.get("type")
+        if event_type == "session_meta":
+            return "codex"
+        if event_type == "session":
+            return "pi"
+        if event_type == "external_session_meta":
+            return "hermes"
+        if event_type in {"response_item", "event_msg"}:
+            return "codex"
+        if event_type == "message" and isinstance(event.get("message"), dict):
+            return "pi"
+        if "sessionId" in event or event_type == "queue-operation":
+            return "claude-code"
+        if isinstance(event.get("messages"), list):
+            return "chat"
+    return "unknown"
+
+
+def _read_trace_events(path: Path) -> list[dict[str, Any]]:
+    events: list[dict[str, Any]] = []
+    try:
+        with path.open("r", encoding="utf-8-sig") as handle:
+            for line in handle:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    event = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                if isinstance(event, dict):
+                    events.append(event)
+                if len(events) >= MAX_TRACE_EVENTS:
+                    break
+    except OSError:
+        return []
+    return events
+
+
+def _trace_preview(root: Path, path: Path) -> dict[str, Any]:
+    events = _read_trace_events(path)
+    provider = _detect_trace_provider(events)
+    if provider == "chat":
+        display: list[dict[str, Any]] = []
+        for row in events:
+            display.extend(summarize_chat_row(row))
+    else:
+        display = summarize_trace_events(provider, events)
+    try:
+        name = path.relative_to(root).as_posix()
+    except ValueError:
+        name = path.name
+    return {
+        "name": name,
+        "provider": provider,
+        "event_count": len(events),
+        "display": display[:MAX_TRACE_DISPLAY],
+        "truncated": len(display) > MAX_TRACE_DISPLAY,
+    }
+
+
+def _readme_text(root: Path) -> str | None:
+    readme = root / "README.md" if root.is_dir() else root.parent / "README.md"
+    if not readme.exists():
+        return None
+    try:
+        text = readme.read_text(encoding="utf-8")
+    except OSError:
+        return None
+    return text[:MAX_README_CHARS]
+
+
+def build_dataset_preview(
+    root: Path,
+    *,
+    repo_id: str | None = None,
+    offset: int = 0,
+    limit: int = 100,
+    search: str | None = None,
+) -> dict[str, Any]:
+    root = root.expanduser().resolve()
+    if not root.exists():
+        raise FileNotFoundError(f"Dataset output path not found: {root}")
+
+    offset = max(offset, 0)
+    limit = max(1, min(limit, 100))
+    trace_files = _jsonl_files(root)
+    file_rows = [
+        {
+            "name": path.relative_to(root).as_posix() if root.is_dir() else path.name,
+            "size_bytes": path.stat().st_size,
+            "rows": _line_count(path),
+        }
+        for path in trace_files
+    ]
+
+    errors: list[str] = []
+    try:
+        rows = convert_traces_to_training_data(root)
+    except Exception as exc:
+        rows = []
+        errors.append(str(exc))
+
+    query = (search or "").strip().casefold()
+    indexed_rows = list(enumerate(rows))
+    if query:
+        indexed_rows = [(index, row) for index, row in indexed_rows if query in _stringify_for_search(row)]
+    page = indexed_rows[offset:offset + limit]
+    page_rows = [
+        {"row_idx": index, "row": row, "preview": _row_preview(row)}
+        for index, row in page
+    ]
+    selected_rows = [row for _, row in indexed_rows]
+    complete_rows = sum(1 for row in selected_rows if trace_is_complete(row))
+
+    return {
+        "root": str(root),
+        "repo_id": repo_id,
+        "hf_embed_url": f"https://huggingface.co/datasets/{repo_id}/embed/viewer" if repo_id else None,
+        "splits": [{"config": "default", "split": "train"}],
+        "files": file_rows,
+        "readme": _readme_text(root),
+        "dataset": {
+            "config": "default",
+            "split": "train",
+            "num_rows": len(selected_rows),
+            "total_rows": len(rows),
+            "offset": offset,
+            "length": len(page_rows),
+            "features": _features(selected_rows),
+            "rows": page_rows,
+            "search": search or "",
+            "complete_rows": complete_rows,
+            "incomplete_rows": max(len(selected_rows) - complete_rows, 0),
+        },
+        "statistics": _column_statistics(selected_rows),
+        "trace_previews": [_trace_preview(root, path) for path in trace_files[:MAX_TRACE_FILES]],
+        "errors": errors,
+        "notes": [
+            "Local preview uses Teich conversion directly. Hugging Face's hosted viewer adds Parquet-backed search, filtering, and SQL after upload."
+        ],
+    }