docworkspace 0.2.7__tar.gz → 0.2.9__tar.gz

{docworkspace-0.2.7 → docworkspace-0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docworkspace
-Version: 0.2.7
+Version: 0.2.9
 Summary: A workspace library for managing Polars dataframes with parent-child relationships and lazy evaluation
 Requires-Python: >=3.14
 Requires-Dist: polars-text>=0.1.6

{docworkspace-0.2.7 → docworkspace-0.2.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "docworkspace"
-version = "0.2.7"
+version = "0.2.9"
 description = "A workspace library for managing Polars dataframes with parent-child relationships and lazy evaluation"
 readme = "README.md"
 requires-python = ">=3.14"

{docworkspace-0.2.7 → docworkspace-0.2.9}/src/docworkspace/__init__.py

@@ -4,8 +4,8 @@ Provides backward compatibility for original imports while exposing
 serialization, analysis, and graph helpers in dedicated submodules.
 """
 
-from .node import Node  # package exposing Node
+from .node import DerivedColumnMeta, Node  # package exposing Node
 from .workspace import Workspace  # shim -> workspace.core.Workspace
 
-__version__ = "0.2.7"
-__all__ = ["Workspace", "Node"]
+__version__ = "0.2.9"
+__all__ = ["Workspace", "Node", "DerivedColumnMeta"]

{docworkspace-0.2.7 → docworkspace-0.2.9}/src/docworkspace/node/__init__.py

@@ -4,7 +4,7 @@ Operations are provided as instance methods on ``Node`` while persistence is
 handled by the dedicated ``docworkspace.node.io`` module.
 """
 
-from .core import Node
+from .core import DerivedColumnMeta, Node
 from .io import dumps, from_dict, loads, to_dict
 
-__all__ = ["Node", "to_dict", "from_dict", "dumps", "loads"]
+__all__ = ["Node", "DerivedColumnMeta", "to_dict", "from_dict", "dumps", "loads"]

{docworkspace-0.2.7 → docworkspace-0.2.9}/src/docworkspace/node/core.py

@@ -8,7 +8,17 @@ from __future__ import annotations
 
 import uuid
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Sequence, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Literal,
+    Mapping,
+    Optional,
+    Sequence,
+    TypedDict,
+    cast,
+)
 
 import polars as pl
 
@@ -16,6 +26,23 @@ if TYPE_CHECKING:  # pragma: no cover
     from ..workspace.core import Workspace  # pragma: no cover
 
 
+class DerivedColumnMeta(TypedDict):
+    """Metadata for a hidden derived analytic column (Phase 2, decision 7).
+
+    Derived columns live alongside the user's columns in the same LazyFrame
+    but are stripped from frontend-facing schema projections. ``source_column``
+    points at the originating user column; ``form`` says what kind of derivation
+    (``"tokens"``, future: ``"pos"``, ``"ner"``); ``model`` identifies the
+    backend that produced it (``"jieba"``, ``"bert-base-uncased"``, ...).
+    """
+
+    source_column: str
+    form: str
+    model: str
+    language: Optional[str]
+    generated_at: str
+
+
 class Node:
     MAX_UNDO_DEPTH = 50
 
@@ -33,6 +60,7 @@ class Node:
         operation: str | None = None,
         id: str | None = None,
         document: str | None = None,
+        derived: Optional[Mapping[str, DerivedColumnMeta]] = None,
     ) -> None:
         self.id = id or str(uuid.uuid4())
         self.name = name or f"node_{self.id[:8]}"
@@ -46,6 +74,14 @@ class Node:
         self._redo_stack: list[pl.LazyFrame] = []
         self._data: pl.LazyFrame = data
         self._document_column: Optional[str] = document
+        # Per-column metadata for hidden derived analytic columns (Phase 2,
+        # decision 7). Keys are the derived column names that exist in the
+        # LazyFrame schema (e.g. "__derived__.tokens.text.jieba"); values
+        # carry source_column / form / model / language / generated_at.
+        # Empty dict on legacy nodes is fully backward compatible.
+        self.derived: dict[str, DerivedColumnMeta] = (
+            {k: dict(v) for k, v in derived.items()} if derived else {}
+        )  # type: ignore[assignment]
         self.parents: list[Node | str] = list(parents)
         self.workspace: Optional[Workspace] = workspace
         self.operation = operation
@@ -75,6 +111,7 @@ class Node:
                     workspace=self.workspace,
                     parents=[self],
                     operation=item,
+                    derived=self.derived,
                 )
                 if self.document:
                     child.document = self.document
@@ -138,16 +175,24 @@ class Node:
             workspace=self.workspace,
             parents=[self],
             operation="filter",
+            derived=self.derived,
         )
 
     def select(self, *exprs: Any, **named_exprs: Any) -> "Node":
         result = self.data.select(*exprs, **named_exprs)
+        # User-driven select may drop derived columns from the schema; keep
+        # only the derived metadata entries whose column still exists.
+        result_columns = set(result.collect_schema().names())
+        retained_derived = {
+            name: meta for name, meta in self.derived.items() if name in result_columns
+        }
         return Node(
             data=result,
             name=f"select_{self.name}",
             workspace=self.workspace,
             parents=[self],
             operation="select",
+            derived=retained_derived,
         )
 
     def join(
@@ -167,12 +212,21 @@ class Node:
         **kwargs: Any,
     ) -> "Node":
         result = self.data.join(other.data, on=on, how=how, **kwargs)
+        # Union derived metadata from both sides; result column set may drop
+        # entries if join columns collide, so filter to the resulting schema.
+        result_columns = set(result.collect_schema().names())
+        merged: dict[str, DerivedColumnMeta] = {}
+        for source in (self.derived, other.derived):
+            for name, meta in source.items():
+                if name in result_columns:
+                    merged[name] = meta
         return Node(
             data=result,
             name=f"join_{self.name}_{other.name}",
             workspace=self.workspace,
             parents=[self, other],
             operation=f"join({how})",
+            derived=merged,
         )
 
     def slice(self, offset: int, length: int | None = None) -> "Node":
@@ -183,6 +237,7 @@ class Node:
             workspace=self.workspace,
             parents=[self],
             operation="slice",
+            derived=self.derived,
         )
 
     def drop(
@@ -194,19 +249,36 @@ class Node:
         """Drop columns using Polars semantics and return a child node.
 
         Mirrors ``polars.LazyFrame.drop`` while preserving DocWorkspace lineage.
+        Cascade rule (decision 7): when a user column is dropped, any derived
+        columns whose ``source_column`` matched are also dropped and removed
+        from ``Node.derived``.
         """
+        before_names = set(self.data.collect_schema().names())
         result = self.data.drop(columns, *more_columns, strict=strict)
+        after_names = set(result.collect_schema().names())
+        dropped_sources = before_names - after_names
+
+        cascade_targets: list[str] = []
+        retained_derived: dict[str, DerivedColumnMeta] = {}
+        for derived_name, meta in self.derived.items():
+            if meta["source_column"] in dropped_sources and derived_name in after_names:
+                cascade_targets.append(derived_name)
+            elif derived_name in after_names:
+                retained_derived[derived_name] = meta
+
+        if cascade_targets:
+            result = result.drop(*cascade_targets, strict=False)
+
         child = Node(
             data=result,
             name=f"drop_{self.name}",
             workspace=self.workspace,
             parents=[self],
             operation="drop",
+            derived=retained_derived,
         )
 
         if self.document:
-            before_names = set(self.data.collect_schema().names())
-            after_names = set(result.collect_schema().names())
             if self.document in before_names and self.document not in after_names:
                 child.document = None
             else:
@@ -215,8 +287,30 @@ class Node:
         return child
 
     def rename(self, mapping: Any, *, strict: bool = True) -> "Node":
-        """Rename columns in-place using Polars semantics and return this node."""
-        self.data = self.data.rename(mapping, strict=strict)
+        """Rename columns in-place using Polars semantics and return this node.
+
+        Cascade rule (decision 7): renaming a source column makes any derived
+        columns referencing it stale — they are dropped from the LazyFrame and
+        from ``Node.derived``. Users can re-tokenise after the rename.
+        """
+        before_names = set(self.data.collect_schema().names())
+        new_data = self.data.rename(mapping, strict=strict)
+        after_names = set(new_data.collect_schema().names())
+        renamed_sources = before_names - after_names
+
+        if self.derived and renamed_sources:
+            cascade_targets = [
+                derived_name
+                for derived_name, meta in self.derived.items()
+                if meta["source_column"] in renamed_sources
+                and derived_name in after_names
+            ]
+            if cascade_targets:
+                new_data = new_data.drop(*cascade_targets, strict=False)
+                for name in cascade_targets:
+                    self.derived.pop(name, None)
+
+        self.data = new_data
 
         if self.document:
             new_document = self.document
@@ -273,6 +367,49 @@ class Node:
     def can_redo(self) -> bool:
         return len(self._redo_stack) > 0
 
+    # ------------------------------------------------------------------
+    # Derived-column metadata (Phase 2, decision 7)
+    # ------------------------------------------------------------------
+    def register_derived_column(
+        self, column_name: str, meta: DerivedColumnMeta
+    ) -> None:
+        """Record metadata for a hidden derived column on this node.
+
+        Caller is responsible for ensuring ``column_name`` exists in the
+        node's LazyFrame schema (typically after a ``with_columns(...)`` that
+        adds it). This method only writes the metadata index.
+        """
+        self.derived[column_name] = dict(meta)  # type: ignore[assignment]
+
+    def unregister_derived_column(self, column_name: str) -> bool:
+        """Remove the metadata entry for ``column_name``. Does not touch the
+        LazyFrame schema. Returns True if an entry was removed.
+        """
+        return self.derived.pop(column_name, None) is not None
+
+    def find_derived_column(
+        self,
+        source_column: str,
+        *,
+        form: str = "tokens",
+        model: str | None = None,
+    ) -> str | None:
+        """Return the name of a derived column for ``source_column``, or None.
+
+        Filters by ``form`` (default ``"tokens"``); if ``model`` is given,
+        further narrows to that backend. When multiple candidates match,
+        returns the first by insertion order.
+        """
+        for name, meta in self.derived.items():
+            if meta.get("source_column") != source_column:
+                continue
+            if meta.get("form") != form:
+                continue
+            if model is not None and meta.get("model") != model:
+                continue
+            return name
+        return None
+
     # ------------------------------------------------------------------
     # Schema utilities
     # ------------------------------------------------------------------
@@ -332,4 +469,4 @@ class Node:
         )
 
 
-__all__ = ["Node"]
+__all__ = ["Node", "DerivedColumnMeta"]

{docworkspace-0.2.7 → docworkspace-0.2.9}/src/docworkspace/node/io.py

@@ -47,6 +47,7 @@ def to_dict(node: Node, *, base_dir: str | Path | None = None) -> dict[str, Any]
             "name": node.name,
             "operation": node.operation,
             "document": node.document,
+            "derived": {name: dict(meta) for name, meta in node.derived.items()},
             "parents": [node._parent_id(parent) for parent in node.parents],
         },
         "data_path": rel_data_path.as_posix(),
@@ -64,6 +65,9 @@ def from_dict(
     node_metadata = dict(payload["node_metadata"])
     data_path = Path(str(payload["data_path"]))
     parent_ids = node_metadata.pop("parents", [])
+    # Legacy workspaces persisted before Phase 2 won't have ``derived`` at
+    # all; default to empty so loading stays backward compatible.
+    node_metadata.setdefault("derived", {})
 
     if workspace is not None:
         root_dir = Path(workspace.ws_root_dir)

{docworkspace-0.2.7 → docworkspace-0.2.9}/src/docworkspace/workspace/analysis.py

@@ -34,7 +34,20 @@ def graph_json(workspace: "Workspace") -> Dict[str, object]:
     edges_payload: List[Dict[str, str]] = []
 
     for node in workspace.nodes.values():
-        nodes_payload.append(node.info())
+        try:
+            nodes_payload.append(node.info())
+        except Exception as exc:
+            # Per-node fallback: one broken node (e.g. missing source file,
+            # undeserializable lazy plan) must not take down the whole graph.
+            nodes_payload.append(
+                {
+                    "id": node.id,
+                    "name": getattr(node, "name", node.id),
+                    "operation": getattr(node, "operation", "unknown"),
+                    "child_ids": [c.id for c in getattr(node, "children", [])],
+                    "error": f"{type(exc).__name__}: {exc}",
+                }
+            )
 
         for child in node.children:
             edges_payload.append({"source": node.id, "target": child.id})

{docworkspace-0.2.7 → docworkspace-0.2.9}/tests/test_fastapi_integration.py

@@ -19,8 +19,8 @@ class TestCoreLibraryIndependence:
         """Test that core library only exports core functionality."""
         from docworkspace import __all__
 
-        # Core library should only export Node and Workspace
-        expected_exports = {"Node", "Workspace"}
+        # Core library should only export Node, Workspace, and their typed helpers
+        expected_exports = {"Node", "Workspace", "DerivedColumnMeta"}
         actual_exports = set(__all__)
 
         assert actual_exports == expected_exports, (

{docworkspace-0.2.7 → docworkspace-0.2.9}/tests/test_node.py

@@ -1,12 +1,12 @@
 """Tests for the Node class."""
 
 from inspect import signature
-from typing import Optional, Sequence, cast, get_type_hints
+from typing import Mapping, Optional, Sequence, cast, get_type_hints
 
 import polars as pl
 import pytest
 
-from docworkspace import Node, Workspace
+from docworkspace import DerivedColumnMeta, Node, Workspace
 
 
 class TestNode:
@@ -63,6 +63,8 @@ class TestNode:
                 "Workspace": Workspace,
                 "Sequence": Sequence,
                 "Optional": Optional,
+                "Mapping": Mapping,
+                "DerivedColumnMeta": DerivedColumnMeta,
                 "pl": pl,
             },
         )
@@ -400,3 +402,36 @@ class TestNodeRelationships:
         assert merged in parent1.children
         assert merged in parent2.children
         assert merged in parent2.children
+
+
+def test_node_shape_does_not_materialise_list_columns():
+    """Phase 2.8: Node.shape must compute height without scanning list columns.
+
+    A future change that pushes Node.shape towards full collect() (e.g. via
+    .height instead of .select(pl.len())) would break tokenised nodes by
+    forcing the List[Struct] column to be materialised. Bound it loosely at
+    100ms on a 50k-row tokenised-style frame.
+    """
+    import time
+
+    N = 50_000
+    tokens_per_doc = 30
+    tokens_struct = [
+        {"token": f"t{i}", "start": i * 5, "end": i * 5 + 4}
+        for i in range(tokens_per_doc)
+    ]
+    df = pl.DataFrame(
+        {
+            "text": [f"doc {i} " * 5 for i in range(N)],
+            "TOKENS_tokens": [tokens_struct] * N,
+        }
+    )
+    node = Node(data=df.lazy(), name="bench")
+
+    start = time.perf_counter()
+    shape = node.shape
+    elapsed = time.perf_counter() - start
+
+    assert shape == (N, 2)
+    # Generous bound — typical observed time on dev hardware is < 1ms.
+    assert elapsed < 0.1, f"Node.shape took {elapsed*1000:.1f}ms; suspect materialisation regression"

{docworkspace-0.2.7 → docworkspace-0.2.9}/tests/test_node_io.py

@@ -30,6 +30,7 @@ def test_node_to_dict_persists_lazyframe_payload(tmp_path: Path):
             "name": "root",
             "operation": "source",
             "document": "text",
+            "derived": {},
             "parents": [],
         },
         "data_path": f"data/{node.id}.plbin",
@@ -233,3 +234,176 @@ def test_node_from_dict_ignores_missing_parent_ids(tmp_path: Path):
 
     assert restored.parents == []
     assert restored.parents == []
+
+
+def test_node_derived_metadata_round_trip(tmp_path: Path):
+    """Phase 2.4 v2: Node.derived survives to_dict / from_dict."""
+    workspace = Workspace("node_io_derived")
+    workspace.ws_root_dir = tmp_path
+    derived_name = "__derived__.tokens.text.jieba"
+    meta = {
+        "source_column": "text",
+        "form": "tokens",
+        "model": "jieba",
+        "language": "zh",
+        "generated_at": "2026-05-12T00:00:00+00:00",
+    }
+    node = workspace.add_node(
+        Node(
+            data=pl.DataFrame({"text": ["今天天气很好"]}).lazy(),
+            name="zh_root",
+            workspace=workspace,
+            operation="source",
+            derived={derived_name: meta},  # type: ignore[arg-type]
+        )
+    )
+    node.document = "text"
+
+    payload = to_dict(node, base_dir=tmp_path)
+    assert payload["node_metadata"]["derived"] == {derived_name: meta}
+
+    # Round-trip into a fresh workspace
+    workspace2 = Workspace("node_io_derived_loaded")
+    workspace2.ws_root_dir = tmp_path
+    restored = from_dict(payload, workspace=workspace2)
+    assert restored.derived == {derived_name: meta}
+    assert restored.find_derived_column("text") == derived_name
+    assert restored.find_derived_column("text", model="jieba") == derived_name
+    assert restored.find_derived_column("text", model="other-model") is None
+
+
+def test_node_legacy_payload_without_derived_loads_with_empty_dict(
+    tmp_path: Path,
+):
+    """Backward compat: workspaces persisted before Phase 2 lacking ``derived``
+    must still load, defaulting it to an empty dict."""
+    workspace = Workspace("legacy_node_io")
+    workspace.ws_root_dir = tmp_path
+    node = workspace.add_node(
+        Node(
+            data=pl.DataFrame({"text": ["legacy"]}).lazy(),
+            name="legacy_root",
+            workspace=workspace,
+            operation="source",
+        )
+    )
+
+    # Build a "legacy" payload — strip the new field the way old files would.
+    payload = to_dict(node, base_dir=tmp_path)
+    legacy_metadata = dict(payload["node_metadata"])
+    legacy_metadata.pop("derived", None)
+    legacy_payload = {**payload, "node_metadata": legacy_metadata}
+
+    workspace2 = Workspace("legacy_loaded")
+    workspace2.ws_root_dir = tmp_path
+    restored = from_dict(legacy_payload, workspace=workspace2)
+    assert restored.derived == {}
+
+
+def test_node_derived_propagates_through_getattr(tmp_path: Path):
+    """Phase 2.4 v2: Node.derived propagates to children spawned by delegated
+    LazyFrame methods (schema-preserving ops like .head / .sort)."""
+    workspace = Workspace("derive_propagate")
+    workspace.ws_root_dir = tmp_path
+    derived_name = "__derived__.tokens.text.jieba"
+    meta = {
+        "source_column": "text",
+        "form": "tokens",
+        "model": "jieba",
+        "language": "zh",
+        "generated_at": "2026-05-12T00:00:00+00:00",
+    }
+    parent = workspace.add_node(
+        Node(
+            data=pl.DataFrame({"text": ["a", "b", "c"]}).lazy(),
+            name="zh_parent",
+            workspace=workspace,
+            operation="source",
+            derived={derived_name: meta},  # type: ignore[arg-type]
+        )
+    )
+    parent.document = "text"
+    child = parent.head(2)
+    assert child.derived == {derived_name: meta}
+
+
+def test_node_drop_cascades_derived_columns(tmp_path: Path):
+    """Decision 7: dropping a source column auto-drops any derived columns
+    that reference it (both schema and metadata)."""
+    workspace = Workspace("derived_drop_cascade")
+    workspace.ws_root_dir = tmp_path
+    parent_lf = pl.DataFrame(
+        {
+            "text": ["a", "b"],
+            "other": [1, 2],
+            "__derived__.tokens.text.jieba": [
+                [{"token": "a", "start": 0, "end": 1}],
+                [{"token": "b", "start": 0, "end": 1}],
+            ],
+        }
+    ).lazy()
+    meta = {
+        "source_column": "text",
+        "form": "tokens",
+        "model": "jieba",
+        "language": "zh",
+        "generated_at": "2026-05-12T00:00:00+00:00",
+    }
+    parent = workspace.add_node(
+        Node(
+            data=parent_lf,
+            name="parent",
+            workspace=workspace,
+            derived={"__derived__.tokens.text.jieba": meta},  # type: ignore[arg-type]
+        )
+    )
+
+    # Dropping an UNRELATED column does NOT cascade.
+    survivor = parent.drop("other")
+    assert "__derived__.tokens.text.jieba" in survivor.derived
+    assert "__derived__.tokens.text.jieba" in survivor.data.collect_schema().names()
+
+    # Dropping the SOURCE column cascades: the derived column disappears from
+    # both the LazyFrame schema and the metadata index.
+    cascaded = parent.drop("text")
+    after_names = cascaded.data.collect_schema().names()
+    assert "__derived__.tokens.text.jieba" not in after_names
+    assert "__derived__.tokens.text.jieba" not in cascaded.derived
+
+
+def test_node_rename_cascades_derived_columns(tmp_path: Path):
+    """Decision 7: renaming a source column drops derived columns that
+    referenced it (they become stale; user can re-tokenise)."""
+    workspace = Workspace("derived_rename_cascade")
+    workspace.ws_root_dir = tmp_path
+    parent_lf = pl.DataFrame(
+        {
+            "text": ["a", "b"],
+            "__derived__.tokens.text.jieba": [
+                [{"token": "a", "start": 0, "end": 1}],
+                [{"token": "b", "start": 0, "end": 1}],
+            ],
+        }
+    ).lazy()
+    meta = {
+        "source_column": "text",
+        "form": "tokens",
+        "model": "jieba",
+        "language": "zh",
+        "generated_at": "2026-05-12T00:00:00+00:00",
+    }
+    node = workspace.add_node(
+        Node(
+            data=parent_lf,
+            name="rename_target",
+            workspace=workspace,
+            derived={"__derived__.tokens.text.jieba": meta},  # type: ignore[arg-type]
+        )
+    )
+
+    node.rename({"text": "body"})
+    after_names = node.data.collect_schema().names()
+    assert "body" in after_names
+    assert "text" not in after_names
+    assert "__derived__.tokens.text.jieba" not in after_names
+    assert node.derived == {}

{docworkspace-0.2.7 → docworkspace-0.2.9}/tests/test_workspace.py

@@ -431,6 +431,38 @@ class TestWorkspaceGraphOperations:
             for field in required_fields:
                 assert field in node_data
 
+    def test_workspace_graph_survives_broken_node_info(self):
+        """One node failing `info()` must not break the whole graph payload."""
+        workspace = Workspace("graph_resilience")
+        good_node = Node(
+            data=pl.DataFrame({"x": [1, 2, 3]}).lazy(),
+            name="good",
+            workspace=workspace,
+        )
+        bad_node = Node(
+            data=pl.DataFrame({"y": [4, 5]}).lazy(),
+            name="bad",
+            workspace=workspace,
+        )
+
+        # Simulate a broken lazy plan / missing source file: info() raises.
+        def _boom() -> dict:
+            raise RuntimeError("source parquet missing")
+
+        bad_node.info = _boom  # type: ignore[method-assign]
+
+        graph_data = workspace.graph_json()
+
+        nodes_by_id = {n["id"]: n for n in graph_data["nodes"]}
+        assert good_node.id in nodes_by_id
+        assert bad_node.id in nodes_by_id
+        # Healthy node still carries its real info.
+        assert "shape" in nodes_by_id[good_node.id]
+        # Broken node carries an error envelope plus identity fields.
+        assert nodes_by_id[bad_node.id]["name"] == "bad"
+        assert "error" in nodes_by_id[bad_node.id]
+        assert "RuntimeError" in nodes_by_id[bad_node.id]["error"]
+
     def test_workspace_with_initial_data_loading(self):
         """Test explicit initial data loading after creating an empty workspace."""
         # Test with DataFrame converted to LazyFrame before creating a Node.

{docworkspace-0.2.7 → docworkspace-0.2.9}/tests/test_workspace_io_absolute_paths.py

@@ -156,3 +156,71 @@ def test_rebase_then_rename_then_save_keeps_parquet(tmp_path: Path):
     # Save (triggers GC) — the parquet must survive.
     ws2.save(folder_b)
     assert (folder_b / "data" / "my_data.parquet").exists()
+
+
+def test_rebase_preserves_tokenized_node_after_move(tmp_path: Path):
+    """Phase 2.9 regression: a node with a List[Struct] tokens column must
+    survive workspace-folder move + rebase_workspace_sources. The rebasing
+    walks scan-source paths inside the plbin, not the dataframe schema, so
+    it should be schema-agnostic — this test locks that in."""
+
+    folder_a = tmp_path / "Tokens"
+    folder_a.mkdir()
+    data_dir = folder_a / "data"
+    data_dir.mkdir()
+
+    parquet_path = data_dir / "docs.parquet"
+    _make_parquet(parquet_path, pl.DataFrame({"text": ["doc one", "doc two"]}))
+
+    ws = Workspace(name="Tokens", ws_root_dir=folder_a)
+    base_node = Node(
+        data=pl.scan_parquet(parquet_path.resolve()),
+        name="docs",
+    )
+    ws.add_node(base_node)
+
+    # Synthesize a derived tokens column on top via with_columns (LazyFrame
+    # plan; represents what worker_tasks_tokenize will produce in Phase 2.3).
+    derived_name = "__derived__.tokens.text.jieba"
+    derived_meta = {
+        "source_column": "text",
+        "form": "tokens",
+        "model": "jieba",
+        "language": "zh",
+        "generated_at": "2026-05-12T00:00:00+00:00",
+    }
+    tokens_frame = base_node.data.with_columns(
+        pl.lit([{"token": "doc", "start": 0, "end": 3}, {"token": "one", "start": 4, "end": 7}])
+        .alias(derived_name)
+    )
+    tokens_node = Node(
+        data=tokens_frame,
+        name="docs_tokens",
+        parents=[base_node],
+        operation="tokenize",
+        derived={derived_name: derived_meta},
+    )
+    ws.add_node(tokens_node)
+    ws.save(folder_a)
+
+    # Move the workspace folder to a new location.
+    folder_b = tmp_path / "Tokens_Moved"
+    shutil.copytree(folder_a, folder_b)
+    shutil.rmtree(folder_a)
+
+    rebase_workspace_sources(folder_b)
+    ws2 = Workspace.load(folder_b)
+
+    # Both nodes should be back, and the tokens node's lineage + metadata
+    # preserved.
+    assert len(ws2.nodes) == 2
+    loaded_tokens_node = next(
+        n for n in ws2.nodes.values() if n.name == "docs_tokens"
+    )
+    assert loaded_tokens_node.derived == {derived_name: derived_meta}
+    assert loaded_tokens_node.operation == "tokenize"
+
+    # The List[Struct] column should still be loadable end-to-end.
+    collected = cast(pl.DataFrame, loaded_tokens_node.data.collect())
+    assert derived_name in collected.columns
+    assert collected.height == 2

{docworkspace-0.2.7 → docworkspace-0.2.9}/uv.lock

@@ -13,7 +13,7 @@ wheels = [
 
 [[package]]
 name = "docworkspace"
-version = "0.2.7"
+version = "0.2.9"
 source = { editable = "." }
 dependencies = [
     { name = "polars-text" },