typedef-data-intelligence 0.40.0__tar.gz → 0.41.0__tar.gz

{typedef_data_intelligence-0.40.0 → typedef_data_intelligence-0.41.0}/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [0.41.0](https://github.com/typedef-ai/typedef/compare/typedef-data-intelligence-v0.40.0...typedef-data-intelligence-v0.41.0) (2026-07-13)
+
+
+### Features
+
+* **graph:** activate lightweight system membership (TD-3882) ([#1173](https://github.com/typedef-ai/typedef/issues/1173)) ([e91c42c](https://github.com/typedef-ai/typedef/commit/e91c42cc491b2fc8265536b8f467b61527e67071))
+
 ## [0.40.0](https://github.com/typedef-ai/typedef/compare/typedef-data-intelligence-v0.39.1...typedef-data-intelligence-v0.40.0) (2026-07-10)
 
 

{typedef_data_intelligence-0.40.0 → typedef_data_intelligence-0.41.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: typedef-data-intelligence
-Version: 0.40.0
+Version: 0.41.0
 Summary: Unified lineage management and AI-powered analytics platform for dbt projects
 Project-URL: Homepage, https://typedef.ai
 Project-URL: Repository, https://github.com/typedef-ai/data-intelligence

{typedef_data_intelligence-0.40.0 → typedef_data_intelligence-0.41.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "typedef-data-intelligence"
-version = "0.40.0"
+version = "0.41.0"
 description = "Unified lineage management and AI-powered analytics platform for dbt projects"
 readme = "README.pypi.md"
 license = "LicenseRef-Proprietary"

{typedef_data_intelligence-0.40.0 → typedef_data_intelligence-0.41.0}/src/lineage/api/desktop_sync.py

@@ -25,6 +25,7 @@ from typing import TYPE_CHECKING, Any
 from agent.api import shutdown_event
 from agent.controlplane_context import get_typedef_context
 from core.backends.controlplane import LLMProxyConfig
+from core.models.desktop import ProjectHealthInfo, ProjectsHealthResponse
 from core.models.observability import Outcome
 from core.observability.manifest import record_op_end, record_op_start
 from fastapi import APIRouter, Depends, HTTPException, Request
@@ -2478,12 +2479,9 @@ def _maybe_migrate_workspace_graph(
         return True
     except RuntimeError:
         raise
-    except Exception:  # noqa: BLE001
-        log.warning(
-            "TD-2959 graph migration probe failed; continuing with sync",
-            exc_info=True,
-        )
-        return False
+    except Exception as exc:  # noqa: BLE001
+        log.error("graph migration probe failed; aborting sync", exc_info=True)
+        raise RuntimeError(f"Graph migration probe failed: {exc}") from exc
 
 
 def _build_one_project(
@@ -3403,8 +3401,8 @@ def _project_has_graph_data(graph_db: str, project_name: str) -> bool:
         return False
 
 
-@router.get("/projects/health")
-async def projects_health():
+@router.get("/projects/health", response_model=ProjectsHealthResponse)
+async def projects_health() -> ProjectsHealthResponse:
     """Per-project sync health for all configured projects.
 
     Returns sync_status and sync_error per project so the frontend
@@ -3458,12 +3456,22 @@ async def projects_health():
         active_graph_db = gdb if (isinstance(gdb, str) and gdb) else active_ws_name
     elif active_ws_name:
         active_graph_db = active_ws_name
-    active_needs_migration = bool(
-        active_ws_name and workspace_needs_graph_migration(active_ws_name)
-    )
+    active_migration_error: str | None = None
+    try:
+        active_needs_migration = bool(
+            active_ws_name and workspace_needs_graph_migration(active_ws_name)
+        )
+    except Exception as exc:  # noqa: BLE001
+        logger.debug(
+            "graph marker probe failed for %r; reporting migration required",
+            active_ws_name,
+            exc_info=True,
+        )
+        active_needs_migration = True
+        active_migration_error = str(exc)
 
     projects = list_all_projects_raw(cfg)
-    result: list[dict] = []
+    result: list[ProjectHealthInfo] = []
     # Pool one SurrealDB connection per graph_db across the per-project
     # ingest-version probes. This endpoint is polled every ~5s, so opening a
     # fresh connection per project was O(projects) churn (TD-3196 review).
@@ -3520,19 +3528,20 @@ async def projects_health():
             _active_running_sync_id(name, ws_name) if sync_status == "running" else None
         )
         result.append(
-            {
-                "name": name,
-                "graph_name": proj.get("graph_name"),
-                "sync_status": sync_status,
-                "sync_error": sync_error,
-                "last_synced_commit": proj.get("last_synced_commit"),
-                "needs_migration": needs_migration,
-                "ingest_version_mismatch": version_mismatch,
-                "workspace": ws_name,
+            ProjectHealthInfo(
+                name=name,
+                graph_name=proj.get("graph_name"),
+                sync_status=sync_status,
+                sync_error=sync_error,
+                last_synced_commit=proj.get("last_synced_commit"),
+                needs_migration=needs_migration,
+                migration_error=active_migration_error,
+                ingest_version_mismatch=version_mismatch,
+                workspace=ws_name,
                 # sync_id of the in-flight sync (when running) so a client that
                 # didn't start it can attach to the progress stream.
-                "active_sync_id": active_sync_id,
-            }
+                active_sync_id=active_sync_id,
+            )
         )
 
     for _c in _conn_cache.values():
@@ -3542,7 +3551,7 @@ async def projects_health():
             except Exception:  # noqa: BLE001
                 logger.debug("graph conn close failed", exc_info=True)
 
-    return {"projects": result}
+    return ProjectsHealthResponse(projects=result)
 
 
 # ── Endpoints ─────────────────────────────────────────────────────────

{typedef_data_intelligence-0.40.0 → typedef_data_intelligence-0.41.0}/tests/api/test_desktop_setup.py

@@ -13,6 +13,7 @@ import yaml
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
 from lineage.api import desktop_setup
+from lineage.api.desktop_workspaces import router as workspaces_router
 from lineage.utils.dbt import DbtDocsResult
 
 
@@ -29,6 +30,38 @@ def _build_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> TestClient
     return TestClient(app)
 
 
+def test_workspace_crud_creation_stamps_current_marker_without_graph_drop(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from core.migrations import graph_migration
+
+    client = _build_client(tmp_path, monkeypatch)
+    core_paths.config_path().write_text(
+        yaml.safe_dump(
+            {
+                "lineage": {
+                    "backend": "surrealdb",
+                    "url": "ws://127.0.0.1:3567",
+                },
+                "repos": {},
+                "workspaces": {},
+            }
+        ),
+        encoding="utf-8",
+    )
+    client.app.include_router(workspaces_router)
+    monkeypatch.setattr(
+        graph_migration,
+        "drop_legacy_graph",
+        lambda *_args, **_kwargs: pytest.fail("fresh creation must not drop a graph"),
+    )
+
+    response = client.post("/desktop/workspaces", json={"name": "ws2"})
+
+    assert response.status_code == 200
+    assert graph_migration.workspace_needs_graph_migration("ws2") is False
+
+
 def _reset_manifest(monkeypatch: pytest.MonkeyPatch) -> str:
     monkeypatch.setattr(session_mod, "_session_id", None)
     monkeypatch.delenv(session_mod.SESSION_ID_ENV, raising=False)
@@ -99,6 +132,42 @@ def _seed_repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[TestCli
     return client, "repo1"
 
 
+def test_setup_creation_stamps_current_marker_without_graph_drop(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from core.migrations import graph_migration
+
+    client, repo_key = _seed_repo(tmp_path, monkeypatch)
+    expected_clone = core_paths.projects_dir() / "analytics"
+    expected_clone.mkdir(parents=True)
+    (expected_clone / "dbt_project.yml").write_text(
+        "name: analytics\nprofile: analytics_profile\n",
+        encoding="utf-8",
+    )
+    monkeypatch.setattr(
+        graph_migration,
+        "drop_legacy_graph",
+        lambda *_args, **_kwargs: pytest.fail("fresh setup must not drop a graph"),
+    )
+    monkeypatch.setattr(
+        "lineage.api.desktop_worktree._queue_gold_provisioning",
+        lambda *_args, **_kwargs: None,
+    )
+
+    response = client.post(
+        "/desktop/setup/create-project",
+        json={
+            "name": repo_key,
+            "git_url": "https://github.com/acme/analytics.git",
+            "backend": "duckdb",
+            "duckdb_path": str(tmp_path / "warehouse.duckdb"),
+        },
+    )
+
+    assert response.status_code == 200, response.text
+    assert graph_migration.workspace_needs_graph_migration(repo_key) is False
+
+
 def _collect_sse(response) -> list[tuple[str, dict]]:
     events: list[tuple[str, dict]] = []
     event_type: str | None = None

{typedef_data_intelligence-0.40.0 → typedef_data_intelligence-0.41.0}/tests/api/test_desktop_sync.py

@@ -1,15 +1,33 @@
 from __future__ import annotations
 
+import json
 import logging
+import sys
+from collections.abc import Iterable
 from pathlib import Path
 from types import SimpleNamespace
+from typing import Callable
+from unittest.mock import AsyncMock, Mock
 
 import core.observability.manifest as manifest_mod
 import core.observability.session as session_mod
 import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
 from lineage.api import desktop_sync
 
 
+def _collect_sse_data(lines: Iterable[str | bytes]) -> list[dict[str, object]]:
+    events: list[dict[str, object]] = []
+    for line in lines:
+        text = line.decode("utf-8") if isinstance(line, bytes) else line
+        if text.startswith("data: "):
+            payload = json.loads(text.removeprefix("data: "))
+            assert isinstance(payload, dict)
+            events.append(payload)
+    return events
+
+
 class EscalatingProcess:
     def __init__(self):
         self.pid = 4242
@@ -337,6 +355,270 @@ class _NotSetEvent:
         return False
 
 
+def _stub_sync_worker_shell(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    shared_pool = SimpleNamespace(
+        install_pool_signal_handlers=lambda: None,
+        shutdown_shared_executor=lambda: None,
+    )
+    monkeypatch.setitem(
+        sys.modules,
+        "ingest.static_loaders.shared_pool",
+        shared_pool,
+    )
+    monkeypatch.setattr(
+        desktop_sync, "_filter_syncable_specs", lambda _ws, specs, _log: specs
+    )
+    monkeypatch.setattr(
+        desktop_sync, "_workspace_syncable_project_names", lambda *_: set()
+    )
+    monkeypatch.setattr(desktop_sync, "_child_resource_diagnostics", lambda: "test")
+    monkeypatch.setattr(desktop_sync, "_capture_sync_failed_analytics", lambda *_: None)
+    monkeypatch.setattr("core.paths.logs_dir", lambda: tmp_path)
+
+
+@pytest.mark.parametrize(
+    "migration_error",
+    (
+        "Graph migration probe failed: marker unreadable",
+        "Graph migration failed: reset denied",
+    ),
+)
+def test_sync_worker_migration_failure_emits_one_terminal_error_and_skips_builds(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+    migration_error: str,
+) -> None:
+    _stub_sync_worker_shell(monkeypatch, tmp_path)
+
+    def _raise_reset_failure(
+        _workspace: str,
+        _projects: list[desktop_sync.ProjectBuildSpec],
+        _queue: _RecordingQueue,
+        _log: logging.Logger,
+    ) -> bool:
+        raise RuntimeError(migration_error)
+
+    build_one_project = Mock()
+    monkeypatch.setattr(
+        desktop_sync,
+        "_maybe_migrate_workspace_graph",
+        _raise_reset_failure,
+    )
+    monkeypatch.setattr(desktop_sync, "_build_one_project", build_one_project)
+    queue = _RecordingQueue()
+
+    desktop_sync._sync_worker(
+        "ws",
+        "sync-reset-failure",
+        [desktop_sync.ProjectBuildSpec(name="analytics")],
+        queue,
+        _NotSetEvent(),
+    )
+
+    terminal_errors = [
+        event
+        for event in queue.events
+        if event["stage"] == "error" and event["status"] == "error"
+    ]
+    assert len(terminal_errors) == 1
+    build_one_project.assert_not_called()
+
+
+def test_sync_probe_failure_serializes_one_terminal_sse_error_without_builds(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    _stub_sync_worker_shell(monkeypatch, tmp_path)
+    _stub_drain_finalization(monkeypatch)
+    desktop_sync._sync_ops.clear()
+    desktop_sync.shutdown_event.clear()
+    build_one_project = Mock()
+
+    def _raise_probe_failure(
+        _workspace: str,
+        _projects: list[desktop_sync.ProjectBuildSpec],
+        _queue: _RecordingQueue,
+        _log: logging.Logger,
+    ) -> bool:
+        raise RuntimeError("Graph migration probe failed: marker unreadable")
+
+    process = Mock(pid=4242, exitcode=0)
+    process.is_alive.return_value = False
+    process_factory = Mock(return_value=process)
+
+    def _run_worker() -> None:
+        call = process_factory.call_args
+        call.kwargs["target"](*call.kwargs["args"])
+
+    process.start.side_effect = _run_worker
+    monkeypatch.setattr(desktop_sync, "Process", process_factory)
+    monkeypatch.setattr(
+        desktop_sync, "_maybe_migrate_workspace_graph", _raise_probe_failure
+    )
+    monkeypatch.setattr(desktop_sync, "_build_one_project", build_one_project)
+    monkeypatch.setattr(
+        desktop_sync, "capture_backend_event_from_request", AsyncMock()
+    )
+    app = FastAPI()
+    app.include_router(desktop_sync.router)
+
+    with TestClient(app) as client:
+        started = client.post(
+            "/desktop/sync/",
+            json={
+                "workspace_name": "ws",
+                "project_name": "analytics",
+            },
+        )
+        assert started.status_code == 200
+        sync_id = started.json()["sync_id"]
+        with client.stream("GET", f"/desktop/sync/{sync_id}/progress") as response:
+            serialized = _collect_sse_data(response.iter_lines())
+
+    terminal_errors = [
+        event
+        for event in serialized
+        if event["stage"] == "error" and event["status"] == "error"
+    ]
+    assert len(terminal_errors) == 1
+    build_one_project.assert_not_called()
+
+
+def test_collect_sse_data_decodes_byte_lines() -> None:
+    assert _collect_sse_data([b'data: {"status": "completed"}']) == [
+        {"status": "completed"}
+    ]
+
+
+def test_migration_forces_every_context_full_then_second_worker_run_is_incremental(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    _stub_sync_worker_shell(monkeypatch, tmp_path)
+    migration_results = iter((True, False))
+    monkeypatch.setattr(
+        desktop_sync,
+        "_maybe_migrate_workspace_graph",
+        lambda *_: next(migration_results),
+    )
+    build_one_project = Mock(
+        return_value=desktop_sync._ProjectOutcome(status="completed")
+    )
+    monkeypatch.setattr(desktop_sync, "_build_one_project", build_one_project)
+    monkeypatch.setattr(desktop_sync, "_workspace_l1_missing", lambda _db: False)
+    monkeypatch.setattr(
+        desktop_sync, "_run_concepts_finalize", lambda *_args, **_kwargs: None
+    )
+    specs = [
+        desktop_sync.ProjectBuildSpec(name="orders"),
+        desktop_sync.ProjectBuildSpec(name="customers"),
+    ]
+
+    for sync_id in ("sync-migrate", "sync-noop"):
+        desktop_sync._sync_worker(
+            "ws",
+            sync_id,
+            specs,
+            _RecordingQueue(),
+            _NotSetEvent(),
+        )
+
+    assert [call.args[3] for call in build_one_project.call_args_list] == [
+        True,
+        True,
+        False,
+        False,
+    ]
+
+
+def test_sync_route_migrates_once_and_forces_every_context_full(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+) -> None:
+    import core.paths as core_paths
+    from core.migrations import graph_migration
+
+    _stub_sync_worker_shell(monkeypatch, tmp_path)
+    _stub_drain_finalization(monkeypatch)
+    monkeypatch.setenv("TYPEDEF_HOME", str(tmp_path))
+    monkeypatch.setenv("TYPEDEF_DEV_MODE", "1")
+    core_paths.typedef_home.cache_clear()
+    desktop_sync._sync_ops.clear()
+    desktop_sync.shutdown_event.clear()
+    marker = graph_migration.graph_marker_path(tmp_path, "ws")
+    marker.parent.mkdir(parents=True)
+    marker.write_text('{"workspace": "ws"}', encoding="utf-8")
+    drop_calls: list[str] = []
+
+    def _drop_and_stamp(
+        workspace: str, *, progress_cb: Callable[[str, int, int], None]
+    ) -> SimpleNamespace:
+        drop_calls.append(workspace)
+        progress_cb("drop", 1, 1)
+        graph_migration.mark_workspace_graph_compatible(
+            workspace, home=tmp_path, reason="test_migration"
+        )
+        return SimpleNamespace(applied=True, cleared_context_entries=2, errors=[])
+
+    monkeypatch.setattr(graph_migration, "drop_legacy_graph", _drop_and_stamp)
+    build_one_project = Mock(
+        return_value=desktop_sync._ProjectOutcome(status="completed")
+    )
+    monkeypatch.setattr(desktop_sync, "_build_one_project", build_one_project)
+    monkeypatch.setattr(desktop_sync, "_workspace_l1_missing", lambda _db: False)
+    monkeypatch.setattr(
+        desktop_sync, "_run_concepts_finalize", lambda *_args, **_kwargs: None
+    )
+    process_factory = Mock()
+    processes = [Mock(pid=4242, exitcode=0), Mock(pid=4243, exitcode=0)]
+    for process in processes:
+        process.is_alive.return_value = False
+
+        def _run_worker() -> None:
+            call = process_factory.call_args
+            call.kwargs["target"](*call.kwargs["args"])
+
+        process.start.side_effect = _run_worker
+    process_factory.side_effect = processes
+    monkeypatch.setattr(desktop_sync, "Process", process_factory)
+    monkeypatch.setattr(
+        desktop_sync, "capture_backend_event_from_request", AsyncMock()
+    )
+    app = FastAPI()
+    app.include_router(desktop_sync.router)
+    request_body = {
+        "workspace_name": "ws",
+        "project_name": "orders",
+        "projects": [
+            {"name": "orders", "full": False},
+            {"name": "customers", "full": False},
+        ],
+    }
+
+    with TestClient(app) as client:
+        for _expected_drop_count in (1, 1):
+            started = client.post("/desktop/sync/", json=request_body)
+            assert started.status_code == 200
+            sync_id = started.json()["sync_id"]
+            with client.stream(
+                "GET", f"/desktop/sync/{sync_id}/progress"
+            ) as response:
+                assert any(
+                    event["status"] == "completed"
+                    for event in _collect_sse_data(response.iter_lines())
+                )
+
+    assert drop_calls == ["ws"]
+    assert [call.args[3] for call in build_one_project.call_args_list] == [
+        True,
+        True,
+        False,
+        False,
+    ]
+
+
 def test_graph_migration_progress_events_are_nonterminal(monkeypatch):
     from core.migrations import graph_migration
 
@@ -365,6 +647,28 @@ def test_graph_migration_progress_events_are_nonterminal(monkeypatch):
     assert any("flagged for resync" in event["message"] for event in migrate_events)
 
 
+def test_graph_migration_probe_error_aborts_before_project_build(monkeypatch) -> None:
+    """An unreadable marker must stop sync before any project work begins."""
+    from core.migrations import graph_migration
+
+    def _raise_marker_io_error(_workspace: str) -> bool:
+        raise OSError("marker unreadable")
+
+    monkeypatch.setattr(
+        graph_migration,
+        "workspace_needs_graph_migration",
+        _raise_marker_io_error,
+    )
+
+    with pytest.raises(RuntimeError, match="Graph migration probe failed"):
+        desktop_sync._maybe_migrate_workspace_graph(
+            "ws",
+            [],
+            _RecordingQueue(),
+            logging.getLogger("test"),
+        )
+
+
 def _write_project_config(
     cfg_path: Path,
     project_name: str,
@@ -1410,18 +1714,75 @@ def test_projects_health_stamps_active_workspace_on_name_collision(
     )
 
     res = asyncio.run(desktop_sync.projects_health())
-    rows = res["projects"]
+    rows = res.projects
 
     # Only the ACTIVE workspace's projects are enumerated.
-    assert {r["name"] for r in rows} == {"shared", "alpha"}
+    assert {r.name for r in rows} == {"shared", "alpha"}
     # Every row — including the colliding ``shared`` — is stamped with the
     # active workspace, NOT ws_b (the old last-writer-wins bug).
     for r in rows:
-        assert r["workspace"] == "ws_a", (
-            f"{r['name']!r} mis-stamped {r['workspace']!r} (expected ws_a)"
+        assert r.workspace == "ws_a", (
+            f"{r.name!r} mis-stamped {r.workspace!r} (expected ws_a)"
         )
 
 
+def test_projects_health_reports_marker_io_error_without_crashing(
+    monkeypatch, tmp_path, caplog
+) -> None:
+    """Health remains available when sync's marker probe must fail closed."""
+    import asyncio
+
+    import yaml
+    from core import paths as core_paths
+    from core.migrations import graph_migration
+
+    cfg_file = tmp_path / "config.yaml"
+    cfg_file.write_text(
+        yaml.safe_dump(
+            {
+                "active_workspace": "ws",
+                "repos": {},
+                "workspaces": {
+                    "ws": {
+                        "graph_db": "ws",
+                        "context": [{"project": {"name": "project"}}],
+                    }
+                },
+            }
+        ),
+        encoding="utf-8",
+    )
+    monkeypatch.setattr(core_paths, "config_path", lambda: cfg_file)
+
+    def _raise_marker_io_error(_workspace: str) -> bool:
+        raise OSError("marker unreadable")
+
+    monkeypatch.setattr(
+        graph_migration,
+        "workspace_needs_graph_migration",
+        _raise_marker_io_error,
+    )
+    monkeypatch.setattr(desktop_sync, "_open_surreal_for_graph", lambda _gdb: None)
+    monkeypatch.setattr(
+        desktop_sync,
+        "_project_has_graph_data",
+        lambda _gdb, _name: False,
+    )
+
+    with caplog.at_level(logging.DEBUG, logger=desktop_sync.logger.name):
+        response = asyncio.run(desktop_sync.projects_health())
+
+    assert len(response.projects) == 1
+    project = response.projects[0]
+    assert project.name == "project"
+    assert project.needs_migration is True
+    assert project.migration_error == "marker unreadable"
+    marker_logs = [
+        record for record in caplog.records if "graph marker probe failed" in record.message
+    ]
+    assert [record.levelno for record in marker_logs] == [logging.DEBUG]
+
+
 # ── TD-2950: silent-sync-death diagnostics ────────────────────────────