interloper-toolkit 0.87.0__tar.gz → 0.89.0__tar.gz

{interloper_toolkit-0.87.0 → interloper_toolkit-0.89.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: interloper-toolkit
-Version: 0.87.0
+Version: 0.89.0
 Summary: Interloper shared read-only tool functions for AI surfaces (agent, MCP)
 Author: Guillaume Onfroy
 Author-email: Guillaume Onfroy <guillaume@digitlcloud.com>

{interloper_toolkit-0.87.0 → interloper_toolkit-0.89.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "interloper-toolkit"
-version = "0.87.0"
+version = "0.89.0"
 description = "Interloper shared read-only tool functions for AI surfaces (agent, MCP)"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -57,7 +57,7 @@ convention = "google"
     "RUF069",
     "PLW0108",
 ]
-"src/interloper_toolkit/{analytics,catalog,collection,lineage,scheduling}.py" = [
+"src/interloper_toolkit/{analytics,catalog,collection,jobs,lineage,scheduling,sources}.py" = [
     "D417",
     "DOC201",
     "BLE001",

{interloper_toolkit-0.87.0 → interloper_toolkit-0.89.0}/pyproject.toml.orig

@@ -3,7 +3,7 @@
 # ###############
 [project]
 name = "interloper-toolkit"
-version = "0.87.0"
+version = "0.89.0"
 description = "Interloper shared read-only tool functions for AI surfaces (agent, MCP)"
 readme = "README.md"
 authors = [{ name = "Guillaume Onfroy", email = "guillaume@digitlcloud.com" }]
@@ -37,4 +37,4 @@ convention = "google"
 [tool.ruff.lint.per-file-ignores]
 "__init__.py" = ["F401", "F403"]
 "tests/**" = ["ANN", "F811", "D101", "D102", "D103", "D104", "RUF069", "PLW0108"]
-"src/interloper_toolkit/{analytics,catalog,collection,lineage,scheduling}.py" = ["D417", "DOC201", "BLE001"]
+"src/interloper_toolkit/{analytics,catalog,collection,jobs,lineage,scheduling,sources}.py" = ["D417", "DOC201", "BLE001"]

{interloper_toolkit-0.87.0 → interloper_toolkit-0.89.0}/src/interloper_toolkit/__init__.py

@@ -7,17 +7,16 @@ the literal ``status`` field, never raising. The docstrings are LLM-facing:
 both the ADK agent and the MCP server surface them verbatim as tool
 descriptions.
 
-Almost every function here is read-only; the sole exceptions are
-:func:`interloper_toolkit.collection.bind_relation` and
-:func:`interloper_toolkit.collection.unbind_relation`, which write and are
-re-exported here as this package's whole write surface. A surface that must
-stay read-only (the MCP server's own registration is one) never registers
-those two; the ADK agent, whose own write tools already live beside them,
-does.
+Reads take no role; writes declare the role they need with
+:func:`~interloper_toolkit.authz.requires_role` and refuse below it. Which
+functions a surface exposes is that surface's registration list (the MCP
+server, for one, never registers ``create_connections``, whose arguments
+would carry credentials through the client).
 """
 
+from interloper_toolkit.authz import requires_role
 from interloper_toolkit.collection import bind_relation, unbind_relation
 from interloper_toolkit.context import ToolkitContext, serialize
 from interloper_toolkit.models import ToolError
 
-__all__ = ["ToolError", "ToolkitContext", "bind_relation", "serialize", "unbind_relation"]
+__all__ = ["ToolError", "ToolkitContext", "bind_relation", "requires_role", "serialize", "unbind_relation"]

interloper_toolkit-0.89.0/src/interloper_toolkit/analytics.py

@@ -0,0 +1,360 @@
+"""Analytics tools — run statistics, partition coverage, and data freshness."""
+
+from __future__ import annotations
+
+import datetime
+from typing import Any
+from uuid import UUID
+
+from interloper.partitioning.time import TimePartition
+
+from interloper_toolkit.context import ToolkitContext
+from interloper_toolkit.models import (
+    AssetCoverage,
+    AssetCoverageRow,
+    FreshnessReport,
+    JobFreshness,
+    JobStats,
+    PartitionCoverage,
+    PartitionRange,
+    RunHistorySummary,
+    RunStats,
+    ToolError,
+)
+from interloper_toolkit.stats import percentile, window
+
+_MISSING_RANGES_LIMIT = 20
+
+
+def run_history_summary(
+    ctx: ToolkitContext,
+    component_id: str | None = None,
+    days: int = 7,
+) -> RunHistorySummary | ToolError:
+    """Summarize run statistics over a period.
+
+    Args:
+        component_id: Filter to a specific job UUID (optional, all jobs if omitted).
+        days: Number of days to look back (default 7).
+
+    Returns aggregate counts (total, success, failed, canceled), success
+    rate, and average duration over the runs that executed within the
+    period, each run stack counted by its latest attempt.
+    """
+    try:
+        jid = UUID(component_id) if component_id else None
+        cutoff = datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(days=days)
+        total = ctx.store.runs.count(ctx.org_id, component_id=jid, after=cutoff)
+        runs = ctx.store.runs.list_all(ctx.org_id, component_id=jid, after=cutoff, limit=total)
+
+        by_status: dict[str, int] = {}
+        durations: list[float] = []
+        for r in runs:
+            by_status[r.status] = by_status.get(r.status, 0) + 1
+            if r.started_at and r.completed_at:
+                durations.append((r.completed_at - r.started_at).total_seconds())
+
+        success = by_status.get("success", 0)
+        return RunHistorySummary(
+            period_days=days,
+            component_id=component_id,
+            total_runs=total,
+            by_status=by_status,
+            success_rate=round(success / total, 2) if total > 0 else None,
+            avg_duration_seconds=round(sum(durations) / len(durations), 1) if durations else None,
+        )
+    except Exception as e:
+        return ToolError(error=str(e))
+
+
+def partition_coverage(
+    ctx: ToolkitContext,
+    component_id: str,
+    start_date: str,
+    end_date: str,
+) -> PartitionCoverage | ToolError:
+    """Check partition coverage for a job over a date range.
+
+    Args:
+        component_id: UUID of the job.
+        start_date: Start date in ISO format (YYYY-MM-DD).
+        end_date: End date in ISO format (YYYY-MM-DD), inclusive.
+
+    Returns which dates have successful runs and which are missing.
+    """
+    try:
+        job = ctx.store.components.get(UUID(component_id), kind="job", org_id=ctx.org_id)
+        total = ctx.store.runs.count(ctx.org_id, component_id=job.id, status="success")
+        runs = ctx.store.runs.list_all(ctx.org_id, component_id=job.id, status="success", limit=total)
+
+        start = datetime.date.fromisoformat(start_date)
+        end = datetime.date.fromisoformat(end_date)
+
+        # Coverage is a daily question, so a run covers every day inside its partition.
+        covered: set[datetime.date] = set()
+        for r in runs:
+            if not r.partition_key:
+                continue
+            p_start, p_end = TimePartition.from_key(r.partition_key).bounds
+            if isinstance(p_start, datetime.datetime):
+                days = [p_start.date()]
+            else:
+                days = []
+                current = p_start
+                while current < p_end:
+                    days.append(current)
+                    current += datetime.timedelta(days=1)
+            covered.update(day for day in days if start <= day <= end)
+
+        # Build expected date range
+        expected: list[datetime.date] = []
+        current = start
+        while current <= end:
+            expected.append(current)
+            current += datetime.timedelta(days=1)
+
+        missing = sorted(set(expected) - covered)
+        coverage_pct = round(len(covered) / len(expected) * 100, 1) if expected else 100.0
+
+        return PartitionCoverage(
+            component_id=component_id,
+            start_date=start_date,
+            end_date=end_date,
+            total_days=len(expected),
+            covered_days=len(covered),
+            missing_days=len(missing),
+            coverage_percent=coverage_pct,
+            missing_dates=[d.isoformat() for d in missing],
+        )
+    except Exception as e:
+        return ToolError(error=str(e))
+
+
+def freshness_check(ctx: ToolkitContext) -> FreshnessReport | ToolError:
+    """Check data freshness for all jobs.
+
+    Returns the last successful run timestamp for each job and flags
+    any that haven't succeeded in over 24 hours.
+    """
+    try:
+        jobs = ctx.store.components.list_all(ctx.org_id, kinds=["job"])
+        now = datetime.datetime.now(tz=datetime.timezone.utc)
+
+        results = []
+        for job in jobs:
+            if not (job.config or {}).get("enabled", True):
+                continue
+            component_id = job.id
+            runs = ctx.store.runs.list_all(ctx.org_id, component_id=component_id, status="success", limit=1)
+            last_success = runs[0] if runs else None
+
+            hours_since = None
+            if last_success and last_success.completed_at:
+                delta = now - last_success.completed_at
+                hours_since = round(delta.total_seconds() / 3600, 1)
+
+            results.append(JobFreshness(
+                job=job,
+                last_success_at=last_success.completed_at if last_success else None,
+                hours_since_success=hours_since,
+                stale=hours_since is None or hours_since > 24,
+            ))
+
+        stale_count = sum(1 for r in results if r.stale)
+        return FreshnessReport(
+            total_jobs=len(results),
+            stale_count=stale_count,
+            jobs=results,
+        )
+    except Exception as e:
+        return ToolError(error=str(e))
+
+
+def run_stats(
+    ctx: ToolkitContext,
+    since: str | None = None,
+    until: str | None = None,
+    component_id: str | None = None,
+    limit: int = 25,
+    offset: int = 0,
+) -> RunStats | ToolError:
+    """Per-job run statistics over a window: verdicts, durations and retries.
+
+    Args:
+        since: ISO date or datetime the window opens at (default: 7 days ago).
+        until: ISO date or datetime the window closes before (default: open).
+        component_id: Restrict to this job UUID.
+        limit: Maximum number of jobs to return (default 25).
+        offset: Number of jobs to skip, for paging past the first page.
+
+    Returns one row per job that ran in the window, most failures first: run
+    stacks by final status, attempts, duration p50/p90/max in seconds, and
+    how many retried stacks healed or are still failing.
+    """
+    try:
+        start, end = window(since, until, default_days=7)
+        filters: dict[str, Any] = {
+            "component_id": UUID(component_id) if component_id else None,
+            "after": start,
+            "before": end,
+            "all_attempts": True,
+        }
+        total = ctx.store.runs.count(ctx.org_id, **filters)
+        runs = ctx.store.runs.list_all(ctx.org_id, **filters, limit=total)
+
+        stacks: dict[UUID | None, dict[UUID, list[Any]]] = {}
+        names: dict[UUID | None, str | None] = {}
+        for r in runs:
+            stacks.setdefault(r.component_id, {}).setdefault(r.root_run_id, []).append(r)
+            names.setdefault(r.component_id, r.target.name if r.target else None)
+
+        rows = []
+        for job_id, by_root in stacks.items():
+            by_status: dict[str, int] = {}
+            durations: list[float] = []
+            attempts = retried = healed = still_failing = 0
+            for chain in by_root.values():
+                latest = max(chain, key=lambda r: r.attempt)
+                by_status[latest.status] = by_status.get(latest.status, 0) + 1
+                attempts += len(chain)
+                durations += [
+                    (r.completed_at - r.started_at).total_seconds() for r in chain if r.started_at and r.completed_at
+                ]
+                if len(chain) > 1:
+                    retried += 1
+                    healed += latest.status == "success"
+                    still_failing += latest.status == "failed"
+            rows.append(
+                JobStats(
+                    job_id=job_id,
+                    job_name=names[job_id],
+                    stacks=by_status,
+                    attempts=attempts,
+                    duration_p50_s=_rounded(percentile(durations, 50)),
+                    duration_p90_s=_rounded(percentile(durations, 90)),
+                    duration_max_s=_rounded(max(durations, default=None)),
+                    stacks_retried=retried,
+                    healed=healed,
+                    still_failing=still_failing,
+                )
+            )
+        rows.sort(key=lambda j: (-j.stacks.get("failed", 0), j.job_name or ""))
+        page = rows[offset : offset + limit]
+        return RunStats(since=start, until=end, count=len(page), total=len(rows), jobs=page)
+    except Exception as e:
+        return ToolError(error=str(e))
+
+
+def asset_coverage(
+    ctx: ToolkitContext,
+    component_id: str,
+    start_key: str,
+    end_key: str,
+    limit: int = 50,
+    offset: int = 0,
+) -> AssetCoverage | ToolError:
+    """Per-asset partition coverage of a job over a range of partition keys.
+
+    Unlike partition_coverage, which needs a whole run to have succeeded, an
+    asset counts as covered for a partition once any run's execution of it
+    succeeded, so a run where most assets succeeded shows what is actually
+    missing. Only assets that executed at least once in the range appear.
+
+    Args:
+        component_id: UUID of the job.
+        start_key: First partition key, in the job's granularity (2026-07-01,
+            2026-07, 2026, or 2026-07-01T13).
+        end_key: Last partition key, inclusive; must share the start key's
+            granularity.
+        limit: Maximum number of assets to return (default 50).
+        offset: Number of assets to skip, for paging past the first page.
+
+    Returns the page of assets, least covered first, each with its covered,
+    failed and never-run partition counts and its missing partitions as
+    ranges ready for a backfill, plus a rollup of the range's partitions by
+    whether all, some or none of the assets are covered.
+    """
+    try:
+        job = ctx.store.components.get(UUID(component_id), kind="job", org_id=ctx.org_id)
+        first, last = TimePartition.from_key(start_key), TimePartition.from_key(end_key)
+        if first.granularity is not last.granularity:
+            return ToolError(error=f"{start_key!r} and {end_key!r} are keys of different granularities")
+        granularity = first.granularity
+        expected = [granularity.format(value) for value in granularity.period_range(first.value, last.value)]
+
+        rows = ctx.store.events.partition_coverage(ctx.org_id, job.id, start_key, end_key)
+        keys: dict[UUID, str | None] = {}
+        attempted: dict[UUID, set[str]] = {}
+        covered: dict[UUID, set[str]] = {}
+        for row in rows:
+            keys.setdefault(row.component_id, row.component_key)
+            attempted.setdefault(row.component_id, set()).add(row.partition_key)
+            if row.succeeded:
+                covered.setdefault(row.component_id, set()).add(row.partition_key)
+
+        assets = []
+        for asset_id, asset_key in keys.items():
+            done = covered.get(asset_id, set())
+            missing = [key for key in expected if key not in done]
+            ranges = _ranges(missing, expected)
+            assets.append(
+                AssetCoverageRow(
+                    asset_id=asset_id,
+                    asset_key=asset_key,
+                    covered=len(done),
+                    failed=len(attempted[asset_id] - done),
+                    never_run=len(missing) - len(attempted[asset_id] - done),
+                    missing=ranges[:_MISSING_RANGES_LIMIT],
+                    missing_ranges_total=len(ranges),
+                )
+            )
+        assets.sort(key=lambda a: (a.covered, a.asset_key or ""))
+
+        per_partition = [sum(key in covered.get(asset_id, ()) for asset_id in keys) for key in expected]
+        page = assets[offset : offset + limit]
+        return AssetCoverage(
+            component_id=job.id,
+            start_key=start_key,
+            end_key=end_key,
+            partitions=len(expected),
+            all_covered=sum(n == len(keys) for n in per_partition) if keys else 0,
+            partly_covered=sum(0 < n < len(keys) for n in per_partition),
+            none_covered=sum(n == 0 for n in per_partition) if keys else len(expected),
+            count=len(page),
+            total=len(assets),
+            assets=page,
+        )
+    except Exception as e:
+        return ToolError(error=str(e))
+
+
+def _ranges(missing: list[str], expected: list[str]) -> list[PartitionRange]:
+    """Collapse the missing keys into runs of consecutive partitions.
+
+    Args:
+        missing: The uncovered keys, in *expected*'s order.
+        expected: Every key of the range, in order.
+
+    Returns:
+        One inclusive range per run of consecutive missing keys.
+    """
+    position = {key: index for index, key in enumerate(expected)}
+    ranges: list[PartitionRange] = []
+    for key in missing:
+        if ranges and position[key] == position[ranges[-1].end_key] + 1:
+            ranges[-1].end_key = key
+        else:
+            ranges.append(PartitionRange(start_key=key, end_key=key))
+    return ranges
+
+
+def _rounded(value: float | None) -> float | None:
+    """Round a duration to a tenth of a second, passing ``None`` through.
+
+    Args:
+        value: Seconds, or ``None`` when there was nothing to measure.
+
+    Returns:
+        The rounded value, or ``None``.
+    """
+    return round(value, 1) if value is not None else None

interloper_toolkit-0.89.0/src/interloper_toolkit/authz.py

@@ -0,0 +1,73 @@
+"""Role gating for the toolkit's write tools.
+
+The context carries the caller's role in the organisation; a write declares
+the role it needs and refuses, as a structured :class:`ToolError`, before its
+body runs. The ranks and the refusal wording mirror the API's route gates, so
+a caller can do through a tool exactly what it could do through the app.
+"""
+
+from __future__ import annotations
+
+import functools
+import inspect
+from collections.abc import Callable
+from typing import Any, TypeVar, cast
+
+from interloper_toolkit.context import ToolkitContext
+from interloper_toolkit.models import ToolError
+
+_ROLE_RANK = {"viewer": 0, "editor": 1, "admin": 2}
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def requires_role(minimum: str) -> Callable[[F], F]:
+    """Refuse a tool call whose context holds a role below *minimum*.
+
+    Works on sync and async tool functions alike; the context is the first
+    positional argument, as it is for every toolkit function. The wrapped
+    function keeps its name and docstring, which the AI surfaces adopt.
+
+    Args:
+        minimum: The lowest role allowed: ``viewer``, ``editor`` or ``admin``.
+
+    Returns:
+        The decorator to apply to a tool function.
+
+    Raises:
+        ValueError: If *minimum* is not a known role.
+    """
+    if minimum not in _ROLE_RANK:
+        raise ValueError(f"Unknown role {minimum!r}; expected one of {sorted(_ROLE_RANK)}")
+
+    def decorate(func: F) -> F:
+        if inspect.iscoroutinefunction(func):
+
+            @functools.wraps(func)
+            async def async_gate(ctx: ToolkitContext, *args: Any, **kwargs: Any) -> Any:
+                return denied(ctx.role, minimum) or await func(ctx, *args, **kwargs)
+
+            return cast(F, async_gate)
+
+        @functools.wraps(func)
+        def gate(ctx: ToolkitContext, *args: Any, **kwargs: Any) -> Any:
+            return denied(ctx.role, minimum) or func(ctx, *args, **kwargs)
+
+        return cast(F, gate)
+
+    return decorate
+
+
+def denied(role: str, minimum: str) -> ToolError | None:
+    """The refusal for a role below *minimum*, or ``None`` when the role suffices.
+
+    Args:
+        role: The caller's role; an unknown one ranks below every known role.
+        minimum: The lowest role allowed.
+
+    Returns:
+        The structured error, or ``None`` when the call may proceed.
+    """
+    if _ROLE_RANK.get(role, -1) < _ROLE_RANK[minimum]:
+        return ToolError(error=f"Requires {minimum} role or higher")
+    return None

{interloper_toolkit-0.87.0 → interloper_toolkit-0.89.0}/src/interloper_toolkit/catalog.py

@@ -81,7 +81,7 @@ def list_definitions(
         if kind not in counts:
             return ToolError(
                 error=f"No '{kind}' definitions in the catalog",
-                valid_kinds=sorted(counts),
+                valid_values=sorted(counts),
             )
 
         collection_counts: dict[str, int] = {}
@@ -179,7 +179,7 @@ def get_asset_schema(ctx: ToolkitContext, source_key: str, asset_key: str) -> As
         return ToolError(error=str(e))
 
 
-def search_fields(ctx: ToolkitContext, query: str) -> FieldSearchResult | ToolError:
+def search_fields(ctx: ToolkitContext, query: str, limit: int = 50, offset: int = 0) -> FieldSearchResult | ToolError:
     """Search for fields across all asset schemas in the catalog matching a query string.
 
     Searches every source definition's schemas, whether or not the source is
@@ -187,8 +187,11 @@ def search_fields(ctx: ToolkitContext, query: str) -> FieldSearchResult | ToolEr
 
     Args:
         query: Substring to search for in field names and descriptions (case-insensitive).
+        limit: Maximum number of matches to return (default 50).
+        offset: Number of matches to skip, for paging past the first page.
 
-    Returns matching fields grouped by source and asset, with field type and description.
+    Returns the page of matching fields grouped by source and asset, with
+    field type and description, plus the total number of matches.
     """
     try:
         query_lower = query.lower()
@@ -215,7 +218,8 @@ def search_fields(ctx: ToolkitContext, query: str) -> FieldSearchResult | ToolEr
                             description=field_desc,
                         ))
 
-        return FieldSearchResult(query=query, match_count=len(matches), matches=matches)
+        page = matches[offset : offset + limit]
+        return FieldSearchResult(query=query, match_count=len(page), total=len(matches), matches=page)
     except Exception as e:
         return ToolError(error=str(e))