interloper-toolkit 0.43.0__py3-none-any.whl

interloper_toolkit/__init__.py

@@ -0,0 +1,14 @@
+"""Read-only tool functions shared by AI surfaces (agent, MCP server).
+
+Every function takes a :class:`~interloper_toolkit.context.ToolkitContext`
+as its first argument and returns ``<SuccessModel> | ToolError`` — typed
+pydantic results (see :mod:`interloper_toolkit.models`) discriminated by
+the literal ``status`` field, never raising. The docstrings are LLM-facing:
+both the ADK agent and the MCP server surface them verbatim as tool
+descriptions.
+"""
+
+from interloper_toolkit.context import ToolkitContext, serialize
+from interloper_toolkit.models import ToolError
+
+__all__ = ["ToolkitContext", "ToolError", "serialize"]

interloper_toolkit/analytics.py

@@ -0,0 +1,153 @@
+"""Analytics tools — run statistics, partition coverage, and data freshness."""
+
+from __future__ import annotations
+
+import datetime
+from uuid import UUID
+
+from interloper_toolkit.context import ToolkitContext
+from interloper_toolkit.models import (
+    FreshnessReport,
+    JobFreshness,
+    PartitionCoverage,
+    RunHistorySummary,
+    ToolError,
+)
+
+
+def run_history_summary(
+    ctx: ToolkitContext,
+    component_id: str | None = None,
+    days: int = 7,
+) -> RunHistorySummary | ToolError:
+    """Summarize run statistics over a period.
+
+    Args:
+        component_id: Filter to a specific job UUID (optional, all jobs if omitted).
+        days: Number of days to look back (default 7).
+
+    Returns aggregate counts (total, success, failed, canceled),
+    success rate, and average duration.
+    """
+    try:
+        runs = ctx.store.list_runs(
+            ctx.org_id,
+            component_id=UUID(component_id) if component_id else None,
+            limit=500,
+        )
+
+        cutoff = datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(days=days)
+        recent = [r for r in runs if r.created_at and r.created_at >= cutoff]
+
+        total = len(recent)
+        by_status: dict[str, int] = {}
+        durations: list[float] = []
+        for r in recent:
+            by_status[r.status] = by_status.get(r.status, 0) + 1
+            if r.started_at and r.completed_at:
+                durations.append((r.completed_at - r.started_at).total_seconds())
+
+        success = by_status.get("success", 0)
+        return RunHistorySummary(
+            period_days=days,
+            component_id=component_id,
+            total_runs=total,
+            by_status=by_status,
+            success_rate=round(success / total, 2) if total > 0 else None,
+            avg_duration_seconds=round(sum(durations) / len(durations), 1) if durations else None,
+        )
+    except Exception as e:
+        return ToolError(error=str(e))
+
+
+def partition_coverage(
+    ctx: ToolkitContext,
+    component_id: str,
+    start_date: str,
+    end_date: str,
+) -> PartitionCoverage | ToolError:
+    """Check partition coverage for a job over a date range.
+
+    Args:
+        component_id: UUID of the job.
+        start_date: Start date in ISO format (YYYY-MM-DD).
+        end_date: End date in ISO format (YYYY-MM-DD), inclusive.
+
+    Returns which dates have successful runs and which are missing.
+    """
+    try:
+        jid = UUID(component_id)
+        runs = ctx.store.list_runs(ctx.org_id, component_id=jid, limit=1000)
+
+        start = datetime.date.fromisoformat(start_date)
+        end = datetime.date.fromisoformat(end_date)
+
+        # Collect dates with successful runs
+        covered: set[datetime.date] = set()
+        for r in runs:
+            if r.status == "success" and r.partition_date:
+                if start <= r.partition_date <= end:
+                    covered.add(r.partition_date)
+
+        # Build expected date range
+        expected: list[datetime.date] = []
+        current = start
+        while current <= end:
+            expected.append(current)
+            current += datetime.timedelta(days=1)
+
+        missing = sorted(set(expected) - covered)
+        coverage_pct = round(len(covered) / len(expected) * 100, 1) if expected else 100.0
+
+        return PartitionCoverage(
+            component_id=component_id,
+            start_date=start_date,
+            end_date=end_date,
+            total_days=len(expected),
+            covered_days=len(covered),
+            missing_days=len(missing),
+            coverage_percent=coverage_pct,
+            missing_dates=[d.isoformat() for d in missing],
+        )
+    except Exception as e:
+        return ToolError(error=str(e))
+
+
+def freshness_check(ctx: ToolkitContext) -> FreshnessReport | ToolError:
+    """Check data freshness for all jobs.
+
+    Returns the last successful run timestamp for each job and flags
+    any that haven't succeeded in over 24 hours.
+    """
+    try:
+        jobs = ctx.store.list_components(ctx.org_id, kinds=["job"])
+        now = datetime.datetime.now(tz=datetime.timezone.utc)
+
+        results = []
+        for job in jobs:
+            if not (job.config or {}).get("enabled", True):
+                continue
+            component_id = job.id
+            runs = ctx.store.list_runs(ctx.org_id, component_id=component_id, status="success", limit=1)
+            last_success = runs[0] if runs else None
+
+            hours_since = None
+            if last_success and last_success.completed_at:
+                delta = now - last_success.completed_at
+                hours_since = round(delta.total_seconds() / 3600, 1)
+
+            results.append(JobFreshness(
+                job=job,
+                last_success_at=last_success.completed_at if last_success else None,
+                hours_since_success=hours_since,
+                stale=hours_since is None or hours_since > 24,
+            ))
+
+        stale_count = sum(1 for r in results if r.stale)
+        return FreshnessReport(
+            total_jobs=len(results),
+            stale_count=stale_count,
+            jobs=results,
+        )
+    except Exception as e:
+        return ToolError(error=str(e))

interloper_toolkit/catalog.py

@@ -0,0 +1,303 @@
+"""Catalog tools — the component definitions the platform ships.
+
+Generic over component kinds, mirroring the framework's component
+architecture: one lister and one getter for any kind, plus the
+asset-schema operations that are irreducibly kind-specific.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from interloper.oauth import is_provider_configured
+
+from interloper_toolkit.context import ToolkitContext
+from interloper_toolkit.models import (
+    AssetSchemaResult,
+    DefinitionCounts,
+    DefinitionDetail,
+    DefinitionEntry,
+    DefinitionList,
+    FieldMatch,
+    FieldSearchResult,
+    SchemaComparison,
+    SharedField,
+    ToolError,
+)
+
+
+def _inline_refs(node: Any, defs: dict[str, Any] | None = None, _stack: frozenset[str] = frozenset()) -> Any:
+    """Resolve local ``#/$defs/*`` references in-place and drop ``$defs``.
+
+    Gemini rejects tool responses containing JSON-Schema ``$ref`` pointers
+    (400 INVALID_ARGUMENT), so every schema leaving the toolkit is inlined:
+    a ``$ref`` is replaced by its definition (ref siblings win on conflict);
+    cyclic or unresolvable refs are dropped, keeping the siblings.
+    """
+    if isinstance(node, list):
+        return [_inline_refs(item, defs, _stack) for item in node]
+    if not isinstance(node, dict):
+        return node
+    if isinstance(node.get("$defs"), dict):
+        defs = {**(defs or {}), **node["$defs"]}
+    ref = node.get("$ref")
+    if isinstance(ref, str) and ref.startswith("#/$defs/"):
+        name = ref.rsplit("/", 1)[-1]
+        if defs and name in defs and name not in _stack:
+            siblings = {k: v for k, v in node.items() if k not in ("$ref", "$defs")}
+            return _inline_refs({**defs[name], **siblings}, defs, _stack | {name})
+    return {k: _inline_refs(v, defs, _stack) for k, v in node.items() if k not in ("$defs", "$ref")}
+
+
+def list_definitions(
+    ctx: ToolkitContext, kind: str | None = None
+) -> DefinitionCounts | DefinitionList | ToolError:
+    """List the component definitions available in the catalog.
+
+    This answers "what does Interloper support / what could we add?" — the
+    org's collection itself is the Collection specialist's domain.
+
+    Source entries note how many instances the org's collection holds
+    (``in_collection`` / ``collection_count``); connection entries note
+    whether OAuth sign-in is supported (``oauth``) and usable in this
+    deployment (``oauth_available``) vs manual credential entry.
+
+    Args:
+        kind: Component kind to list — e.g. 'source', 'connection',
+            'destination'. Omit for per-kind counts only; call again with a
+            kind for the entries.
+    """
+    try:
+        catalog = ctx.catalog
+        counts: dict[str, int] = {}
+        for defn in catalog.values():
+            counts[defn.get("kind", "?")] = counts.get(defn.get("kind", "?"), 0) + 1
+
+        if kind is None:
+            return DefinitionCounts(
+                definition_counts=counts,
+                message="Call again with a kind for the entries.",
+            )
+        if kind not in counts:
+            return ToolError(
+                error=f"No '{kind}' definitions in the catalog",
+                valid_kinds=sorted(counts),
+            )
+
+        collection_counts: dict[str, int] = {}
+        if kind == "source":
+            for s in ctx.store.list_components(ctx.org_id, kinds=["source"]):
+                collection_counts[s.key] = collection_counts.get(s.key, 0) + 1
+
+        results = []
+        for key, defn in catalog.items():
+            if defn.get("kind") != kind:
+                continue
+            entry = DefinitionEntry(
+                key=key,
+                name=defn.get("name", key),
+                description=defn.get("description"),
+                icon=defn.get("icon"),
+                tags=defn.get("tags", []),
+            )
+            if kind == "source":
+                count = collection_counts.get(key, 0)
+                entry.asset_count = len(defn.get("assets", []))
+                entry.in_collection = count > 0
+                entry.collection_count = count
+            elif kind == "connection":
+                schema = defn.get("config_schema") or {}
+                oauth = schema.get("x-oauth")
+                entry.provider = defn.get("provider")
+                entry.required_fields = schema.get("required", [])
+                entry.oauth = oauth is not None
+                entry.oauth_available = is_provider_configured(oauth["provider"]) if oauth else False
+            results.append(entry)
+
+        return DefinitionList(kind=kind, count=len(results), definitions=results)
+    except Exception as e:
+        return ToolError(error=str(e))
+
+
+def get_definition(ctx: ToolkitContext, key: str) -> DefinitionDetail | ToolError:
+    """Get a component definition's full catalog detail.
+
+    For a source this includes the config schema, resource slots,
+    destination types, and all its assets with their schemas. This is the
+    catalog definition (the component *type*), not an instance from the
+    org's collection.
+
+    Args:
+        key: The definition's catalog key (e.g. 'facebook_ads',
+            'facebook_ads_connection').
+    """
+    try:
+        defn = ctx.catalog.get(key)
+        if defn is None:
+            return ToolError(error=f"Definition '{key}' not found in catalog")
+        return DefinitionDetail(definition=_inline_refs(defn))
+    except Exception as e:
+        return ToolError(error=str(e))
+
+
+# -- Asset schemas (kind-specific by nature) -------------------------------------
+
+
+def get_asset_schema(ctx: ToolkitContext, source_key: str, asset_key: str) -> AssetSchemaResult | ToolError:
+    """Get the JSON schema for a specific asset within a source.
+
+    Schemas come from the catalog: they are a property of the source
+    definition, shared by every instance in the org's collection.
+
+    Args:
+        source_key: The source key (e.g. 'facebook_ads').
+        asset_key: The asset key within the source (e.g. 'ad_insights').
+
+    Returns the asset schema with field names, types, and descriptions.
+    """
+    try:
+        defn = ctx.catalog.get(source_key)
+        if defn is None or defn.get("kind") != "source":
+            return ToolError(error=f"Source '{source_key}' not found in catalog")
+
+        for asset_def in defn.get("assets", []):
+            if asset_def.get("key") == asset_key or asset_def.get("qualified_key") == f"{source_key}.{asset_key}":
+                asset_def = _inline_refs(asset_def)
+                return AssetSchemaResult(
+                    source_key=source_key,
+                    asset_key=asset_key,
+                    qualified_key=f"{source_key}.{asset_key}",
+                    asset_schema=asset_def.get("asset_schema"),
+                    partitioning=asset_def.get("partitioning"),
+                    tags=asset_def.get("tags", []),
+                    requires=asset_def.get("requires", {}),
+                    optional_requires=asset_def.get("optional_requires", {}),
+                )
+
+        return ToolError(error=f"Asset '{asset_key}' not found in source '{source_key}'")
+    except Exception as e:
+        return ToolError(error=str(e))
+
+
+def search_fields(ctx: ToolkitContext, query: str) -> FieldSearchResult | ToolError:
+    """Search for fields across all asset schemas in the catalog matching a query string.
+
+    Searches every source definition's schemas, whether or not the source is
+    in the org's collection.
+
+    Args:
+        query: Substring to search for in field names and descriptions (case-insensitive).
+
+    Returns matching fields grouped by source and asset, with field type and description.
+    """
+    try:
+        query_lower = query.lower()
+        matches: list[FieldMatch] = []
+
+        for key, defn in ctx.catalog.items():
+            if defn.get("kind") != "source":
+                continue
+            source_key = key
+            for asset_def in defn.get("assets", []):
+                asset_key = asset_def.get("key", "")
+                schema = asset_def.get("asset_schema")
+                if not schema or "properties" not in schema:
+                    continue
+                for field_name, field_info in schema["properties"].items():
+                    field_desc = field_info.get("description", "")
+                    if query_lower in field_name.lower() or query_lower in field_desc.lower():
+                        matches.append(FieldMatch(
+                            source_key=source_key,
+                            asset_key=asset_key,
+                            qualified_key=f"{source_key}.{asset_key}",
+                            field_name=field_name,
+                            field_type=_extract_type(field_info),
+                            description=field_desc,
+                        ))
+
+        return FieldSearchResult(query=query, match_count=len(matches), matches=matches)
+    except Exception as e:
+        return ToolError(error=str(e))
+
+
+def compare_schemas(
+    ctx: ToolkitContext,
+    source_key_a: str,
+    asset_key_a: str,
+    source_key_b: str,
+    asset_key_b: str,
+) -> SchemaComparison | ToolError:
+    """Compare the schemas of two assets side by side.
+
+    Args:
+        source_key_a: Source key for the first asset.
+        asset_key_a: Asset key for the first asset.
+        source_key_b: Source key for the second asset.
+        asset_key_b: Asset key for the second asset.
+
+    Returns shared fields, fields unique to each asset, and any type mismatches.
+    """
+    try:
+        schema_a = _get_schema_properties(ctx, source_key_a, asset_key_a)
+        schema_b = _get_schema_properties(ctx, source_key_b, asset_key_b)
+
+        if isinstance(schema_a, ToolError):
+            return schema_a
+        if isinstance(schema_b, ToolError):
+            return schema_b
+
+        fields_a = set(schema_a.keys())
+        fields_b = set(schema_b.keys())
+
+        shared = fields_a & fields_b
+        only_a = fields_a - fields_b
+        only_b = fields_b - fields_a
+
+        shared_details = []
+        for field in sorted(shared):
+            type_a = _extract_type(schema_a[field])
+            type_b = _extract_type(schema_b[field])
+            shared_details.append(SharedField(
+                field=field,
+                type_a=type_a,
+                type_b=type_b,
+                type_match=type_a == type_b,
+            ))
+
+        return SchemaComparison(
+            asset_a=f"{source_key_a}.{asset_key_a}",
+            asset_b=f"{source_key_b}.{asset_key_b}",
+            shared_count=len(shared),
+            only_a_count=len(only_a),
+            only_b_count=len(only_b),
+            shared_fields=shared_details,
+            only_in_a=sorted(only_a),
+            only_in_b=sorted(only_b),
+        )
+    except Exception as e:
+        return ToolError(error=str(e))
+
+
+# -- Helpers -------------------------------------------------------------------
+
+
+def _extract_type(field_info: dict[str, Any]) -> str:
+    """Extract a human-readable type string from a JSON schema field definition."""
+    if "anyOf" in field_info:
+        types = [t.get("type", "unknown") for t in field_info["anyOf"] if t.get("type") != "null"]
+        return types[0] if len(types) == 1 else " | ".join(types) if types else "any"
+    return field_info.get("type", "unknown")
+
+
+def _get_schema_properties(ctx: ToolkitContext, source_key: str, asset_key: str) -> dict[str, Any] | ToolError:
+    """Retrieve the properties dict from an asset's JSON schema."""
+    defn = ctx.catalog.get(source_key)
+    if defn is None or defn.get("kind") != "source":
+        return ToolError(error=f"Source '{source_key}' not found in catalog")
+    for asset_def in defn.get("assets", []):
+        if asset_def.get("key") == asset_key:
+            schema = asset_def.get("asset_schema")
+            if not schema or "properties" not in schema:
+                return ToolError(error=f"Asset '{source_key}.{asset_key}' has no schema")
+            return _inline_refs(schema)["properties"]
+    return ToolError(error=f"Asset '{asset_key}' not found in source '{source_key}'")

interloper_toolkit/collection.py

@@ -0,0 +1,60 @@
+"""Collection tools — the org's component instances, read-only.
+
+Creation and connection operations stay with the agent; this module is
+shared with surfaces that must stay read-only.
+"""
+
+from __future__ import annotations
+
+from interloper.component import KINDS
+
+from interloper_toolkit.context import ToolkitContext
+from interloper_toolkit.models import ComponentCounts, ComponentList, ComponentSummary, ToolError
+
+
+def list_components(
+    ctx: ToolkitContext, kind: str | None = None
+) -> ComponentCounts | ComponentList | ToolError:
+    """List the components in the organisation's collection.
+
+    This answers "what do we have?" — what *could* be added (the catalog of
+    definitions) is the Catalog specialist's domain. Sensitive kinds
+    (connections, configs, resources) always return identity and metadata
+    only — never credential or config values.
+
+    Args:
+        kind: Component kind to list — e.g. 'source', 'connection',
+            'destination'. Omit for per-kind counts only; call again with a
+            kind for the entries.
+    """
+    try:
+        if kind is None:
+            counts: dict[str, int] = {}
+            for c in ctx.store.list_components(ctx.org_id):
+                counts[c.kind] = counts.get(c.kind, 0) + 1
+            return ComponentCounts(
+                component_counts=counts,
+                message="Call again with a kind for the entries.",
+            )
+        if kind not in KINDS:
+            return ToolError(error=f"Unknown kind '{kind}'", valid_kinds=sorted(KINDS.keys()))
+
+        results = []
+        for c in ctx.store.list_components(ctx.org_id, kinds=[kind]):
+            entry = ComponentSummary(
+                id=str(c.id),
+                key=c.key,
+                name=c.name,
+                type_name=(ctx.catalog.get(c.key) or {}).get("name", c.key),
+                created_at=c.created_at,
+            )
+            # Fail closed: a sensitive kind's config is (or wraps) credentials.
+            if not KINDS[kind].sensitive:
+                entry.config = c.config
+            if kind == "source":
+                entry.asset_count = len(c.children)
+            results.append(entry)
+
+        return ComponentList(kind=kind, count=len(results), components=results)
+    except Exception as e:
+        return ToolError(error=str(e))

interloper_toolkit/context.py

@@ -0,0 +1,55 @@
+"""Execution context and serialization for toolkit functions."""
+
+from __future__ import annotations
+
+import datetime
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+from uuid import UUID
+
+if TYPE_CHECKING:
+    from interloper_db.store import Store
+
+
+@dataclass(frozen=True)
+class ToolkitContext:
+    """Everything a toolkit function needs: persistence, catalog, tenant.
+
+    Attributes:
+        store: The Store to query.
+        catalog: The *dumped* catalog (``Catalog.dump()``) — a plain dict of
+            component definitions keyed by catalog key.
+        org_id: The organisation every query is scoped to.
+    """
+
+    store: Store
+    catalog: dict[str, Any]
+    org_id: UUID
+
+
+def serialize(obj: Any) -> Any:
+    """Convert a SQLModel instance (or collection) to a JSON-safe dict.
+
+    Recursively handles UUIDs, datetimes, dates, lists, and nested models.
+
+    Args:
+        obj: A SQLModel row, dict, list, or primitive.
+    """
+    if obj is None:
+        return None
+    if isinstance(obj, (str, int, float, bool)):
+        return obj
+    if isinstance(obj, UUID):
+        return str(obj)
+    if isinstance(obj, datetime.datetime):
+        return obj.isoformat()
+    if isinstance(obj, datetime.date):
+        return obj.isoformat()
+    if isinstance(obj, dict):
+        return {k: serialize(v) for k, v in obj.items()}
+    if isinstance(obj, (list, tuple)):
+        return [serialize(item) for item in obj]
+    # SQLModel / Pydantic BaseModel
+    if hasattr(obj, "model_dump"):
+        return serialize(obj.model_dump())
+    return str(obj)