view-scheduler 0.4.0__py3-none-any.whl

view_scheduler/__init__.py

@@ -0,0 +1,34 @@
+"""View materialization scheduler.
+
+Two tools for a Unity Catalog view:
+
+  * `advise` — suggest the time each day the view can run with every upstream
+    job already finished. Computed from history. ADVISORY.
+  * `can_refresh` — a runtime GATE. Right now, has every dependency of this
+    view refreshed recently enough? READY or BLOCKED.
+
+READ ONLY. This package never writes, alters, optimizes, vacuums or drops a
+user table. Metadata and history reads only.
+
+`advise` predicts. `can_refresh` checks. Use the first to set the schedule,
+the second to guard the run. See README.md.
+"""
+
+from .advice import Advice, DependencyRun, advise
+from .client import Workspace
+from .config import Config
+from .gate import Dependency, GateResult, ViewResult, can_refresh
+from .run import analyze
+
+__all__ = [
+    "advise",
+    "Advice",
+    "analyze",
+    "can_refresh",
+    "Config",
+    "Dependency",
+    "DependencyRun",
+    "GateResult",
+    "ViewResult",
+    "Workspace",
+]

view_scheduler/advice.py

@@ -0,0 +1,162 @@
+"""Advise when a view can run, as an object you can read or log.
+
+`advise()` is the easy front door to the analyzer. It prints a plain report and
+returns an `Advice` that carries the same numbers as fields.
+
+    from view_scheduler import advise
+
+    advice = advise("sales.gold.customer_360")   # WAREHOUSE_ID from the env
+    print(advice.safe_time)                      # 06:40   or None
+
+`analyze()` is the low-level call. It returns a tuple of raw pieces. Use it
+only when you need the per-table history objects.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import date, datetime, time
+
+from . import report
+from .client import Workspace, workspace_from_env
+from .config import DEFAULT, Config
+from .models import Skipped
+from .recommend import Recommendation
+from .run import collect
+
+
+@dataclass(frozen=True)
+class DependencyRun:
+    """One upstream job, and what its history looked like."""
+
+    name: str
+    runs: str                 # "daily", "Mondays", "never", ...
+    usual: time | None        # its usual finish time
+    latest: time | None       # its latest finish time
+    days: int                 # days it ran in the period
+    missed: int               # days it did not run
+    note: str = ""
+
+
+@dataclass(frozen=True)
+class Advice:
+    """When a view can safely run. One object, ready to read or log."""
+
+    view: str
+    safe_time: time | None
+    earlier_time: time | None
+    next_day: bool
+    earlier_next_day: bool
+    days_used: int
+    period_days: int
+    timezone: str
+    dependencies: tuple[DependencyRun, ...] = ()
+    unreadable: tuple[Skipped, ...] = ()
+    finished_by_day: tuple[tuple[date, time], ...] = ()
+    warnings: tuple[str, ...] = ()
+    text: str = field(default="")
+
+    def render(self) -> str:
+        return self.text
+
+    def to_dict(self) -> dict:
+        """JSON-ready. Safe to log from a job."""
+        return {
+            "view": self.view,
+            "safe_time": _clock(self.safe_time),
+            "earlier_time": _clock(self.earlier_time),
+            "next_day": self.next_day,
+            "days_used": self.days_used,
+            "period_days": self.period_days,
+            "timezone": self.timezone,
+            "dependencies": [
+                {
+                    "name": row.name,
+                    "runs": row.runs,
+                    "usual": _clock(row.usual),
+                    "latest": _clock(row.latest),
+                    "days": row.days,
+                    "missed": row.missed,
+                }
+                for row in self.dependencies
+            ],
+            "unreadable": [
+                {"name": item.name, "reason": item.reason}
+                for item in self.unreadable
+            ],
+            "finished_by_day": [
+                {"day": day.isoformat(), "time": moment.strftime("%H:%M")}
+                for day, moment in self.finished_by_day
+            ],
+            "warnings": list(self.warnings),
+        }
+
+    def __str__(self) -> str:
+        return self.text
+
+
+def advise(
+    view: str,
+    ws: Workspace | None = None,
+    config: Config | None = None,
+    *,
+    today: date | None = None,
+    quiet: bool = False,
+) -> Advice:
+    """Suggest the time each day this view can run with every input ready.
+
+    `ws` defaults to a Workspace built from the WAREHOUSE_ID environment
+    variable. Pass `today` to make a test reproducible. `quiet=True` suppresses
+    the printed report; the result is returned either way.
+    """
+    settings = config or DEFAULT
+    workspace = ws or workspace_from_env()
+    moment = today or datetime.now(settings.tz).date()
+
+    result, _, skipped = collect(view, workspace, settings, moment)
+    advice = _build(view, result, skipped, settings)
+
+    if not quiet:
+        print(advice)
+    return advice
+
+
+def _build(view: str, result: Recommendation, skipped: list[Skipped],
+           config: Config) -> Advice:
+    rows = tuple(
+        DependencyRun(
+            name=row.name,
+            runs=row.refreshes,
+            usual=row.usual,
+            latest=row.worst,
+            days=row.observed_days,
+            missed=row.missed,
+        )
+        for row in sorted(result.rows, key=lambda r: r.name)
+    )
+    earlier = (result.earlier
+               if result.earlier and result.earlier != result.safe else None)
+    finished = tuple(
+        (day, moment)
+        for day, moment in sorted(result.all_clear.items(),
+                                  key=lambda kv: kv[1].time(), reverse=True)
+    )
+    return Advice(
+        view=view,
+        safe_time=result.safe,
+        earlier_time=earlier,
+        next_day=bool(result.safe and result.safe_next_day),
+        earlier_next_day=bool(earlier and result.earlier_next_day),
+        days_used=result.n_days,
+        period_days=result.span_days,
+        timezone=config.timezone,
+        dependencies=rows,
+        unreadable=tuple(skipped),
+        finished_by_day=finished,
+        warnings=tuple(report.warnings(result, skipped)),
+        text=report.render(view, result, skipped, config.timezone),
+    )
+
+
+def _clock(value: time | None) -> str | None:
+    return value.strftime("%H:%M") if value else None

view_scheduler/cli.py

@@ -0,0 +1,71 @@
+"""Command line entry point: `view-scheduler <catalog.schema.view>`."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+
+from . import obs
+from .advice import advise
+from .client import Workspace
+from .config import Config
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="view-scheduler",
+        description="Suggest the time each day a Unity Catalog view can run with "
+                    "every upstream job already finished.",
+        epilog="ADVISORY: this suggests a time from history. It does not check "
+               "today. For a runtime check, use can_refresh() from the library.",
+    )
+    parser.add_argument("view", help="fully qualified view, e.g. catalog.schema.view")
+    parser.add_argument(
+        "--warehouse-id", default=os.environ.get("WAREHOUSE_ID"),
+        help="SQL warehouse to run history reads against "
+             "(default: $WAREHOUSE_ID)")
+    # The three analysis settings. The gate's fresh_hours is library-only.
+    parser.add_argument("--days", type=int, default=Config.days_history,
+                        metavar="N", help="days of history to read")
+    parser.add_argument("--buffer", type=int, default=Config.buffer_min,
+                        metavar="MIN", help="safety gap added to the safe time")
+    parser.add_argument("--timezone", default=Config.timezone,
+                        help="timezone the advice is expressed in")
+    parser.add_argument("--json", action="store_true",
+                        help="print the machine-readable JSON instead of the report")
+    parser.add_argument("--quiet", action="store_true",
+                        help="suppress the per-call log; print only the report")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+
+    if not args.warehouse_id:
+        # Fail on the missing input rather than deep inside the first SQL call.
+        print("error: no warehouse. Pass --warehouse-id or set $WAREHOUSE_ID.",
+              file=sys.stderr)
+        return 2
+
+    obs.setup(level=40 if args.quiet else 20)
+    config = Config(days_history=args.days, buffer_min=args.buffer,
+                    timezone=args.timezone)
+    workspace = Workspace(warehouse_id=args.warehouse_id)
+
+    try:
+        advice = advise(args.view, workspace, config, quiet=True)
+    except Exception as exc:
+        print(f"error: {type(exc).__name__}: {exc}", file=sys.stderr)
+        return 1
+
+    if args.json:
+        print(json.dumps(advice.to_dict(), indent=2))
+    else:
+        print("\n" + str(advice) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

view_scheduler/client.py

@@ -0,0 +1,222 @@
+"""I/O layer. Read-only: metadata lookups, DESCRIBE HISTORY, event log queries."""
+
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime
+from typing import Any
+from urllib.parse import quote
+
+from databricks.sdk import WorkspaceClient
+from databricks.sdk.service.sql import ExecuteStatementRequestOnWaitTimeout
+
+from . import obs
+from .history import qualifying
+
+
+def quote_name(full_name: str) -> str:
+    """Backtick each part of a dotted Unity Catalog name.
+
+    A hyphen, a space, or a reserved word is a syntax error unless the name is
+    quoted, and an unquoted name is an injection surface. An internal backtick
+    is doubled so it cannot close the identifier early.
+    """
+    parts = [part.strip() for part in str(full_name).split(".")]
+    if not parts or any(not part for part in parts):
+        raise ValueError(f"not a Unity Catalog name: {full_name!r}")
+    return ".".join("`" + part.replace("`", "``") + "`" for part in parts)
+
+
+class Workspace:
+    """Thin wrapper over the SDK. One method per external call we make."""
+
+    def __init__(self, client: WorkspaceClient | None = None,
+                 warehouse_id: str | None = None):
+        self.w = client or WorkspaceClient()
+        self.warehouse_id = warehouse_id
+
+    # --- Unity Catalog -------------------------------------------------------
+
+    def get_table(self, full_name: str) -> dict:
+        """Raw GET for one table's metadata, including view_dependencies."""
+        path = quote(full_name, safe=".")
+        with obs.call("uc.tables.get", full_name):
+            return self.w.api_client.do(
+                "GET", f"/api/2.1/unity-catalog/tables/{path}")
+
+    # --- SQL -----------------------------------------------------------------
+
+    def sql(self, statement: str, target: str) -> list[dict[str, Any]]:
+        """Run a read-only statement. Return every row as a dict of strings.
+
+        Follows result chunking to the end, and refuses a truncated result, so a
+        caller never silently sees only the first page of a large result.
+        """
+        if self.warehouse_id is None:
+            raise ValueError("warehouse_id is required for SQL calls")
+
+        with obs.call("sql", target):
+            resp = self.w.statement_execution.execute_statement(
+                warehouse_id=self.warehouse_id,
+                statement=statement,
+                wait_timeout="50s",
+                on_wait_timeout=ExecuteStatementRequestOnWaitTimeout.CANCEL,
+            )
+            state = resp.status.state.value if resp.status and resp.status.state else "?"
+            if state != "SUCCEEDED":
+                message = (resp.status.error.message
+                           if resp.status and resp.status.error else state)
+                raise RuntimeError(message)
+
+            if resp.manifest and resp.manifest.truncated:
+                raise RuntimeError(
+                    f"the warehouse truncated the result for {target}; "
+                    f"the answer would be incomplete")
+
+            columns = [c.name for c in resp.manifest.schema.columns]
+            rows = list((resp.result.data_array if resp.result else None) or [])
+            chunk = resp.result.next_chunk_index if resp.result else None
+            while chunk is not None:
+                page = self.w.statement_execution.get_statement_result_chunk_n(
+                    resp.statement_id, chunk)
+                rows.extend(page.data_array or [])
+                chunk = page.next_chunk_index
+
+            return [dict(zip(columns, row)) for row in rows]
+
+
+def workspace_from_env() -> "Workspace":
+    """Build a Workspace from $WAREHOUSE_ID, or say exactly what is missing."""
+    warehouse = os.environ.get("WAREHOUSE_ID")
+    if not warehouse:
+        raise ValueError(
+            "no warehouse: pass Workspace(warehouse_id='...') "
+            "or set the WAREHOUSE_ID environment variable")
+    return Workspace(warehouse_id=warehouse)
+
+
+def failure_category(exc: Exception) -> str:
+    """Classify an SDK/SQL error as 'permission', 'not_found', 'mv_no_history', or 'other'."""
+    text = str(exc).lower()
+    if ("permission" in text or "does not have" in text or "403" in text
+            or "401" in text or "unauthorized" in text):
+        return "permission"
+    if "does not exist" in text or "not found" in text or "404" in text:
+        return "not_found"
+    if "expect_table_not_view" in text:
+        return "mv_no_history"
+    return "other"
+
+
+def parse_json_field(value: Any) -> dict | None:
+    """Parse a JSON string or pass through a dict. None if absent or unparseable."""
+    if value is None:
+        return None
+    if isinstance(value, dict):
+        return value
+    try:
+        parsed = json.loads(value)
+    except (TypeError, ValueError):
+        return None
+    return parsed if isinstance(parsed, dict) else None
+
+
+# --- history readers ---------------------------------------------------------
+
+def _from_epoch(value):
+    """Seconds since the epoch → an aware UTC datetime.
+
+    Timestamps are read as epochs rather than as the API's rendered string
+    because a Spark TIMESTAMP is an instant, and `unix_timestamp` returns that
+    instant regardless of the warehouse's session timezone. The rendered string
+    carries a "Z" suffix whose accuracy depends on that session setting, which
+    is warehouse configuration this tool does not control. Verified identical to
+    the string path on a UTC warehouse; correct on any other.
+    """
+    from datetime import timezone
+    return datetime.fromtimestamp(int(value), tz=timezone.utc)
+
+
+def read_history(ws: "Workspace", table: str, *,
+                 with_metrics: bool = True) -> list:
+    """Every commit for a physical Delta table, newest first.
+
+    `with_metrics=False` drops the wide operationMetrics column. The gate only
+    classifies commits, so it never needs it; analyze() does.
+    """
+    from .models import Commit
+    metrics = ", operationMetrics" if with_metrics else ""
+    rows = ws.sql(
+        "SELECT version, unix_timestamp(timestamp) AS ts_epoch, operation, "
+        f"operationParameters{metrics} "
+        f"FROM (DESCRIBE HISTORY {quote_name(table)}) "
+        "ORDER BY version DESC",
+        target=table)
+    return [
+        Commit(
+            version=int(r.get("version") or 0),
+            timestamp=_from_epoch(r["ts_epoch"]),
+            operation=str(r.get("operation") or ""),
+            params=parse_json_field(r.get("operationParameters")) or {},
+            metrics=(parse_json_field(r.get("operationMetrics"))
+                     if with_metrics else None),
+        )
+        for r in rows
+    ]
+
+
+def read_mv_refreshes(ws: "Workspace", mv: str) -> list:
+    """Completed MV refreshes from the pipeline event log.
+
+    DESCRIBE HISTORY is rejected on a materialized view, so this is the only
+    way to see when it last refreshed.
+    """
+    from .models import Commit
+    statement = (
+        "SELECT unix_timestamp(timestamp) AS ts_epoch "
+        f"FROM event_log(TABLE({quote_name(mv)})) "
+        "WHERE event_type = 'update_progress' "
+        "AND get_json_object(details, '$.update_progress.state') = 'COMPLETED'"
+    )
+    rows = ws.sql(statement, target=mv)
+    return [
+        Commit(version=i, timestamp=_from_epoch(r["ts_epoch"]),
+               operation="MV REFRESH", params={}, metrics=None)
+        for i, r in enumerate(rows)
+    ]
+
+
+def read_commits(ws: "Workspace", table: str, *,
+                 with_metrics: bool = True) -> list:
+    """History for any physical object: a table, streaming table, or MV.
+
+    DESCRIBE HISTORY covers tables and streaming tables. A materialized view
+    rejects it, so fall back to the pipeline event log.
+    """
+    try:
+        return read_history(ws, table, with_metrics=with_metrics)
+    except Exception as exc:
+        if "expect_table_not_view" in str(exc).lower():
+            with obs.call("mv.event_log", table):
+                return read_mv_refreshes(ws, table)
+        raise
+
+
+def read_latest_commit(ws: "Workspace", table: str, *,
+                       not_after: int | None = None) -> datetime | None:
+    """The newest qualifying write for a table or an MV, or None.
+
+    This is the gate's one question. A write dated after `not_after` (epoch
+    seconds) is ignored, so a clock-skewed future commit cannot make a stale
+    table look fresh.
+    """
+    newest: datetime | None = None
+    for commit in read_commits(ws, table, with_metrics=False):
+        if not qualifying(commit):
+            continue
+        if not_after is not None and commit.timestamp.timestamp() > not_after:
+            continue
+        if newest is None or commit.timestamp > newest:
+            newest = commit.timestamp
+    return newest

view_scheduler/config.py

@@ -0,0 +1,29 @@
+"""Configuration. Keep it small — ask before adding a field."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from zoneinfo import ZoneInfo
+
+
+@dataclass(frozen=True)
+class Config:
+
+    days_history: int = 30
+    buffer_min: int = 15
+    timezone: str = "Asia/Kolkata"
+    #: Gate only. A dependency is fresh when it wrote within this many hours.
+    fresh_hours: int = 24
+
+    def __post_init__(self) -> None:
+        if self.fresh_hours <= 0:
+            raise ValueError(f"fresh_hours must be positive, got {self.fresh_hours}")
+        if self.days_history <= 0:
+            raise ValueError(f"days_history must be positive, got {self.days_history}")
+
+    @property
+    def tz(self) -> ZoneInfo:
+        return ZoneInfo(self.timezone)
+
+
+DEFAULT = Config()