datalake-doco 0.1.27__py3-none-any.whl

datalake_doco/cli.py

@@ -0,0 +1,165 @@
+import argparse
+import os
+import sys
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+
+from datalake_doco.config import DEFAULT_HISTORY_LIMIT, OPEN_FILE_LIMIT
+from datalake_doco.models import Catalog, CatalogEntry, TableDoc
+from datalake_doco.readers.delta import DeltaReader
+from datalake_doco.scanner import scan
+from datalake_doco.writer import (
+    copy_frontend,
+    init_output_dir,
+    write_catalog,
+    write_llm_docs,
+    write_table_doc,
+)
+
+
+def generate(
+    uris: list[str],
+    output: Path,
+    sample_data_count: int,
+    sample_data_exclude_tables: str,
+    history_limit: int | None,
+    base_url: str | None = None,
+) -> None:
+    generated_at = datetime.now(UTC).isoformat()
+
+    excluded = {t.strip() for t in sample_data_exclude_tables.split(",") if t.strip()}
+
+    endpoint_url = os.environ.get("AWS_ENDPOINT_URL_S3")
+    storage_options: dict | None = {"AWS_ENDPOINT_URL": endpoint_url} if endpoint_url else None
+
+    all_discovered: list[tuple[str, str]] = []
+    for uri in uris:
+        print(f"Scanning {uri} ...")
+        discovered = scan(uri, endpoint_url=endpoint_url)
+        print(f"  Found {len(discovered)} table(s)")
+        all_discovered.extend(discovered)
+
+    total = len(all_discovered)
+    init_output_dir(output)
+    copy_frontend(output)
+    write_catalog(Catalog(generatedAt=generated_at, sources=uris, tables=[]), output)
+
+    table_docs = []
+    catalog_entries = []
+
+    def record(table_uri: str, doc: TableDoc) -> None:
+        table_docs.append(doc)
+        write_table_doc(output, doc)
+
+        doc_path = f"tables/{doc.fullyQualifiedName.replace('.', '/')}.json"
+        catalog_entries.append(
+            CatalogEntry(
+                fullyQualifiedName=doc.fullyQualifiedName,
+                name=doc.name,
+                uri=table_uri,
+                docPath=doc_path,
+                updatedAt=doc.updatedAt,
+                rowCount=doc.tableProfile.rowCount,
+                columnCount=doc.tableProfile.columnCount,
+                description=doc.description,
+                freshness=doc.freshness,
+            )
+        )
+
+        write_catalog(build_catalog(), output)
+
+    def build_catalog() -> Catalog:
+        return Catalog(
+            generatedAt=generated_at,
+            sources=uris,
+            tables=sorted(catalog_entries, key=lambda e: e.fullyQualifiedName),
+        )
+
+    for idx, (table_uri, table_name) in enumerate(all_discovered, start=1):
+        fqn = table_name.replace("/", ".")
+        sample_count = 0 if fqn in excluded else sample_data_count
+
+        sys.stdout.write(f"  [{idx}/{total}] {fqn} ... ")
+        sys.stdout.flush()
+        start = time.monotonic()
+
+        try:
+            doc = DeltaReader().read(table_uri, table_name, sample_count=sample_count, storage_options=storage_options, history_limit=history_limit)
+        except Exception as exc:
+            elapsed = time.monotonic() - start
+            sys.stdout.write(f"[error] ({elapsed:.1f}s): {exc}\n")
+            sys.stdout.flush()
+            continue
+
+        elapsed = time.monotonic() - start
+        sys.stdout.write(f"done ({elapsed:.1f}s)\n")
+        sys.stdout.flush()
+        record(table_uri, doc)
+
+    write_llm_docs(build_catalog(), table_docs, output, base_url)
+
+    print(f"\nDone. {len(table_docs)} table(s) documented → {output}")
+    print(f"Serve with: python -m http.server 8000 --directory {output}")
+
+
+def publish(output: Path) -> None:
+    """Copy the frontend (HTML/CSS/JS) to the output directory without re-profiling tables."""
+    copy_frontend(output)
+    print(f"Frontend published → {output}")
+    print(f"Serve with: python -m http.server 8000 --directory {output}")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="datalake-doco",
+        description="Generate static documentation for a datalake.",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    generate_parser = subparsers.add_parser("generate", help="Scan URIs, profile tables, write output.")
+    generate_parser.add_argument("uris", nargs="+", help="One or more URIs to scan (S3 or local paths).")
+    generate_parser.add_argument("--output", "-o", type=Path, default=Path("./datalake-docs"), help="Output directory.")
+    generate_parser.add_argument("--sample-data-count", type=int, default=0, help="Number of sample rows to include per table.")
+    generate_parser.add_argument("--sample-data-exclude-tables", default="", help="Comma-separated table names to exclude from sample data.")
+    generate_parser.add_argument(
+        "--history-limit", type=int, default=DEFAULT_HISTORY_LIMIT,
+        help=f"Max number of history entries to read per table; 0 reads all of them (default: {DEFAULT_HISTORY_LIMIT}).",
+    )
+    generate_parser.add_argument("--base-url", default=None, help="Public URL of the output directory; makes links in llms.txt and table Markdown absolute.")
+    generate_parser.set_defaults(func=lambda args: generate(
+        args.uris, args.output, args.sample_data_count, args.sample_data_exclude_tables, args.history_limit or None, args.base_url,
+    ))
+
+    publish_parser = subparsers.add_parser("publish", help="Copy frontend assets without re-profiling tables.")
+    publish_parser.add_argument("--output", "-o", type=Path, default=Path("./datalake-docs"), help="Output directory.")
+    publish_parser.set_defaults(func=lambda args: publish(args.output))
+
+    return parser
+
+
+def raise_open_file_limit(target: int = OPEN_FILE_LIMIT) -> None:
+    """Raise the soft open-file limit towards `target`, capped at the hard limit.
+
+    Reading S3-backed tables opens many connections in parallel, and the storage clients keep
+    them pooled, so the common default soft limit of 1024 can run out mid-run.
+    """
+    try:
+        import resource
+    except ImportError:  # not available on Windows
+        return
+    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+    wanted = target if hard == resource.RLIM_INFINITY else min(target, hard)
+    if soft != resource.RLIM_INFINITY and soft < wanted:
+        resource.setrlimit(resource.RLIMIT_NOFILE, (wanted, hard))
+
+
+def main(argv: list[str] | None = None) -> None:
+    raise_open_file_limit()
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

datalake_doco/config.py

@@ -0,0 +1,31 @@
+"""Tunable constants for time distributions and table freshness, kept in one place."""
+
+# Time zone that tz-aware timestamps are converted to before bucketing into days, weekdays and
+# hours. Naive timestamps and dates are always bucketed as stored.
+BUCKET_TIMEZONE = "UTC"
+
+# How many gap ranges the Markdown page lists before summarising the rest.
+MAX_GAP_RANGES = 5
+
+# Delta operations that change a table's data, as opposed to compaction or metadata commits.
+DATA_OPERATIONS = frozenset({"WRITE", "MERGE", "DELETE", "UPDATE"})
+
+# The typical write interval is the median of up to this many of the most recent gaps between
+# data commits.
+FRESHNESS_GAPS = 49
+
+# Data commits closer together than this are one burst (e.g. a backfill writing hundreds of
+# commits a second apart), so the gaps inside it don't count towards the typical interval.
+# Tables that genuinely write this often leave fewer than MIN_INTERVAL_GAPS longer gaps, and
+# fall back to the median of all gaps.
+BURST_GAP_SECONDS = 60
+MIN_INTERVAL_GAPS = 3
+
+# History entries read per table unless --history-limit says otherwise. Each entry is a separate
+# commit file to fetch, and busy tables keep thousands; 100 still covers FRESHNESS_GAPS gaps
+# between data commits when about half the commits are compaction.
+DEFAULT_HISTORY_LIMIT = 100
+
+# Soft open-file limit the CLI raises itself to at startup (capped at the hard limit), since
+# S3 reads keep many pooled connections open.
+OPEN_FILE_LIMIT = 65536

datalake_doco/freshness.py

@@ -0,0 +1,50 @@
+"""When a table's data was last written, and how that compares to how often it's written."""
+
+from datetime import UTC, datetime
+from itertools import pairwise
+from statistics import median
+
+from datalake_doco.config import (
+    BURST_GAP_SECONDS,
+    DATA_OPERATIONS,
+    FRESHNESS_GAPS,
+    MIN_INTERVAL_GAPS,
+)
+from datalake_doco.models import HistoryEntry
+
+
+def freshness(history: list[HistoryEntry], now: datetime) -> dict | None:
+    """Facts from the data-changing commits in `history`, measured up to `now`.
+
+    The typical interval is the median of the last FRESHNESS_GAPS gaps between data commits,
+    leaving out gaps under BURST_GAP_SECONDS so a burst of commits (a backfill) counts as one
+    write; with fewer than MIN_INTERVAL_GAPS such gaps, the table writes that often and every
+    gap counts. `ratio` is the time since the last write divided by the typical interval. Both
+    are None with fewer than two data commits. Returns None when there are no data commits.
+    """
+    writes = sorted(
+        (_parse(h.timestamp) for h in history if h.operation in DATA_OPERATIONS and h.timestamp),
+        reverse=True,
+    )
+    if not writes:
+        return None
+    since = (now - writes[0]).total_seconds()
+    gaps = [(a - b).total_seconds() for a, b in pairwise(writes)]
+    spaced = [g for g in gaps if g >= BURST_GAP_SECONDS][:FRESHNESS_GAPS]
+    burst_gap = BURST_GAP_SECONDS if len(spaced) >= MIN_INTERVAL_GAPS else None
+    used = spaced if burst_gap else gaps[:FRESHNESS_GAPS]
+    interval = median(used) if used else None
+    return {
+        "lastWrite": writes[0].isoformat(),
+        "secondsSinceLastWrite": round(since),
+        "typicalIntervalSeconds": round(interval, 1) if interval is not None else None,
+        "intervalGaps": len(used),
+        "burstGapSeconds": burst_gap,
+        "dataCommits": len(writes),
+        "ratio": round(since / interval, 2) if interval else None,
+    }
+
+
+def _parse(ts: str) -> datetime:
+    parsed = datetime.fromisoformat(ts)
+    return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)