sqf-py 0.1.0__py3-none-any.whl

sqf/__init__.py

@@ -0,0 +1,70 @@
+"""
+sqf — Semantic Query Fingerprinting for Snowflake
+==================================================
+
+A Python library that assigns a stable, content-addressed fingerprint to any
+SQL query by normalizing away syntactic noise.  Queries that are logically
+identical but written differently collapse to the same fingerprint, enabling
+deduplication analysis, cost attribution, and query-cache optimization.
+
+Quick start::
+
+    from sqf import fingerprint, are_equivalent, SQFAnalyzer
+
+    # Single query fingerprinting
+    h = fingerprint("SELECT a, b FROM t WHERE id = 1")
+
+    # Equivalence check
+    are_equivalent(
+        "SELECT a AS col1, b AS col2 FROM t WHERE id = 1",
+        "SELECT b, a FROM t WHERE id = ?",
+    )  # → True
+
+    # Bulk workload analysis
+    analyzer = SQFAnalyzer()
+    analyzer.ingest_sql(my_query_list)
+    print(analyzer.report().summary())
+"""
+
+from .fingerprint import (
+    fingerprint,
+    canonical_form,
+    are_equivalent,
+    QueryRecord,
+    SQFCluster,
+)
+from .normalizer import normalize
+from .analyzer import SQFAnalyzer, SQFReport
+from .generator import SyntheticWorkloadGenerator, FAMILIES, FAMILY_BY_ID
+from .snowflake import SnowflakeIngestor, ClusterStore, load_sql, SQL_FILES
+from .benchmark import (
+    BenchmarkRun,
+    BenchmarkSuite,
+    run_single,
+    run_benchmark_suite,
+    make_charts,
+)
+
+__version__ = "0.1.0"
+__all__ = [
+    "fingerprint",
+    "canonical_form",
+    "are_equivalent",
+    "normalize",
+    "QueryRecord",
+    "SQFCluster",
+    "SQFAnalyzer",
+    "SQFReport",
+    "SyntheticWorkloadGenerator",
+    "FAMILIES",
+    "FAMILY_BY_ID",
+    "SnowflakeIngestor",
+    "ClusterStore",
+    "load_sql",
+    "SQL_FILES",
+    "BenchmarkRun",
+    "BenchmarkSuite",
+    "run_single",
+    "run_benchmark_suite",
+    "make_charts",
+]

sqf/analyzer.py

@@ -0,0 +1,239 @@
+"""
+sqf.analyzer
+------------
+SQFAnalyzer: ingests a list of QueryRecord objects (or raw SQL strings),
+groups them into SQFClusters, and produces deduplication + cost metrics.
+
+This is the layer that connects to Snowflake's QUERY_HISTORY in production,
+or to the synthetic generator in benchmarks.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass, field
+from typing import Iterable
+
+from .fingerprint import QueryRecord, SQFCluster, fingerprint, canonical_form
+
+
+# ---------------------------------------------------------------------------
+# Analyzer
+# ---------------------------------------------------------------------------
+
+class SQFAnalyzer:
+    """
+    Core analysis engine.
+
+    Example::
+
+        analyzer = SQFAnalyzer()
+        analyzer.ingest(records)          # list[QueryRecord]
+        report = analyzer.report()
+        print(report.summary())
+    """
+
+    def __init__(self, dialect: str = "snowflake"):
+        self.dialect = dialect
+        self._clusters: dict[str, SQFCluster] = {}
+        self._unparseable: list[QueryRecord] = []
+
+    # ------------------------------------------------------------------
+    # Ingestion
+    # ------------------------------------------------------------------
+
+    def ingest(self, records: Iterable[QueryRecord]) -> "SQFAnalyzer":
+        """
+        Fingerprint each record and add it to the appropriate cluster.
+        Returns self for chaining.
+        """
+        for rec in records:
+            rec.compute_fingerprint(dialect=self.dialect)
+            if rec.sqf_hash is None:
+                self._unparseable.append(rec)
+                continue
+            if rec.sqf_hash not in self._clusters:
+                self._clusters[rec.sqf_hash] = SQFCluster(
+                    sqf_hash=rec.sqf_hash,
+                    canonical_form=rec.canonical or "",
+                )
+            self._clusters[rec.sqf_hash].records.append(rec)
+        return self
+
+    def ingest_sql(
+        self,
+        queries: Iterable[str],
+        credits_per_query: float = 0.01,
+    ) -> "SQFAnalyzer":
+        """
+        Convenience method: ingest plain SQL strings with uniform cost.
+        """
+        records = [
+            QueryRecord(
+                query_id=f"q{i:06d}",
+                sql=sql,
+                credits_used=credits_per_query,
+            )
+            for i, sql in enumerate(queries)
+        ]
+        return self.ingest(records)
+
+    # ------------------------------------------------------------------
+    # Accessors
+    # ------------------------------------------------------------------
+
+    @property
+    def clusters(self) -> list[SQFCluster]:
+        return list(self._clusters.values())
+
+    @property
+    def duplicate_clusters(self) -> list[SQFCluster]:
+        """Clusters with more than one execution (i.e. actual duplicates)."""
+        return [c for c in self.clusters if c.size > 1]
+
+    @property
+    def total_queries(self) -> int:
+        return sum(c.size for c in self.clusters) + len(self._unparseable)
+
+    @property
+    def total_credits(self) -> float:
+        return sum(c.total_credits for c in self.clusters)
+
+    # ------------------------------------------------------------------
+    # Report
+    # ------------------------------------------------------------------
+
+    def report(self) -> "SQFReport":
+        return SQFReport(
+            clusters=self.clusters,
+            unparseable_count=len(self._unparseable),
+            dialect=self.dialect,
+        )
+
+
+# ---------------------------------------------------------------------------
+# Report
+# ---------------------------------------------------------------------------
+
+@dataclass
+class SQFReport:
+    """
+    Immutable analysis report.  Produced by SQFAnalyzer.report().
+    """
+    clusters: list[SQFCluster]
+    unparseable_count: int = 0
+    dialect: str = "snowflake"
+
+    # ------------------------------------------------------------------
+    # Core metrics
+    # ------------------------------------------------------------------
+
+    @property
+    def total_executions(self) -> int:
+        return sum(c.size for c in self.clusters)
+
+    @property
+    def unique_fingerprints(self) -> int:
+        return len(self.clusters)
+
+    @property
+    def duplicate_executions(self) -> int:
+        """Total executions that are semantic duplicates of a prior run."""
+        return sum(max(0, c.size - 1) for c in self.clusters)
+
+    @property
+    def dedup_hit_rate(self) -> float:
+        """
+        Fraction of executions that are semantic duplicates.
+        This is the headline metric for the white paper.
+        """
+        if self.total_executions == 0:
+            return 0.0
+        return self.duplicate_executions / self.total_executions
+
+    @property
+    def total_credits(self) -> float:
+        return sum(c.total_credits for c in self.clusters)
+
+    @property
+    def wasted_credits(self) -> float:
+        return sum(c.wasted_credits for c in self.clusters)
+
+    @property
+    def credit_waste_rate(self) -> float:
+        if self.total_credits == 0:
+            return 0.0
+        return self.wasted_credits / self.total_credits
+
+    @property
+    def multi_variant_clusters(self) -> list[SQFCluster]:
+        """
+        Clusters where the same logical query was written in multiple
+        syntactically distinct ways — the most compelling white paper examples.
+        """
+        return [
+            c for c in self.clusters
+            if c.size > 1 and c.syntactic_variant_count > 1
+        ]
+
+    # ------------------------------------------------------------------
+    # Top offenders
+    # ------------------------------------------------------------------
+
+    def top_clusters_by_waste(self, n: int = 10) -> list[SQFCluster]:
+        return sorted(
+            [c for c in self.clusters if c.wasted_credits > 0],
+            key=lambda c: c.wasted_credits,
+            reverse=True,
+        )[:n]
+
+    def top_clusters_by_size(self, n: int = 10) -> list[SQFCluster]:
+        return sorted(self.clusters, key=lambda c: c.size, reverse=True)[:n]
+
+    # ------------------------------------------------------------------
+    # Summary text
+    # ------------------------------------------------------------------
+
+    def summary(self) -> str:
+        lines = [
+            "═" * 60,
+            "  Semantic Query Fingerprint (SQF) Analysis Report",
+            "═" * 60,
+            f"  Total query executions   : {self.total_executions:>8,}",
+            f"  Unique SQF fingerprints  : {self.unique_fingerprints:>8,}",
+            f"  Duplicate executions     : {self.duplicate_executions:>8,}",
+            f"  Unparseable queries      : {self.unparseable_count:>8,}",
+            "─" * 60,
+            f"  Dedup hit rate           : {self.dedup_hit_rate:>8.1%}",
+            f"  Total credits consumed   : {self.total_credits:>8.4f}",
+            f"  Credits wasted           : {self.wasted_credits:>8.4f}",
+            f"  Credit waste rate        : {self.credit_waste_rate:>8.1%}",
+            "─" * 60,
+            f"  Multi-variant clusters   : {len(self.multi_variant_clusters):>8,}",
+            "═" * 60,
+        ]
+        return "\n".join(lines)
+
+    def to_dict(self) -> dict:
+        """Serialize report metrics to a dict (for JSON export / charts)."""
+        return {
+            "total_executions": self.total_executions,
+            "unique_fingerprints": self.unique_fingerprints,
+            "duplicate_executions": self.duplicate_executions,
+            "unparseable_count": self.unparseable_count,
+            "dedup_hit_rate": round(self.dedup_hit_rate, 4),
+            "total_credits": round(self.total_credits, 6),
+            "wasted_credits": round(self.wasted_credits, 6),
+            "credit_waste_rate": round(self.credit_waste_rate, 4),
+            "multi_variant_clusters": len(self.multi_variant_clusters),
+            "top_clusters": [
+                {
+                    "sqf_hash": c.sqf_hash[:12] + "...",
+                    "size": c.size,
+                    "syntactic_variants": c.syntactic_variant_count,
+                    "wasted_credits": round(c.wasted_credits, 6),
+                    "canonical_preview": c.canonical_form[:120],
+                }
+                for c in self.top_clusters_by_waste(5)
+            ],
+        }