catalogkit-query 0.1.0__tar.gz

catalogkit_query-0.1.0/PKG-INFO

@@ -0,0 +1,104 @@
+Metadata-Version: 2.4
+Name: catalogkit-query
+Version: 0.1.0
+Summary: Map one SQL statement into deterministic CatalogKit dependencies.
+Author: ClearMetric Labs
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/Clearmetric-Labs/CatalogKit
+Project-URL: Source, https://github.com/Clearmetric-Labs/CatalogKit
+Project-URL: Issues, https://github.com/Clearmetric-Labs/CatalogKit/issues
+Keywords: sql,lineage,dependencies,ctes,sqlglot
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: catalogkit-core>=0.1.0
+Requires-Dist: sqlglot>=25.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Provides-Extra: release
+Requires-Dist: build>=1.2.2; extra == "release"
+Requires-Dist: twine>=5.1.1; extra == "release"
+
+# catalogkit-query
+
+`catalogkit-query` maps one supported SQL statement into a deterministic `QueryMap`
+artifact so you can answer "what feeds what in this query?" fast.
+
+It is a narrow static-analysis tool:
+
+- input: exactly one SQL statement from one SQL file
+- output: canonical relations, relation usages, dependency edges, and warnings
+- no warehouse credentials
+- no dbt project
+- no AI key
+
+## Install
+
+```bash
+python -m pip install catalogkit-query
+```
+
+## Imports
+
+```python
+from catalogkit.query import build_catalog_artifact, build_query_map
+```
+
+For local development:
+
+```bash
+python -m pip install -e ../catalogkit-core
+python -m pip install -e ".[dev,release]"
+```
+
+## Quickstart
+
+```bash
+catalogkit-query --dialect postgres ./examples/ugly_real_world.sql
+catalogkit-query --dialect postgres --format json ./examples/ugly_real_world.sql
+```
+
+## Output Contract
+
+`catalogkit-query` preserves its public `QueryMap` shape:
+
+- `summary`
+- `relations`
+- `relation_usages`
+- `edges`
+- `outputs`
+- `warnings`
+
+For CatalogKit composition, the package also exposes a shared
+`CatalogArtifact` builder backed by `catalogkit-core`.
+
+The shared core artifact contains:
+
+- `version`
+- `nodes`
+- `edges`
+- `warnings`
+
+## Supported Statements
+
+`catalogkit-query` accepts exactly one supported statement per invocation:
+
+- `SELECT ...`
+- `INSERT ... SELECT ...`
+- `CREATE ... AS SELECT ...`
+
+Unsupported statement shapes fail loudly.
+
+## Contract Docs
+
+- [`../catalogkit-core/docs/contract.md`](../catalogkit-core/docs/contract.md)
+- [`docs/limitations.md`](docs/limitations.md)

catalogkit_query-0.1.0/README.md

@@ -0,0 +1,74 @@
+# catalogkit-query
+
+`catalogkit-query` maps one supported SQL statement into a deterministic `QueryMap`
+artifact so you can answer "what feeds what in this query?" fast.
+
+It is a narrow static-analysis tool:
+
+- input: exactly one SQL statement from one SQL file
+- output: canonical relations, relation usages, dependency edges, and warnings
+- no warehouse credentials
+- no dbt project
+- no AI key
+
+## Install
+
+```bash
+python -m pip install catalogkit-query
+```
+
+## Imports
+
+```python
+from catalogkit.query import build_catalog_artifact, build_query_map
+```
+
+For local development:
+
+```bash
+python -m pip install -e ../catalogkit-core
+python -m pip install -e ".[dev,release]"
+```
+
+## Quickstart
+
+```bash
+catalogkit-query --dialect postgres ./examples/ugly_real_world.sql
+catalogkit-query --dialect postgres --format json ./examples/ugly_real_world.sql
+```
+
+## Output Contract
+
+`catalogkit-query` preserves its public `QueryMap` shape:
+
+- `summary`
+- `relations`
+- `relation_usages`
+- `edges`
+- `outputs`
+- `warnings`
+
+For CatalogKit composition, the package also exposes a shared
+`CatalogArtifact` builder backed by `catalogkit-core`.
+
+The shared core artifact contains:
+
+- `version`
+- `nodes`
+- `edges`
+- `warnings`
+
+## Supported Statements
+
+`catalogkit-query` accepts exactly one supported statement per invocation:
+
+- `SELECT ...`
+- `INSERT ... SELECT ...`
+- `CREATE ... AS SELECT ...`
+
+Unsupported statement shapes fail loudly.
+
+## Contract Docs
+
+- [`../catalogkit-core/docs/contract.md`](../catalogkit-core/docs/contract.md)
+- [`docs/limitations.md`](docs/limitations.md)

catalogkit_query-0.1.0/pyproject.toml

@@ -0,0 +1,65 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "catalogkit-query"
+dynamic = ["version"]
+description = "Map one SQL statement into deterministic CatalogKit dependencies."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "Apache-2.0"
+authors = [
+    {name = "ClearMetric Labs"},
+]
+keywords = [
+    "sql",
+    "lineage",
+    "dependencies",
+    "ctes",
+    "sqlglot",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Database",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+    "catalogkit-core>=0.1.0",
+    "sqlglot>=25.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+]
+release = [
+    "build>=1.2.2",
+    "twine>=5.1.1",
+]
+
+[project.scripts]
+catalogkit-query = "catalogkit.query.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/Clearmetric-Labs/CatalogKit"
+Source = "https://github.com/Clearmetric-Labs/CatalogKit"
+Issues = "https://github.com/Clearmetric-Labs/CatalogKit/issues"
+
+[tool.setuptools.dynamic]
+version = {attr = "catalogkit.query._version.__version__"}
+
+[tool.setuptools.package-dir]
+"" = "src"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["catalogkit.query*"]
+namespaces = true

catalogkit_query-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

catalogkit_query-0.1.0/src/catalogkit/query/__init__.py

@@ -0,0 +1,39 @@
+"""Public package surface for catalogkit-query."""
+
+from __future__ import annotations
+
+from catalogkit.core import CatalogArtifact
+
+from ._version import __version__
+from .api import build_catalog_artifact, build_query_map, render_json, render_text
+from .errors import QueryMapContractError, QueryMapError, QueryMapParseError
+from .models import (
+    OutputColumn,
+    OutputSourceHint,
+    QueryMap,
+    QuerySummary,
+    Relation,
+    RelationEdge,
+    RelationUsage,
+    WarningEntry,
+)
+
+__all__ = [
+    "__version__",
+    "build_catalog_artifact",
+    "build_query_map",
+    "CatalogArtifact",
+    "OutputColumn",
+    "OutputSourceHint",
+    "QueryMap",
+    "QuerySummary",
+    "QueryMapContractError",
+    "QueryMapError",
+    "QueryMapParseError",
+    "Relation",
+    "RelationEdge",
+    "RelationUsage",
+    "render_json",
+    "render_text",
+    "WarningEntry",
+]

catalogkit_query-0.1.0/src/catalogkit/query/__main__.py

@@ -0,0 +1,9 @@
+"""Module entrypoint for `python -m catalogkit.query`."""
+
+from __future__ import annotations
+
+from .cli import main
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

catalogkit_query-0.1.0/src/catalogkit/query/_version.py

@@ -0,0 +1,3 @@
+"""Package version."""
+
+__version__ = "0.1.0"

catalogkit_query-0.1.0/src/catalogkit/query/api.py

@@ -0,0 +1,39 @@
+"""Public API for catalogkit-query."""
+
+from __future__ import annotations
+
+from catalogkit.core import CatalogArtifact
+
+from .build import build_catalog_artifact_from_parsed, build_query_map_from_parsed
+from .models import QueryMap
+from .parser import parse_statement
+from .render.json import render_json
+from .render.text import render_text
+
+
+def build_query_map(
+    sql: str,
+    *,
+    dialect: str,
+) -> QueryMap:
+    """Build the public catalogkit-query artifact for one SQL statement."""
+    parsed = parse_statement(sql, dialect=dialect)
+    return build_query_map_from_parsed(parsed)
+
+
+def build_catalog_artifact(
+    sql: str,
+    *,
+    dialect: str,
+) -> CatalogArtifact:
+    """Build the shared catalog artifact for CatalogKit composition."""
+    parsed = parse_statement(sql, dialect=dialect)
+    return build_catalog_artifact_from_parsed(parsed)
+
+
+__all__ = [
+    "build_catalog_artifact",
+    "build_query_map",
+    "render_json",
+    "render_text",
+]

catalogkit_query-0.1.0/src/catalogkit/query/ast_utils.py

@@ -0,0 +1,72 @@
+"""Shared sqlglot AST helpers for catalogkit-query."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+from sqlglot import exp
+
+
+def iter_ctes(root_expression: exp.Expression) -> Iterable[exp.CTE]:
+    """Yield every CTE in the parsed query."""
+    yield from root_expression.find_all(exp.CTE)
+
+
+def cte_name(cte: exp.CTE) -> str:
+    """Return the raw CTE name from the AST."""
+    alias = cte.alias
+    if isinstance(alias, str):
+        return alias
+    return str(alias or "").strip()
+
+
+def qualified_table_name(table: exp.Table) -> str:
+    """Return the dotted table name as it appears in the AST."""
+    parts = [
+        str(part).strip()
+        for part in (table.catalog, table.db, table.name)
+        if str(part or "").strip()
+    ]
+    return ".".join(parts)
+
+
+def has_join_ancestor(node: exp.Expression, *, stop_node: exp.Expression) -> bool:
+    """Detect whether a table node appears under a JOIN subtree."""
+    current = node.parent
+    while current is not None and current is not stop_node:
+        if isinstance(current, exp.Join):
+            return True
+        current = current.parent
+    return False
+
+
+def iter_table_nodes(expression: exp.Expression | None) -> Iterable[exp.Table]:
+    """Yield all table nodes under an expression."""
+    if expression is None:
+        return
+    for table in expression.find_all(exp.Table):
+        yield table
+
+
+def iter_table_nodes_skipping_ctes(
+    expression: exp.Expression | None,
+) -> Iterable[exp.Table]:
+    """Yield table nodes while skipping traversal into nested CTE definitions."""
+    if expression is None:
+        return
+    yield from _walk_tables(expression, skip_ctes=True)
+
+
+def _walk_tables(expression: exp.Expression, *, skip_ctes: bool) -> Iterable[exp.Table]:
+    if skip_ctes and isinstance(expression, exp.CTE):
+        return
+    if isinstance(expression, exp.Table):
+        yield expression
+        return
+    for child in expression.args.values():
+        if isinstance(child, list):
+            for item in child:
+                if isinstance(item, exp.Expression):
+                    yield from _walk_tables(item, skip_ctes=skip_ctes)
+        elif isinstance(child, exp.Expression):
+            yield from _walk_tables(child, skip_ctes=skip_ctes)

catalogkit_query-0.1.0/src/catalogkit/query/build.py

@@ -0,0 +1,249 @@
+"""Artifact assembly for catalogkit-query."""
+
+from __future__ import annotations
+
+from typing import Any, cast
+
+from catalogkit.core import CatalogArtifact, Edge, Evidence, Warning, merge
+from sqlglot import exp
+
+from .ctes import extract_dependency_edges
+from .errors import QueryMapContractError
+from .models import (
+    EdgeKind as QueryMapEdgeKind,
+    QueryMap,
+    QuerySummary,
+    RelationEdge,
+    RelationUsage,
+    WarningCode,
+    WarningEntry,
+)
+from .parser import ParsedStatement
+from .relations import RelationExtraction, extract_relations
+
+_QUERYMAP_EDGE_KINDS = {"depends_on", "joins"}
+_QUERYMAP_WARNING_CODES = {
+    "parse_recovered",
+    "select_star",
+    "table_star",
+    "ambiguous_output_source",
+    "unresolved_output_source",
+    "non_equi_join",
+    "unsupported_construct",
+}
+
+
+def build_query_map_from_parsed(parsed: ParsedStatement) -> QueryMap:
+    """Build the public catalogkit-query artifact for one parsed SQL statement."""
+    artifact, relation_extraction = _build_catalog_artifact_from_parsed(parsed)
+
+    edges = [_relation_edge_from_catalog_edge(edge) for edge in artifact.edges]
+    warnings = [_warning_entry_from_warning(warning) for warning in artifact.warnings]
+    summary = QuerySummary(
+        dialect=parsed.dialect,
+        statement_type=parsed.statement.key.lower(),
+        has_ctes=bool(relation_extraction.cte_nodes),
+        relation_count=len(relation_extraction.relations),
+        cte_count=len(relation_extraction.cte_nodes),
+        output_count=0,
+    )
+
+    return QueryMap(
+        version=artifact.version,
+        summary=summary,
+        relations=relation_extraction.relations,
+        relation_usages=[
+            RelationUsage(
+                relation_id=usage.relation_id,
+                alias=usage.alias,
+                context=usage.context,
+                sql=usage.sql,
+                normalized_sql=usage.sql,
+            )
+            for usage in relation_extraction.relation_usages
+        ],
+        edges=_dedupe_query_map_edges(edges),
+        outputs=[],
+        warnings=_dedupe_warnings(warnings),
+    )
+
+
+def build_catalog_artifact_from_parsed(parsed: ParsedStatement) -> CatalogArtifact:
+    """Build the shared catalog artifact used for CatalogKit composition."""
+    artifact, _ = _build_catalog_artifact_from_parsed(parsed)
+    return artifact
+
+
+def _build_catalog_artifact_from_parsed(
+    parsed: ParsedStatement,
+) -> tuple[CatalogArtifact, RelationExtraction]:
+    """Build the internal shared artifact and keep extraction metadata nearby."""
+    relation_extraction = extract_relations(parsed.root_expression, dialect=parsed.dialect)
+    dependency_extraction = extract_dependency_edges(
+        parsed.root_expression,
+        relation_extraction=relation_extraction,
+        dialect=parsed.dialect,
+    )
+
+    if not relation_extraction.nodes:
+        raise QueryMapContractError("No tables or CTE relations were found in the SQL statement.")
+
+    nodes_by_id = {node.id: node.model_copy(deep=True) for node in relation_extraction.nodes}
+    for usage in relation_extraction.relation_usages:
+        node = nodes_by_id[usage.relation_id]
+        node.evidence.append(
+            Evidence(
+                location=usage.context,
+                expression=usage.sql,
+                confidence="high",
+            )
+        )
+
+    artifact = CatalogArtifact(
+        nodes=sorted(nodes_by_id.values(), key=lambda node: node.id),
+        edges=dependency_extraction.edges,
+        warnings=[
+            *dependency_extraction.warnings,
+            *_extract_contract_warnings(parsed.root_expression, dialect=parsed.dialect),
+        ],
+    )
+    return merge(artifact), relation_extraction
+
+
+def _extract_contract_warnings(
+    root_expression: Any,
+    *,
+    dialect: str,
+) -> list[Warning]:
+    warnings: list[Warning] = []
+
+    union_expression = root_expression.find(exp.Union)
+    if union_expression is not None:
+        warnings.append(
+            Warning(
+                code="unsupported_construct",
+                message="UNION queries are mapped at the relation level only in the MVP.",
+                location=union_expression.sql(dialect=dialect),
+            )
+        )
+    intersect_expression = root_expression.find(exp.Intersect)
+    if intersect_expression is not None:
+        warnings.append(
+            Warning(
+                code="unsupported_construct",
+                message="INTERSECT queries are mapped at the relation level only in the MVP.",
+                location=intersect_expression.sql(dialect=dialect),
+            )
+        )
+    except_expression = root_expression.find(exp.Except)
+    if except_expression is not None:
+        warnings.append(
+            Warning(
+                code="unsupported_construct",
+                message="EXCEPT queries are mapped at the relation level only in the MVP.",
+                location=except_expression.sql(dialect=dialect),
+            )
+        )
+
+    for select in root_expression.find_all(exp.Select):
+        for selection in select.expressions:
+            if isinstance(selection, exp.Star):
+                warnings.append(
+                    Warning(
+                        code="select_star",
+                        message="SELECT * was detected; output mapping is deferred in the MVP.",
+                        location=selection.sql(dialect=dialect),
+                    )
+                )
+            elif isinstance(selection, exp.Column) and str(selection.name or "").strip() == "*":
+                warnings.append(
+                    Warning(
+                        code="table_star",
+                        message="table.* was detected; output mapping is deferred in the MVP.",
+                        location=selection.sql(dialect=dialect),
+                    )
+                )
+
+    for join in root_expression.find_all(exp.Join):
+        on_clause = join.args.get("on")
+        using_clause = join.args.get("using")
+        if on_clause is None and using_clause is None:
+            warnings.append(
+                Warning(
+                    code="unsupported_construct",
+                    message="JOIN without ON/USING is not modeled beyond relation dependency mapping.",
+                    location=join.sql(dialect=dialect),
+                )
+            )
+            continue
+        if on_clause is not None and not _is_equality_join(on_clause):
+            warnings.append(
+                Warning(
+                    code="non_equi_join",
+                    message="Non-equality join detected; MVP preserves relation dependencies but does not model join semantics.",
+                    location=join.sql(dialect=dialect),
+                )
+            )
+
+    return warnings
+
+
+def _is_equality_join(expression: Any) -> bool:
+    if isinstance(expression, exp.EQ):
+        return True
+    if isinstance(expression, exp.And):
+        return _is_equality_join(expression.left) and _is_equality_join(expression.right)
+    return False
+
+
+def _relation_edge_from_catalog_edge(edge: Edge) -> RelationEdge:
+    if edge.kind not in _QUERYMAP_EDGE_KINDS:
+        raise QueryMapContractError(
+            f"catalogkit-query cannot emit unsupported edge kind {edge.kind!r} in its public contract."
+        )
+    sql = edge.evidence[0].expression if edge.evidence else None
+    return RelationEdge(
+        kind=cast(QueryMapEdgeKind, edge.kind),
+        source_id=edge.source_id,
+        target_id=edge.target_id,
+        label=edge.label,
+        confidence=edge.confidence,
+        sql=sql,
+        normalized_sql=sql,
+    )
+
+
+def _warning_entry_from_warning(warning: Warning) -> WarningEntry:
+    if warning.code not in _QUERYMAP_WARNING_CODES:
+        raise QueryMapContractError(
+            f"catalogkit-query cannot emit unsupported warning code {warning.code!r} in its public contract."
+        )
+    return WarningEntry(
+        code=cast(WarningCode, warning.code),
+        message=warning.message,
+        location=warning.location,
+    )
+
+
+def _dedupe_query_map_edges(edges: list[RelationEdge]) -> list[RelationEdge]:
+    seen: set[tuple[str, str, str | None]] = set()
+    deduped: list[RelationEdge] = []
+    for edge in edges:
+        key = (edge.source_id, edge.target_id, edge.sql)
+        if key in seen:
+            continue
+        seen.add(key)
+        deduped.append(edge)
+    return deduped
+
+
+def _dedupe_warnings(warnings: list[WarningEntry]) -> list[WarningEntry]:
+    seen: set[tuple[str, str, str | None]] = set()
+    deduped: list[WarningEntry] = []
+    for warning in warnings:
+        key = (warning.code, warning.message, warning.location)
+        if key in seen:
+            continue
+        seen.add(key)
+        deduped.append(warning)
+    return deduped