sql-glider 0.1.14__tar.gz → 0.1.16__tar.gz

{sql_glider-0.1.14 → sql_glider-0.1.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sql-glider
-Version: 0.1.14
+Version: 0.1.16
 Summary: SQL Utility Toolkit for better understanding, use, and governance of your queries in a native environment.
 Project-URL: Homepage, https://github.com/rycowhi/sql-glider/
 Project-URL: Repository, https://github.com/rycowhi/sql-glider/

{sql_glider-0.1.14 → sql_glider-0.1.16}/src/sqlglider/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.14'
-__version_tuple__ = version_tuple = (0, 1, 14)
+__version__ = version = '0.1.16'
+__version_tuple__ = version_tuple = (0, 1, 16)
 
 __commit_id__ = commit_id = None

{sql_glider-0.1.14 → sql_glider-0.1.16}/src/sqlglider/cli.py

@@ -171,6 +171,12 @@ def lineage(
         "--no-star",
         help="Fail if SELECT * cannot be resolved to actual columns",
     ),
+    provide_schema: Optional[Path] = typer.Option(
+        None,
+        "--provide-schema",
+        exists=True,
+        help="Path to a schema file (JSON, CSV, or text) for star resolution",
+    ),
 ) -> None:
     """
     Analyze column or table lineage for a SQL file.
@@ -266,8 +272,15 @@ def lineage(
             source_path=source_path,
         )
 
+        # Load provided schema if specified
+        schema = None
+        if provide_schema:
+            from sqlglider.graph.formatters import load_schema_file
+
+            schema = load_schema_file(provide_schema)
+
         # Create analyzer
-        analyzer = LineageAnalyzer(sql, dialect=dialect, no_star=no_star)
+        analyzer = LineageAnalyzer(sql, dialect=dialect, no_star=no_star, schema=schema)
 
         # Unified lineage analysis (handles both single and multi-query files)
         results = analyzer.analyze_queries(
@@ -1292,6 +1305,13 @@ def graph_build(
         "--dump-schema-format",
         help="Format for dumped schema: 'text' (default), 'json', or 'csv'",
     ),
+    provide_schema: Optional[Path] = typer.Option(
+        None,
+        "--provide-schema",
+        exists=True,
+        help="Path to a schema file (JSON, CSV, or text) to use for star resolution. "
+        "Can be combined with --resolve-schema to merge file-extracted schema on top.",
+    ),
     strict_schema: bool = typer.Option(
         False,
         "--strict-schema",
@@ -1434,6 +1454,17 @@ def graph_build(
             strict_schema=strict_schema,
         )
 
+        # Load provided schema file if specified
+        if provide_schema:
+            from sqlglider.graph.formatters import load_schema_file
+
+            loaded_schema = load_schema_file(provide_schema)
+            builder.set_schema(loaded_schema)
+            console.print(
+                f"[green]Loaded schema from {provide_schema} "
+                f"({len(loaded_schema)} table(s))[/green]"
+            )
+
         # Collect file paths for schema extraction
         manifest_files, path_files = _collect_sql_files(
             paths, manifest, recursive, glob_pattern

{sql_glider-0.1.14 → sql_glider-0.1.16}/src/sqlglider/graph/builder.py

@@ -303,6 +303,21 @@ class GraphBuilder:
                 self.add_file(file_path, dialect)
         return self
 
+    def set_schema(self, schema: Dict[str, Dict[str, str]]) -> "GraphBuilder":
+        """Pre-seed the resolved schema from an external source.
+
+        This allows skipping the schema extraction pass when the schema
+        is already known (e.g., loaded from a file).
+
+        Args:
+            schema: Schema dictionary mapping table names to column dicts.
+
+        Returns:
+            self for method chaining
+        """
+        self._resolved_schema = schema
+        return self
+
     def extract_schemas(
         self,
         file_paths: List[Path],
@@ -325,6 +340,7 @@ class GraphBuilder:
             file_paths,
             dialect=file_dialect,
             sql_preprocessor=self.sql_preprocessor,
+            initial_schema=self._resolved_schema if self._resolved_schema else None,
             strict_schema=self.strict_schema,
             catalog_type=self.catalog_type,
             catalog_config=self.catalog_config,

{sql_glider-0.1.14 → sql_glider-0.1.16}/src/sqlglider/graph/formatters.py

@@ -1,8 +1,9 @@
-"""Output formatters for resolved schema data."""
+"""Output formatters and parsers for resolved schema data."""
 
 import csv
 import json
 from io import StringIO
+from pathlib import Path
 from typing import Dict
 
 SchemaDict = Dict[str, Dict[str, str]]
@@ -96,3 +97,93 @@ def format_schema(schema: SchemaDict, output_format: str = "text") -> str:
             f"Invalid schema format '{output_format}'. Use 'text', 'json', or 'csv'."
         )
     return formatter(schema)
+
+
+def parse_schema_json(content: str) -> SchemaDict:
+    """Parse schema from JSON format.
+
+    Args:
+        content: JSON string with table -> {column -> type} structure.
+
+    Returns:
+        Parsed schema dictionary.
+    """
+    return json.loads(content)  # type: ignore[no-any-return]
+
+
+def parse_schema_csv(content: str) -> SchemaDict:
+    """Parse schema from CSV format.
+
+    Expects columns: table, column, type.
+
+    Args:
+        content: CSV string with header row.
+
+    Returns:
+        Parsed schema dictionary.
+    """
+    schema: SchemaDict = {}
+    reader = csv.DictReader(StringIO(content))
+    for row in reader:
+        table = row["table"]
+        column = row["column"]
+        col_type = row.get("type", "UNKNOWN")
+        if table not in schema:
+            schema[table] = {}
+        schema[table][column] = col_type
+    return schema
+
+
+def parse_schema_text(content: str) -> SchemaDict:
+    """Parse schema from indented text format.
+
+    Expected format:
+        table_name
+          column1
+          column2
+
+        other_table
+          col_a
+
+    Args:
+        content: Text-formatted schema string.
+
+    Returns:
+        Parsed schema dictionary.
+    """
+    schema: SchemaDict = {}
+    current_table: str | None = None
+    for line in content.splitlines():
+        if not line or not line.strip():
+            continue
+        if line.startswith("  "):
+            if current_table is not None:
+                schema[current_table][line.strip()] = "UNKNOWN"
+        else:
+            current_table = line.strip()
+            schema[current_table] = {}
+    return schema
+
+
+def load_schema_file(path: Path) -> SchemaDict:
+    """Load a schema file, auto-detecting format from extension.
+
+    `.json` → JSON, `.csv` → CSV, otherwise text.
+
+    Args:
+        path: Path to schema file.
+
+    Returns:
+        Parsed schema dictionary.
+
+    Raises:
+        FileNotFoundError: If the file does not exist.
+    """
+    content = path.read_text(encoding="utf-8")
+    suffix = path.suffix.lower()
+    if suffix == ".json":
+        return parse_schema_json(content)
+    elif suffix == ".csv":
+        return parse_schema_csv(content)
+    else:
+        return parse_schema_text(content)

{sql_glider-0.1.14 → sql_glider-0.1.16}/src/sqlglider/lineage/analyzer.py

@@ -11,6 +11,48 @@ from sqlglot.lineage import Node, lineage
 from sqlglider.global_models import AnalysisLevel
 
 
+def _flat_schema_to_nested(
+    schema: Dict[str, Dict[str, str]],
+) -> Dict[str, object]:
+    """Convert flat dot-notation schema keys to the nested dict structure sqlglot expects.
+
+    sqlglot's MappingSchema requires consistent nesting depth across all tables.
+    Flat keys like ``"db.table"`` are split on dots and nested accordingly.
+    Shorter keys are padded with empty-string prefixes to match the max depth.
+
+    Examples::
+
+        {"users": {"id": "UNKNOWN"}}
+        → {"users": {"id": "UNKNOWN"}}  (depth 1, no change)
+
+        {"db.users": {"id": "UNKNOWN"}, "my_view": {"x": "UNKNOWN"}}
+        → {"db": {"users": {"id": "UNKNOWN"}}, "": {"my_view": {"x": "UNKNOWN"}}}
+    """
+    if not schema:
+        return {}
+
+    # Split all keys into parts
+    entries = [(key.split("."), cols) for key, cols in schema.items()]
+    max_depth = max(len(parts) for parts, _ in entries)
+
+    # If all keys are single-part (unqualified), return as-is
+    if max_depth == 1:
+        return schema  # type: ignore[return-value]
+
+    # Pad shorter keys with empty-string prefixes to match max depth
+    nested: Dict[str, object] = {}
+    for parts, cols in entries:
+        while len(parts) < max_depth:
+            parts.insert(0, "")
+        d: Dict[str, object] = nested
+        for part in parts[:-1]:
+            if part not in d:
+                d[part] = {}
+            d = d[part]  # type: ignore[assignment]
+        d[parts[-1]] = cols
+    return nested
+
+
 class StarResolutionError(Exception):
     """Raised when SELECT * cannot be resolved and no_star mode is enabled."""
 
@@ -860,8 +902,10 @@ class LineageAnalyzer:
             current_query_sql = self.expr.sql(dialect=self.dialect)
 
         # Prune schema to only tables referenced in this query to avoid
-        # sqlglot.lineage() performance degradation with large schema dicts
-        pruned_schema: Optional[Dict[str, Dict[str, str]]] = None
+        # sqlglot.lineage() performance degradation with large schema dicts.
+        # Then convert from flat dot-notation keys to the nested dict structure
+        # that sqlglot's MappingSchema expects.
+        lineage_schema: Optional[Dict[str, object]] = None
         if self._file_schema:
             referenced = {t.lower() for t in self._get_query_tables()}
             pruned_schema = {
@@ -869,8 +913,8 @@ class LineageAnalyzer:
                 for table, cols in self._file_schema.items()
                 if table.lower() in referenced
             }
-            if not pruned_schema:
-                pruned_schema = None
+            if pruned_schema:
+                lineage_schema = _flat_schema_to_nested(pruned_schema)
 
         for col in columns_to_analyze:
             try:
@@ -883,7 +927,7 @@ class LineageAnalyzer:
                     lineage_col,
                     current_query_sql,
                     dialect=self.dialect,
-                    schema=pruned_schema,
+                    schema=lineage_schema,
                 )
 
                 # Collect all source columns

{sql_glider-0.1.14 → sql_glider-0.1.16}/src/sqlglider/schema/extractor.py

@@ -153,6 +153,7 @@ def extract_and_resolve_schema(
     file_paths: List[Path],
     dialect: str = "spark",
     sql_preprocessor: Optional[SqlPreprocessor] = None,
+    initial_schema: Optional[SchemaDict] = None,
     strict_schema: bool = False,
     catalog_type: Optional[str] = None,
     catalog_config: Optional[Dict[str, object]] = None,
@@ -167,6 +168,7 @@ def extract_and_resolve_schema(
         file_paths: SQL files to extract schema from.
         dialect: SQL dialect.
         sql_preprocessor: Optional SQL preprocessor.
+        initial_schema: Optional starting schema to build upon.
         strict_schema: If True, fail on ambiguous column attribution.
         catalog_type: Optional catalog provider name.
         catalog_config: Optional provider-specific configuration dict.
@@ -183,6 +185,7 @@ def extract_and_resolve_schema(
         file_paths,
         dialect=dialect,
         sql_preprocessor=sql_preprocessor,
+        initial_schema=initial_schema,
         strict_schema=strict_schema,
         console=console,
     )

{sql_glider-0.1.14 → sql_glider-0.1.16}/tests/sqlglider/graph/test_formatters.py

@@ -7,6 +7,10 @@ from sqlglider.graph.formatters import (
     format_schema_csv,
     format_schema_json,
     format_schema_text,
+    load_schema_file,
+    parse_schema_csv,
+    parse_schema_json,
+    parse_schema_text,
 )
 
 
@@ -84,3 +88,61 @@ class TestFormatSchema:
     def test_invalid_format(self, sample_schema):
         with pytest.raises(ValueError, match="Invalid schema format"):
             format_schema(sample_schema, "xml")
+
+
+class TestParseSchemaJson:
+    def test_round_trip(self, sample_schema):
+        content = format_schema_json(sample_schema)
+        parsed = parse_schema_json(content)
+        assert parsed == sample_schema
+
+    def test_empty(self):
+        assert parse_schema_json("{}") == {}
+
+
+class TestParseSchemaCsv:
+    def test_round_trip(self, sample_schema):
+        content = format_schema_csv(sample_schema)
+        parsed = parse_schema_csv(content)
+        assert parsed == sample_schema
+
+    def test_empty(self):
+        parsed = parse_schema_csv("table,column,type\n")
+        assert parsed == {}
+
+
+class TestParseSchemaText:
+    def test_round_trip(self, sample_schema):
+        content = format_schema_text(sample_schema)
+        parsed = parse_schema_text(content)
+        assert parsed == sample_schema
+
+    def test_empty(self):
+        assert parse_schema_text("") == {}
+
+    def test_single_table(self):
+        content = "users\n  id\n  name\n"
+        parsed = parse_schema_text(content)
+        assert parsed == {"users": {"id": "UNKNOWN", "name": "UNKNOWN"}}
+
+
+class TestLoadSchemaFile:
+    def test_json_extension(self, tmp_path, sample_schema):
+        f = tmp_path / "schema.json"
+        f.write_text(format_schema_json(sample_schema))
+        assert load_schema_file(f) == sample_schema
+
+    def test_csv_extension(self, tmp_path, sample_schema):
+        f = tmp_path / "schema.csv"
+        f.write_text(format_schema_csv(sample_schema))
+        assert load_schema_file(f) == sample_schema
+
+    def test_txt_extension(self, tmp_path, sample_schema):
+        f = tmp_path / "schema.txt"
+        f.write_text(format_schema_text(sample_schema))
+        assert load_schema_file(f) == sample_schema
+
+    def test_no_extension_treated_as_text(self, tmp_path, sample_schema):
+        f = tmp_path / "schema"
+        f.write_text(format_schema_text(sample_schema))
+        assert load_schema_file(f) == sample_schema

{sql_glider-0.1.14 → sql_glider-0.1.16}/tests/sqlglider/lineage/test_analyzer.py

@@ -3,7 +3,11 @@
 import pytest
 
 from sqlglider.global_models import AnalysisLevel
-from sqlglider.lineage.analyzer import LineageAnalyzer, StarResolutionError
+from sqlglider.lineage.analyzer import (
+    LineageAnalyzer,
+    StarResolutionError,
+    _flat_schema_to_nested,
+)
 
 
 class TestCaseInsensitiveForwardLineage:
@@ -3181,3 +3185,97 @@ class TestSchemaPruning:
         output_names = {item.output_name for r in results for item in r.lineage_items}
         assert "id" in output_names
         assert "email" in output_names
+
+
+class TestFlatSchemaToNested:
+    """Tests for _flat_schema_to_nested conversion utility."""
+
+    def test_empty(self):
+        assert _flat_schema_to_nested({}) == {}
+
+    def test_unqualified_passthrough(self):
+        schema = {"users": {"id": "UNKNOWN"}}
+        assert _flat_schema_to_nested(schema) == schema
+
+    def test_two_part_keys(self):
+        schema = {"db.users": {"id": "UNKNOWN"}}
+        result = _flat_schema_to_nested(schema)
+        assert result == {"db": {"users": {"id": "UNKNOWN"}}}
+
+    def test_three_part_keys(self):
+        schema = {"cat.db.users": {"id": "UNKNOWN"}}
+        result = _flat_schema_to_nested(schema)
+        assert result == {"cat": {"db": {"users": {"id": "UNKNOWN"}}}}
+
+    def test_mixed_depth_pads_shorter_keys(self):
+        schema = {
+            "my_view": {"x": "UNKNOWN"},
+            "db.users": {"id": "UNKNOWN"},
+        }
+        result = _flat_schema_to_nested(schema)
+        assert result == {
+            "": {"my_view": {"x": "UNKNOWN"}},
+            "db": {"users": {"id": "UNKNOWN"}},
+        }
+
+
+class TestQualifiedSchemaKeys:
+    """Tests for schema with qualified (dotted) table names."""
+
+    def test_qualified_star_expansion(self):
+        """SELECT * resolves correctly with qualified schema keys."""
+        sql = "SELECT * FROM mydb.users"
+        schema = {"mydb.users": {"id": "UNKNOWN", "name": "UNKNOWN"}}
+        analyzer = LineageAnalyzer(sql, dialect="spark", schema=schema)
+        results = analyzer.analyze_queries(level=AnalysisLevel.COLUMN)
+        items = {
+            (item.source_name, item.output_name)
+            for r in results
+            for item in r.lineage_items
+        }
+        assert ("mydb.users.id", "id") in items
+        assert ("mydb.users.name", "name") in items
+
+    def test_qualified_explicit_columns(self):
+        """Explicit columns trace sources correctly with qualified schema keys."""
+        sql = "SELECT id, name FROM mydb.users"
+        schema = {"mydb.users": {"id": "UNKNOWN", "name": "UNKNOWN"}}
+        analyzer = LineageAnalyzer(sql, dialect="spark", schema=schema)
+        results = analyzer.analyze_queries(level=AnalysisLevel.COLUMN)
+        items = {
+            (item.source_name, item.output_name)
+            for r in results
+            for item in r.lineage_items
+        }
+        assert ("mydb.users.id", "mydb.users.id") in items
+        assert ("mydb.users.name", "mydb.users.name") in items
+
+    def test_three_part_qualified(self):
+        """3-part qualified names (catalog.db.table) work correctly."""
+        sql = "SELECT id FROM catalog.mydb.users"
+        schema = {"catalog.mydb.users": {"id": "UNKNOWN"}}
+        analyzer = LineageAnalyzer(sql, dialect="spark", schema=schema)
+        results = analyzer.analyze_queries(level=AnalysisLevel.COLUMN)
+        items = [
+            (item.source_name, item.output_name)
+            for r in results
+            for item in r.lineage_items
+        ]
+        assert len(items) == 1
+        assert items[0] == ("catalog.mydb.users.id", "catalog.mydb.users.id")
+
+    def test_mixed_qualified_and_unqualified(self):
+        """Mix of qualified and unqualified table names in schema."""
+        sql = "SELECT * FROM my_view"
+        schema = {
+            "my_view": {"id": "UNKNOWN"},
+            "mydb.users": {"id": "UNKNOWN", "name": "UNKNOWN"},
+        }
+        analyzer = LineageAnalyzer(sql, dialect="spark", schema=schema)
+        results = analyzer.analyze_queries(level=AnalysisLevel.COLUMN)
+        items = {
+            (item.source_name, item.output_name)
+            for r in results
+            for item in r.lineage_items
+        }
+        assert ("my_view.id", "id") in items

{sql_glider-0.1.14 → sql_glider-0.1.16}/tests/sqlglider/test_cli.py

@@ -1884,3 +1884,141 @@ class TestTablesScrapeCommand:
 
         data = json.loads(result.stdout)
         assert "customers" in data
+
+
+class TestProvideSchema:
+    """Tests for --provide-schema on lineage and graph build commands."""
+
+    @pytest.fixture
+    def star_query_file(self, tmp_path):
+        """SQL file with SELECT * that needs schema to resolve."""
+        sql_file = tmp_path / "star.sql"
+        sql_file.write_text("SELECT * FROM users")
+        return sql_file
+
+    @pytest.fixture
+    def schema_json_file(self, tmp_path):
+        schema = tmp_path / "schema.json"
+        schema.write_text('{"users": {"id": "UNKNOWN", "name": "UNKNOWN"}}')
+        return schema
+
+    def test_lineage_with_provide_schema(self, star_query_file, schema_json_file):
+        """Test that --provide-schema resolves SELECT * in lineage."""
+        result = runner.invoke(
+            app,
+            [
+                "lineage",
+                str(star_query_file),
+                "--provide-schema",
+                str(schema_json_file),
+                "--output-format",
+                "json",
+            ],
+        )
+
+        assert result.exit_code == 0
+        import json
+
+        data = json.loads(result.stdout)
+        columns = [item["output_name"] for item in data["queries"][0]["lineage"]]
+        assert "id" in columns
+        assert "name" in columns
+
+    def test_graph_build_with_provide_schema(
+        self, star_query_file, schema_json_file, tmp_path
+    ):
+        """Test that --provide-schema works with graph build."""
+        output = tmp_path / "graph.json"
+        result = runner.invoke(
+            app,
+            [
+                "graph",
+                "build",
+                str(star_query_file),
+                "-o",
+                str(output),
+                "--provide-schema",
+                str(schema_json_file),
+            ],
+        )
+
+        assert result.exit_code == 0
+        assert output.exists()
+        import json
+
+        graph = json.loads(output.read_text())
+        assert graph["metadata"]["total_nodes"] > 0
+
+
+class TestProvideSchemaRoundTrip:
+    """Integration: tables scrape -> schema file -> graph build --provide-schema."""
+
+    @pytest.fixture
+    def sql_dir(self, tmp_path):
+        d = tmp_path / "sql"
+        d.mkdir()
+        (d / "a.sql").write_text(
+            "CREATE TABLE output_table AS SELECT c.id, c.name FROM customers c;"
+        )
+        (d / "b.sql").write_text("SELECT * FROM output_table")
+        return d
+
+    @pytest.mark.parametrize(
+        "fmt,ext", [("json", ".json"), ("csv", ".csv"), ("text", ".txt")]
+    )
+    def test_round_trip(self, sql_dir, tmp_path, fmt, ext):
+        """Scrape schema, save to file, then use --provide-schema to build graph."""
+        schema_file = tmp_path / f"schema{ext}"
+        graph_provided = tmp_path / "graph_provided.json"
+        graph_resolved = tmp_path / "graph_resolved.json"
+
+        # Step 1: Scrape schema
+        scrape_result = runner.invoke(
+            app,
+            ["tables", "scrape", str(sql_dir), "-f", fmt, "-o", str(schema_file)],
+        )
+        assert scrape_result.exit_code == 0
+        assert schema_file.exists()
+
+        # Step 2: Build graph with --provide-schema
+        result_provided = runner.invoke(
+            app,
+            [
+                "graph",
+                "build",
+                str(sql_dir),
+                "-o",
+                str(graph_provided),
+                "--provide-schema",
+                str(schema_file),
+            ],
+        )
+        assert result_provided.exit_code == 0
+
+        # Step 3: Build graph with --resolve-schema
+        result_resolved = runner.invoke(
+            app,
+            [
+                "graph",
+                "build",
+                str(sql_dir),
+                "-o",
+                str(graph_resolved),
+                "--resolve-schema",
+            ],
+        )
+        assert result_resolved.exit_code == 0
+
+        # Step 4: Compare graphs (nodes and edges should match)
+        import json
+
+        g1 = json.loads(graph_provided.read_text())
+        g2 = json.loads(graph_resolved.read_text())
+
+        nodes1 = sorted([n["identifier"] for n in g1["nodes"]])
+        nodes2 = sorted([n["identifier"] for n in g2["nodes"]])
+        assert nodes1 == nodes2
+
+        edges1 = sorted([(e["source_node"], e["target_node"]) for e in g1["edges"]])
+        edges2 = sorted([(e["source_node"], e["target_node"]) for e in g2["edges"]])
+        assert edges1 == edges2