sql-code-graph 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

{sql_code_graph-1.0.0.dist-info → sql_code_graph-1.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sql-code-graph
-Version: 1.0.0
+Version: 1.0.1
 Summary: SQL code graph analyzer and lineage tracer
 Project-URL: Homepage, https://github.com/Warhorze/sql-code-graph
 Project-URL: Repository, https://github.com/Warhorze/sql-code-graph

{sql_code_graph-1.0.0.dist-info → sql_code_graph-1.0.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-sqlcg/__init__.py,sha256=2lT2oiKX19arg1oTOFf13dXA3qyyQNpRevdvKHZIOp4,115
+sqlcg/__init__.py,sha256=rhzbVCLAOlBWTlliY-J8bh3jG3Hn7-5PPLDJwujXW_g,115
 sqlcg/__main__.py,sha256=1YoFLcqEgTwYq1J3TbUwpkdG0zeeLIf2fJvwWI-CLFU,109
 sqlcg/cli/__init__.py,sha256=W8fD0LpMq2xm_5WKGNMvJh2WBL1ho5E8hUeAqXQYT1g,28
 sqlcg/cli/main.py,sha256=WmdTjsOlz1ozi2Y3Aq4ezR_FCRl-Lc1YOKw3_d48dlY,1650
@@ -8,7 +8,7 @@ sqlcg/cli/commands/db.py,sha256=Yd4ZDz1BFwjO4Lyt3NefQnowkjdUxFDFmsPykBVH2Pk,6518
 sqlcg/cli/commands/find.py,sha256=4cEWQ0otxNIzzwwzZ0WB_Tms0EoKzcFfhB3FJt8Q5V4,2025
 sqlcg/cli/commands/gain.py,sha256=bOvia7CVla_fESrDEdftYze8Mm0xDio3SpCzIyoXg7A,8925
 sqlcg/cli/commands/git.py,sha256=96hmWYd861FC8RZqPQ_eBG8yLXSXaB9SLxmuwx00nWU,3347
-sqlcg/cli/commands/index.py,sha256=6f-kaoY5roY4DDvEOi_HrDnBG9Jrqy0_A47gsxZsNUQ,7421
+sqlcg/cli/commands/index.py,sha256=b6ns4_1ZVnHE5GeIb2N8YogjUvhjkzn_F9HrwCqrt_Y,8253
 sqlcg/cli/commands/install.py,sha256=mNVXdGlQ4JtCaaibuzU-inf519T97mC-Nj9K-G2gMQY,7525
 sqlcg/cli/commands/mcp.py,sha256=H1j6b5Tqr5VXja2GafgD5sJD6hZ5rsgfPwIikK1PZqc,1903
 sqlcg/cli/commands/reindex.py,sha256=iZXxYGI2m2wxkvIA1mB9uvOEp66QaT5zF5TGd0OpqlU,6275
@@ -29,8 +29,8 @@ sqlcg/indexer/__init__.py,sha256=Wh20Unz2OHs1oIyWLrpurPAasF0BET2g4iXtNk7mh2U,56
 sqlcg/indexer/dbt_adapter.py,sha256=EB5x1WU5Z9d-I97ADDj88S_hG1C4z4nbrv8JUCzXfy8,686
 sqlcg/indexer/error_classify.py,sha256=eWmc9WdOFe9kY_DMgKL0vv9gfcKnFw8e8U7cpUUw9wU,5139
 sqlcg/indexer/git_delta.py,sha256=V7WiNgiYPRo97K_mB3ymkJDZGoFExqwTZ2ut0Nqua5o,4383
-sqlcg/indexer/indexer.py,sha256=Jes0SybIDXLWQlWbRrDAbxVfJ7OsdS3PDAVSoRcv3Tc,50605
-sqlcg/indexer/pool.py,sha256=Q9DQmgUsSeKL1S-gNAzMbCNPGI9WsG6Nmt_noh_O8M8,16069
+sqlcg/indexer/indexer.py,sha256=0B0BCUaLPdV9XtlCzhqR3hwHyD3w83o-tYG7yNr18Yo,50507
+sqlcg/indexer/pool.py,sha256=n8u_z2IjW-rX1m0wlJ9-N-jxQby_Y4J9blMEPYaf19Q,18360
 sqlcg/indexer/walker.py,sha256=C__JuDcTzKxFqVjGFRr5cj9hgxvf8zffTz-0HMn1qTY,1746
 sqlcg/indexer/watcher.py,sha256=mJQq1LASRLKKwhz0WhCUWPLLqyPR2_-FD_8efYU6gE8,8442
 sqlcg/lineage/__init__.py,sha256=Da1DlYwtK13WHv_RnHjAtNkHTOuFbhxqCjT1Le7DsWM,46
@@ -40,7 +40,7 @@ sqlcg/metrics/__init__.py,sha256=hLJ6wm4St8qqYwKh3o9QG7lcEt1BEYM31ccqO9tGpIg,133
 sqlcg/metrics/store.py,sha256=BaMf7QYTmYMlX_Jzi1GNU8R2sMVkWdn07f-ZSndtcNk,8879
 sqlcg/parsers/__init__.py,sha256=AamA8wBbDZV9_zEtZCI4Hyen5UAVKHmBwjTghTt2PZE,785
 sqlcg/parsers/ansi_parser.py,sha256=KruZn5CYjpktKmMRVWackshRI_AR6ehc-ReCsDeWNkQ,14321
-sqlcg/parsers/base.py,sha256=aw-gueAMdt551peUY0g7lWbswQLPWx0FDCK4RDfUjDE,43205
+sqlcg/parsers/base.py,sha256=nkhl2jVBFRPKHtr2PKfYy6vTdW64v7KKUnfMwVG2ZMU,43941
 sqlcg/parsers/bigquery_parser.py,sha256=mOnWTfXB_Dp4JwFE1PVYOB6CDPf5nYE0Dea8kJCl9uQ,2827
 sqlcg/parsers/postgres_parser.py,sha256=lYfUpQY6j4Qm7ndXBtXbgPoGzYqYddWt5YeFnWKdA6I,946
 sqlcg/parsers/registry.py,sha256=LXy1F6rqQI6VdxpRvZg_tNpoEucW3mXZHYBMlMONbX4,1496
@@ -57,7 +57,7 @@ sqlcg/utils/__init__.py,sha256=--iqt5ThTXmT8Wz7da8hs3n0zDfYPl8P-z5OgRJ_77E,154
 sqlcg/utils/hashing.py,sha256=H25-sYfxHKb3_IERFnHyAIYNiXN470Oqo5sJT_D3YOA,438
 sqlcg/utils/ignore.py,sha256=NfInsHPGubfKFJQraH-wE7ATPb5Be_Igu5mIh7p21cU,973
 sqlcg/utils/logging.py,sha256=u0fCmYsLj9o81vawm3xZTHaw68GQYVm7JxG-gP81u8A,840
-sql_code_graph-1.0.0.dist-info/METADATA,sha256=HQdFHBzEKTlPlqnwRCT9n0iKrmWqkmM5mhM3fOi5lvo,12806
-sql_code_graph-1.0.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
-sql_code_graph-1.0.0.dist-info/entry_points.txt,sha256=Wfe49sVzV9p4eVFGo5RxcV-frr3HOP0yzzst8JBxQLQ,46
-sql_code_graph-1.0.0.dist-info/RECORD,,
+sql_code_graph-1.0.1.dist-info/METADATA,sha256=vFhNG1uWAym_RQ21vDWG0tlogTOe2DDjCmrJp8X1txg,12806
+sql_code_graph-1.0.1.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+sql_code_graph-1.0.1.dist-info/entry_points.txt,sha256=Wfe49sVzV9p4eVFGo5RxcV-frr3HOP0yzzst8JBxQLQ,46
+sql_code_graph-1.0.1.dist-info/RECORD,,

sqlcg/__init__.py

@@ -1,5 +1,5 @@
 """SQL Code Graph - SQL lineage and dependency analysis tool."""
 
-__version__ = "1.0.0"
+__version__ = "1.0.1"
 
 __all__ = ["__version__"]

sqlcg/cli/commands/index.py

@@ -84,6 +84,35 @@ def index_cmd(  # noqa: B008
     db_path = get_db_path()
     db_path.parent.mkdir(parents=True, exist_ok=True)
 
+    try:
+        _run_index(
+            path=path,
+            dialect=dialect,
+            dbt_manifest=dbt_manifest,
+            timeout_per_file=timeout_per_file,
+            no_ddl=no_ddl,
+            quiet=quiet,
+            batch_size=batch_size,
+            profile=profile,
+        )
+    except KeyboardInterrupt:
+        # The backend context manager (inside _run_index) has already closed the
+        # KuzuDB connection and released the lock by the time we get here.
+        console.print("\n[yellow]Interrupted — no partial graph written. Re-run to index.[/yellow]")
+        raise typer.Exit(130) from None
+
+
+def _run_index(
+    *,
+    path: Path,
+    dialect: str | None,
+    dbt_manifest: Path | None,
+    timeout_per_file: int,
+    no_ddl: bool,
+    quiet: bool,
+    batch_size: int,
+    profile: bool,
+) -> None:
     with get_backend() as backend:
         backend.init_schema()
 

sqlcg/indexer/indexer.py

@@ -251,9 +251,11 @@ class Indexer:
                     _t_pass2_end = time.perf_counter()
 
         except KeyboardInterrupt:
-            logger.info("SIGINT received — flushing pass-1 progress")
-            # pass1_results may be partial; upsert what we have
-            self._upsert_all(pass1_results, db)
+            # Kill workers and abort immediately. A partial pass-1-only result is
+            # an incomplete graph (no cross-file resolution, no star expansion);
+            # writing it would leave a misleading half-index. Re-run `sqlcg index`
+            # to index — re-indexing is the migration path.
+            logger.warning("Interrupted — workers killed; no partial graph written.")
             raise
 
         # Assemble final pass-2 results: start from pass-1, overlay pass-2 where available
@@ -1104,16 +1106,6 @@ class Indexer:
 
         return counts
 
-    def _upsert_all(self, results: list[ParsedFile], db: GraphBackend) -> None:
-        """Upsert all parsed files.
-
-        Args:
-            results: List of ParsedFile objects
-            db: GraphBackend instance
-        """
-        for parsed in results:
-            self._upsert_parsed_file(parsed, db)
-
     def _expand_star_sources(self, db: GraphBackend) -> int:
         """Run the post-ingestion star expansion query.
 

sqlcg/indexer/pool.py

@@ -194,7 +194,14 @@ class HardKillPool:
     ) -> None:
         self._dialect = dialect
         self._schema_aliases: dict[str, str] = schema_aliases or {}
-        self._n = n_workers or os.cpu_count() or 4
+        # Leave 2 logical cores of headroom rather than spawning one worker per
+        # logical core. Parsing is CPU-bound, and the main process also does work
+        # between passes (closure resolution, batched upserts); saturating every
+        # core makes the largest files miss the per-file wall-clock timeout.
+        # Measured on the 1,453-file DWH corpus (after the once-per-statement parser
+        # fixes): cpu_count → 2 timeouts / 186s; cpu_count-2 → 0 timeouts / 131s
+        # (fewer timeouts AND faster, since timed-out files waste work + respawn churn).
+        self._n = n_workers or max(1, (os.cpu_count() or 4) - 2)
         self._ctx = mp.get_context("spawn")
         self._workers: list[_WorkerState] = []
 
@@ -322,6 +329,28 @@ class HardKillPool:
             w.task_start = time.monotonic()
             busy.add(slot)
 
+        try:
+            return self._run_map_loop(
+                tasks, results, busy, kill_counts, _assign, per_task_timeout, on_result, n_tasks
+            )
+        except KeyboardInterrupt:
+            # Workers ignore SIGINT and are CPU-bound, so they will not notice a
+            # graceful SHUTDOWN sentinel until their current parse finishes. On
+            # interrupt the user wants the process gone now — hard-kill outright.
+            self.terminate()
+            raise
+
+    def _run_map_loop(
+        self,
+        tasks: list[dict],
+        results: list[ParsedFile | None],
+        busy: set[int],
+        kill_counts: dict[str, int],
+        _assign: Callable[[int], None],
+        per_task_timeout: float,
+        on_result: Callable[[], None] | None,
+        n_tasks: int,
+    ) -> list[ParsedFile | None]:
         # Initial dispatch: fill all worker slots
         for i in range(min(self._n, n_tasks)):
             _assign(i)
@@ -405,6 +434,31 @@ class HardKillPool:
     # Shutdown
     # ------------------------------------------------------------------
 
+    def terminate(self) -> None:
+        """Immediately SIGKILL every worker without a graceful handshake.
+
+        Unlike :meth:`shutdown`, this sends no ``_SHUTDOWN`` sentinel and does
+        not wait for in-flight parses. Workers ignore SIGINT and are CPU-bound,
+        so a graceful stop would block on the longest running parse; on
+        interrupt we kill outright so the process dies promptly.
+        """
+        for w in self._workers:
+            try:
+                w.conn.close()
+            except Exception:
+                pass
+            try:
+                if w.process.is_alive():
+                    w.process.kill()
+            except Exception:
+                pass
+        for w in self._workers:
+            try:
+                w.process.join(timeout=1)
+            except Exception:
+                pass
+        self._workers.clear()
+
     def shutdown(self) -> None:
         """Gracefully stop all workers, then force-kill any that linger."""
         for w in self._workers:

sqlcg/parsers/base.py

@@ -619,10 +619,6 @@ class SqlParser(ABC):
             else:
                 return LineageExtraction(edges=edges, star_sources=star_sources)
 
-            # Build scope once from the body for all-column reuse (T-05 optimization)
-            # Defer scope building to just before the column loop to ensure sources
-            # are expanded first (avoid rebuilding for each column, but only build
-            # after sources are known)
             body_scope = None
             combined_sources = {**(sources or {})}
 
@@ -644,6 +640,54 @@ class SqlParser(ABC):
                                 key = cte_alias.lower()
                                 combined_sources[key] = cte.this
 
+            # Build body_scope ONCE per statement, before the column loop, and reuse
+            # it for every column (CLAUDE.md invariant: "body_scope built once per
+            # statement"). If schema-qualify fails, retry schema-free so we STILL get
+            # a scope for the copy=False fast path; only if both fail do we fall back
+            # to the per-column sources= path. Building this lazily inside the loop
+            # (regressed in 4234e5d) meant a single qualify failure re-ran
+            # expand+qualify+build_scope for EVERY column → O(N_cols) full-body
+            # deepcopies per statement (measured: 229 qualify calls on one 460-line file).
+            if scope is None:
+                expanded_body = body
+                expand_sources = {
+                    k: v for k, v in (sources or {}).items() if isinstance(v, exp.Query)
+                }
+                if expand_sources:
+                    try:
+                        expanded_body = exp.expand(
+                            body,
+                            expand_sources,  # type: ignore
+                            dialect=self.DIALECT,
+                            copy=True,
+                        )
+                    except Exception:
+                        expanded_body = body
+                try:
+                    qualified_body = qualify(
+                        expanded_body,
+                        dialect=self.DIALECT,
+                        schema=schema,
+                        validate_qualify_columns=False,
+                        identify=False,
+                    )
+                    body_scope = build_scope(qualified_body)
+                except Exception as _qualify_exc:
+                    out.errors.append(
+                        f"col_lineage_skip:qualify_failed:{type(_qualify_exc).__name__}"
+                    )
+                    # Schema-free retry: still yields a scope for the copy=False path.
+                    try:
+                        qualified_body = qualify(
+                            expanded_body,
+                            dialect=self.DIALECT,
+                            validate_qualify_columns=False,
+                            identify=False,
+                        )
+                        body_scope = build_scope(qualified_body)
+                    except Exception:
+                        body_scope = None
+
             # Extract output columns
             for col_expr in col_expressions:
                 # Skip star projections — sg_lineage requires a concrete column name.
@@ -723,53 +767,23 @@ class SqlParser(ABC):
                         continue
 
                 try:
-                    # Build scope on first column for reuse across all columns (T-05 optimization).
-                    # NOTE: We build body_scope locally from the extracted body rather than
-                    # using a pre-built scope from the statement, because CREATE/INSERT statements
-                    # have their scope rooted at the outer statement, but the body passed here
-                    # is the inner SELECT. Reusing the outer scope would produce incorrect
-                    # qualification. The pre-built scope from parse_file would only be useful
-                    # if we had a mechanism to extract the matching inner scope, which is
-                    # complex and not yet implemented (see sprint_06 T-05 deviation for details).
-                    if body_scope is None and scope is None:
-                        try:
-                            # Expand only file-level sources (CTEs, temp tables, CTAS bodies).
-                            expanded_body = body
-                            expand_sources = {
-                                k: v for k, v in (sources or {}).items() if isinstance(v, exp.Query)
-                            }
-                            if expand_sources:
-                                expanded_body = exp.expand(
-                                    body,
-                                    expand_sources,  # type: ignore
-                                    dialect=self.DIALECT,
-                                    copy=True,
-                                )
-
-                            # Qualify the expanded body to prepare for scope building
-                            qualified_body = qualify(
-                                expanded_body,
-                                dialect=self.DIALECT,
-                                schema=schema,
-                                validate_qualify_columns=False,
-                                identify=False,
-                            )
-                            body_scope = build_scope(qualified_body)
-                        except Exception as _qualify_exc:
-                            # qualify() failure is non-fatal: sg_lineage falls back to
-                            # its own qualification. Record for observability.
-                            out.errors.append(
-                                f"col_lineage_skip:qualify_failed:{type(_qualify_exc).__name__}:{_qualify_exc}"
-                            )
-                            body_scope = None
-
                     # When a scope is available it embeds full column→table resolution.
                     # On the qualify-failed fallback path (no scope), pass only the small
                     # set of file-level sources so sg_lineage can resolve CTEs/CTAS bodies.
                     active_scope = scope if scope is not None else body_scope
                     sg_kwargs: dict = {"dialect": self.DIALECT}
                     if active_scope is not None:
+                        # scope= path: the pre-built scope already embeds full
+                        # column→table resolution. copy=False + trim_selects=False
+                        # suppress sqlglot's per-call AST deepcopy and per-column
+                        # trim — neither is needed when the scope is built once and
+                        # reused across all columns. Dropping these (regressed in
+                        # 4234e5d) makes lineage() deepcopy the whole scope per
+                        # column → O(columns × scope_size) (measured: 3.2M deepcopy
+                        # calls / ~3.8s on a 359-line file).
                         sg_kwargs["scope"] = active_scope
+                        sg_kwargs["copy"] = False
+                        sg_kwargs["trim_selects"] = False
                     else:
                         sg_kwargs["sources"] = sources or {}
                     root = sg_lineage(col_name, body, **sg_kwargs)
@@ -912,6 +926,13 @@ class SqlParser(ABC):
             # stops sg_lineage at the CTE name boundary (doesn't expand into bodies).
             if isinstance(stmt, exp.Insert) and isinstance(stmt.this, exp.Schema):
                 insert_cols = [c.name for c in stmt.this.expressions]
+                # Build the WITH-stripped body ONCE before the loop and only swap its
+                # single projection per column (regressed in 4234e5d, which moved the
+                # full-body body.copy() inside the loop → O(N_cols) full-body deepcopies
+                # for wide INSERT ... SELECT). Stripping WITH stops sg_lineage at the CTE
+                # name boundary.
+                body_no_with = body.copy()
+                body_no_with.set("with_", None)
                 for idx, col_expr in enumerate(col_expressions):
                     if idx >= len(insert_cols):
                         break
@@ -920,10 +941,8 @@ class SqlParser(ABC):
                     insert_col = insert_cols[idx]
                     if not insert_col:
                         continue
-                    # Build a patched SELECT: strip WITH, alias the expression with the
-                    # INSERT column name so sg_lineage can trace it.
-                    body_no_with = body.copy()
-                    body_no_with.set("with_", None)
+                    # Patch the shared body with this column's aliased expression so
+                    # sg_lineage can trace it to the INSERT column name.
                     aliased = exp.Alias(this=col_expr.copy(), alias=insert_col)
                     body_no_with.set("expressions", [aliased])
                     patched_sql = body_no_with.sql(dialect=self.DIALECT)