interlaced 2.0.2__tar.gz → 2.0.3__tar.gz

{interlaced-2.0.2/src/interlaced.egg-info → interlaced-2.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: interlaced
-Version: 2.0.2
+Version: 2.0.3
 Summary: Python/SQL-first data platform: transformation, built-in orchestration, and durable streaming ingestion
 Author-email: Mark <mark@interlace.sh>
 License-Expression: MIT

{interlaced-2.0.2 → interlaced-2.0.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "interlaced"
-version = "2.0.2"
+version = "2.0.3"
 description = "Python/SQL-first data platform: transformation, built-in orchestration, and durable streaming ingestion"
 readme = "README.md"
 requires-python = ">=3.12"

{interlaced-2.0.2 → interlaced-2.0.3}/src/interlace/cli/main.py

@@ -655,23 +655,28 @@ def serve(
         token = secrets.token_hex(8)
         console.print(f"[bold]quack[/bold] warehouse at [cyan]{quack}[/cyan] · token [yellow]{token}[/yellow]")
         console.print("Clients: set [bold]database: quack:...[/bold] and INTERLACE_QUACK_TOKEN in the environment.")
-    uvicorn.run(
-        create_app(
-            path,
-            environment,
-            quack=quack or None,
-            quack_token=token or None,
-            scheduler=scheduler,
-            scheduler_interval=interval,
-        ),
+    app = create_app(
+        path,
+        environment,
+        quack=quack or None,
+        quack_token=token or None,
+        scheduler=scheduler,
+        scheduler_interval=interval,
+    )
+    config = uvicorn.Config(
+        app,
         host=host,
         port=port,
-        # SSE clients (/events/stream) hold their response open forever; without a
-        # bound, uvicorn's graceful shutdown waits on them indefinitely ("Waiting
-        # for connections to close") and Ctrl+C appears to hang. Lifespan cleanup
-        # (flush, store/engine close) runs only after this drain completes.
+        # SSE clients (/events/stream) hold their response open forever. The app's
+        # shutdown watcher ends them the instant this server flips `should_exit`, so
+        # the graceful-shutdown drain finds the connections already closed instead of
+        # force-cancelling them (which used to dump a CancelledError traceback on
+        # Ctrl+C). The bound is a backstop; lifespan cleanup runs after the drain.
         timeout_graceful_shutdown=3,
     )
+    server = uvicorn.Server(config)
+    app.state.uvicorn_server = server  # let the app release SSE streams as shutdown begins
+    server.run()
 
 
 @app.command("models")

{interlaced-2.0.2 → interlaced-2.0.3}/src/interlace/engines/base.py

@@ -85,9 +85,13 @@ class EngineAdapter(ABC):
     async def fetch_sandboxed(self, ast: exp.Expression) -> pa.RecordBatchReader:
         """Like :meth:`fetch`, but for untrusted queries (the HTTP query console):
         the engine must not touch anything outside the warehouse — no local files,
-        no network. The default cannot enforce that, so it is the caller's job to
-        gate this path; adapters over an embedded engine that CAN read the host
-        filesystem (DuckDB) MUST override to lock the connection down."""
+        no network. The engine cannot always enforce that itself (DuckDB's
+        ``enable_external_access`` is instance-wide and one-way, so disabling it on
+        the shared warehouse connection would permanently break the writer's own
+        file writes), so **the caller MUST gate this path** — the service validates
+        the SELECT's AST (real tables only, no table functions, no file paths) before
+        calling here. An engine with a genuinely isolated read-only mode may override
+        to add defence in depth."""
         return await self.fetch(ast)
 
     def transpile(self, ast: exp.Expression) -> str:

{interlaced-2.0.2 → interlaced-2.0.3}/src/interlace/engines/duckdb.py

@@ -203,8 +203,11 @@ class DuckDBAdapter(EngineAdapter):
     async def fetch_sql(self, sql: str) -> pa.RecordBatchReader:
         return await asyncio.to_thread(self._fetch_sync, sql)
 
-    async def fetch_sandboxed(self, ast: exp.Expression) -> pa.RecordBatchReader:
-        return await asyncio.to_thread(self._fetch_sync, self.transpile(ast), sandboxed=True)
+    # No fetch_sandboxed override: DuckDB cannot sandbox one query on the shared
+    # warehouse connection without poisoning the writer (enable_external_access is
+    # instance-wide and one-way, and a DuckLake catalog can be held by only one
+    # connection per process). The untrusted /query path is fenced at parse time by
+    # the service's _guard_console_query instead — as the base contract expects.
 
     async def create_schema(self, name: str) -> None:
         await self.execute_sql(f"CREATE SCHEMA IF NOT EXISTS {exp.to_identifier(name).sql(dialect=self.dialect)}")
@@ -244,19 +247,12 @@ class DuckDBAdapter(EngineAdapter):
                 cur.close()
         return counts
 
-    def _fetch_sync(self, sql: str, *, sandboxed: bool = False) -> pa.RecordBatchReader:
+    def _fetch_sync(self, sql: str) -> pa.RecordBatchReader:
         # Read-only: deliberately not locked, so scans run concurrently (DuckDB MVCC).
         # The cursor must outlive the stream that reads from it, so it is closed when
         # that stream ends rather than left for the garbage collector to reclaim on
         # whatever thread happens to drop the last reference.
         cur = self._cursor()
-        if sandboxed:
-            # Untrusted query (the HTTP console). enable_external_access is
-            # connection-local, so this cursor can neither read host files nor
-            # open network connections — closing the query()/read_csv()/httpfs
-            # escape hatches at the engine level, not by name-matching. The build
-            # path (other cursors) keeps external access for exports and ATTACH.
-            cur.execute("SET enable_external_access = false")
         cur.execute(sql)
         reader = cur.to_arrow_reader()
 

{interlaced-2.0.2 → interlaced-2.0.3}/src/interlace/service/app.py

@@ -858,24 +858,58 @@ _EXTERNAL_FN = re.compile(
     r"|(_scan|_query)$",
     re.IGNORECASE,
 )
+# Table functions the console may use: pure in-memory row generators, no I/O.
+# `range(...)` normalises to generate_series in the sqlglot AST. Anything else in
+# table position (read_csv / read_parquet / query / query_table / glob / …) is
+# rejected, named or not — the allowlist is the robust part.
+_SAFE_TABLE_FUNCTIONS = frozenset({"generate_series"})
+
+
+def _console_fn_name(node: object) -> str | None:
+    from sqlglot import exp as _exp
+
+    if isinstance(node, _exp.Anonymous):
+        return str(node.this).lower()
+    if isinstance(node, _exp.Func):
+        return str(node.sql_name()).lower()
+    return None
+
+
 _QUERY_MAX_CELL_BYTES = 8_000_000  # ~8 MB of rendered cells; the console inspects, never extracts
 
 
 def _guard_console_query(parsed: object) -> None:
+    """Reject anything the read-only console must never run. This is the *fence*, not
+    merely defence in depth: an engine-level lockdown is not available to us. On a
+    DuckLake warehouse the console shares the writer's DuckDB connection (a process
+    can hold the catalog file only once), and ``enable_external_access`` is
+    instance-wide and one-way — flipping it off to sandbox a query would permanently
+    disable the writer's own file writes (the flusher, apply, exports).
+
+    So the SELECT is validated structurally. The key rule: every table source must be
+    a real table/view identifier, never a table function. ``read_csv`` / ``read_parquet``
+    / ``query`` / ``query_table`` / ``glob`` / … all parse as a *function* in table
+    position (``Table.this`` is a function node, not an ``Identifier``), so this
+    rejects every one of them — including any not yet named. A function-name denylist
+    and a file-path check backstop it for the rare scalar-position reader."""
     from sqlglot import exp as _exp
 
-    for node in parsed.walk():  # type: ignore[attr-defined]
-        name = None
-        if isinstance(node, _exp.Anonymous):
-            name = str(node.this)
-        elif isinstance(node, _exp.Func):
-            name = node.sql_name()
-        if name and _EXTERNAL_FN.search(name):
-            raise ClientException(detail=f"function {name!r} reads outside the warehouse — not in the console")
-        if isinstance(node, _exp.Table):
-            raw = node.name or ""
+    parsed_expr: _exp.Expression = parsed  # type: ignore[assignment]
+    for table in parsed_expr.find_all(_exp.Table):
+        if isinstance(table.this, _exp.Identifier):  # a real table/view reference
+            raw = table.name or ""
             if "/" in raw or "\\" in raw or "://" in raw:
                 raise ClientException(detail="file paths are not queryable from the console")
+            continue
+        name = _console_fn_name(table.this)  # a table function in FROM/JOIN position
+        if name not in _SAFE_TABLE_FUNCTIONS:
+            raise ClientException(
+                detail=f"table function {name or 'call'!r} is not queryable from the console — read tables only"
+            )
+    for node in parsed_expr.walk():  # backstop: file/network readers in scalar position
+        name = _console_fn_name(node)
+        if name and _EXTERNAL_FN.search(name):
+            raise ClientException(detail=f"function {name!r} reads outside the warehouse — not in the console")
 
 
 @post("/query", opt={"scope": "read"})
@@ -897,14 +931,14 @@ async def post_query(data: QueryRequest, state: State) -> QueryResponse:
     parsed = statements[0]
     if not isinstance(parsed, (_exp.Select, _exp.Union)):
         raise ClientException(detail="SELECT only — the console never writes")
-    _guard_console_query(parsed)  # defense in depth; the sandboxed cursor is the real fence
+    _guard_console_query(parsed)  # the fence: SELECT-only, tables (not table functions), no file paths
     limit = max(1, min(data.limit, 10_000))
     bounded = _exp.select("*").from_(parsed.subquery("q")).limit(limit + 1)  # +1: exact truncation
     started = _time.perf_counter()
 
     async def _run() -> pa.Table:
-        # fetch_sandboxed runs with external access disabled: query()/read_csv()/
-        # httpfs cannot reach host files or the network however they are spelled
+        # The query was already fenced by _guard_console_query (SELECT-only, no table
+        # functions, no file paths), so no host file or network read can be expressed.
         reader = await state.engine.fetch_sandboxed(bounded)
         return await asyncio.to_thread(reader.read_all)
 
@@ -1469,9 +1503,28 @@ def create_app(
                     logger.exception("scheduler tick failed; retrying next interval")
                 await asyncio.sleep(scheduler_interval)
 
+        async def shutdown_watch() -> None:
+            """End every open SSE stream the moment uvicorn begins shutting down.
+
+            SSE responses (/events/stream) block forever on their queue, so at Ctrl+C
+            uvicorn's graceful-shutdown drain waits the full timeout and then force-
+            cancels them — which surfaces as a CancelledError traceback from the held-
+            open stream. Poisoning the subscribers (their tail() returns on ``None``)
+            lets the drain find the connections already closed, so shutdown is clean
+            and immediate. ``uvicorn_server`` is injected by the serve command."""
+            server = getattr(app.state, "uvicorn_server", None)
+            if server is None:
+                return
+            while not getattr(server, "should_exit", False):
+                await asyncio.sleep(0.2)
+            for subscriber in list(app.state.sse_subscribers):
+                with contextlib.suppress(asyncio.QueueFull):
+                    subscriber.put_nowait(None)
+
         tail_task = asyncio.create_task(event_tail())
         flusher_task = asyncio.create_task(flusher_loop()) if streams else None
         loop_task = asyncio.create_task(scheduler_loop()) if scheduler else None
+        watch_task = asyncio.create_task(shutdown_watch())
         try:
             yield
         finally:
@@ -1481,7 +1534,12 @@ def create_app(
             # and the store/log/engine handles would leak — shield the whole thing.
             import anyio
 
-            for task in (loop_task, flusher_task, tail_task):
+            # Belt-and-braces: release any SSE stream still open (the watcher above
+            # normally does this the instant should_exit flips).
+            for subscriber in list(app.state.sse_subscribers):
+                with contextlib.suppress(asyncio.QueueFull):
+                    subscriber.put_nowait(None)
+            for task in (loop_task, flusher_task, tail_task, watch_task):
                 if task is not None:
                     task.cancel()
                     # suppress Exception too: a task that already died must not

{interlaced-2.0.2 → interlaced-2.0.3/src/interlaced.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: interlaced
-Version: 2.0.2
+Version: 2.0.3
 Summary: Python/SQL-first data platform: transformation, built-in orchestration, and durable streaming ingestion
 Author-email: Mark <mark@interlace.sh>
 License-Expression: MIT