duckstring 0.2.0__tar.gz → 0.3.0__tar.gz

duckstring-0.3.0/MANIFEST.in

@@ -0,0 +1,8 @@
+# The sdist ships the installable package only — keep internal design notes and the test suite
+# out of the PyPI source tarball (they're in the git repo for contributors).
+prune tests
+prune plans
+prune experiment
+prune docs
+prune frontend
+prune playground

{duckstring-0.2.0 → duckstring-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: duckstring
-Version: 0.2.0
+Version: 0.3.0
 Summary: Build data pipelines the way you build software: version each transform, declare its dependencies, and Duckstring resolves the execution DAG automatically.
 Author-email: Duckstring <dev@duckstring.com>
 License:                                  Apache License
@@ -221,6 +221,10 @@ Requires-Dist: uvicorn[standard]>=0.29
 Requires-Dist: httpx>=0.25
 Requires-Dist: tomli>=2.0; python_version < "3.11"
 Requires-Dist: tomli-w>=1.0
+Requires-Dist: pyiceberg>=0.7
+Requires-Dist: pyarrow>=14
+Requires-Dist: pytz
+Provides-Extra: iceberg
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0; extra == "dev"
 Requires-Dist: pytest-timeout>=0.5; extra == "dev"
@@ -244,6 +248,7 @@ The main elements:
 - **Catchment**: Control environment (FastAPI + UI + CLI)
 - **Pond**: Versioned container with declared upstream dependencies
 - **Ripple**: Unit operation within a Pond (e.g. a single transformation producing a table)
+- **Trickle**: DBSP-based incremental engine operating within a Ripple
 
 Ponds are typed or referred to in context:
 
@@ -292,10 +297,6 @@ Ponds execute on demand signals sent to an Outlet, in two flavours — **push**
 
 A Tide keeps an Outlet no staler than a bound (`duckstring trigger tide reports 1d`); a Wave keeps it as fresh as the pipeline can supply. See [Triggers](https://docs.duckstring.com/guides/triggers) for the full semantics.
 
-## Scope & maturity
-
-Duckstring targets single-node workloads (DuckDB under the hood) — pipelines in the tens of millions of rows, not big-data scale. Incremental transforms (the **Trickle** concept) are not yet implemented: transforms recompute their tables each run, with incremental state via self-reads.
-
 ## Going further
 
 Full documentation lives at **[docs.duckstring.com](https://docs.duckstring.com)**:

{duckstring-0.2.0 → duckstring-0.3.0}/README.md

@@ -14,6 +14,7 @@ The main elements:
 - **Catchment**: Control environment (FastAPI + UI + CLI)
 - **Pond**: Versioned container with declared upstream dependencies
 - **Ripple**: Unit operation within a Pond (e.g. a single transformation producing a table)
+- **Trickle**: DBSP-based incremental engine operating within a Ripple
 
 Ponds are typed or referred to in context:
 
@@ -62,10 +63,6 @@ Ponds execute on demand signals sent to an Outlet, in two flavours — **push**
 
 A Tide keeps an Outlet no staler than a bound (`duckstring trigger tide reports 1d`); a Wave keeps it as fresh as the pipeline can supply. See [Triggers](https://docs.duckstring.com/guides/triggers) for the full semantics.
 
-## Scope & maturity
-
-Duckstring targets single-node workloads (DuckDB under the hood) — pipelines in the tens of millions of rows, not big-data scale. Incremental transforms (the **Trickle** concept) are not yet implemented: transforms recompute their tables each run, with incremental state via self-reads.
-
 ## Going further
 
 Full documentation lives at **[docs.duckstring.com](https://docs.duckstring.com)**:

{duckstring-0.2.0 → duckstring-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "duckstring"
-version = "0.2.0"
+version = "0.3.0"
 description = "Build data pipelines the way you build software: version each transform, declare its dependencies, and Duckstring resolves the execution DAG automatically."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -19,9 +19,22 @@ dependencies = [
   "httpx>=0.25",
   "tomli>=2.0; python_version < '3.11'",
   "tomli-w>=1.0",
+  # The default data plane: an Apache Iceberg base layer (snapshots + schema metadata over the
+  # Parquet files we already write). Core deps because Iceberg is default-on; set
+  # DUCKSTRING_DATA_PLANE=parquet to opt out (the lighter fallback, also for offline Catchments that
+  # can't fetch DuckDB's iceberg extension). NB: we use our own file-backed catalog
+  # (duckstring.iceberg_catalog) rather than pyiceberg's SqlCatalog, so SQLAlchemy is NOT pulled in.
+  "pyiceberg>=0.7",
+  "pyarrow>=14",
+  # DuckDB requires pytz to convert TIMESTAMPTZ columns to Python datetimes on fetch
+  # (our _duckstring_f freshness column is TIMESTAMPTZ, read back in dataplane/trickle export).
+  # It is otherwise only present transitively (pandas), so a clean install lacks it.
+  "pytz",
 ]
 
 [project.optional-dependencies]
+# Retained for back-compat: the Iceberg deps are in core now (Iceberg is the default plane).
+iceberg = []
 dev = [
   "pytest>=8.0",
   "pytest-timeout>=0.5",

{duckstring-0.2.0 → duckstring-0.3.0}/src/duckstring/__init__.py

@@ -6,7 +6,6 @@ from .core import (
     Pond,
     Puddle,
     Ripple,
-    Trickle,
     puddle,
     ripple,
 )
@@ -21,7 +20,6 @@ __all__ = [
     "Pond",
     "Puddle",
     "Ripple",
-    "Trickle",
     "puddle",
     "ripple",
     "__version__",

duckstring-0.3.0/src/duckstring/acc.py

@@ -0,0 +1,13 @@
+"""Backwards-compatible alias — scan (order-dependent) metric specs live in :mod:`duckstring.trickle.acc`.
+
+Kept so ``from duckstring import acc`` works alongside ``from duckstring import agg``. See
+:mod:`duckstring.trickle_io` for the rationale.
+"""
+
+from __future__ import annotations
+
+from .trickle import acc as _acc
+
+
+def __getattr__(name):  # PEP 562: forward all names to the relocated module
+    return getattr(_acc, name)

duckstring-0.3.0/src/duckstring/agg.py

@@ -0,0 +1,12 @@
+"""Backwards-compatible alias — aggregate metric specs now live in :mod:`duckstring.trickle.agg`.
+
+Kept so ``from duckstring import agg`` continues to work. See :mod:`duckstring.trickle_io` for the rationale.
+"""
+
+from __future__ import annotations
+
+from .trickle import agg as _agg
+
+
+def __getattr__(name):  # PEP 562: forward all names to the relocated module
+    return getattr(_agg, name)

{duckstring-0.2.0 → duckstring-0.3.0}/src/duckstring/catchment/app.py

@@ -11,7 +11,7 @@ from fastapi import FastAPI
 from fastapi.responses import JSONResponse
 from fastapi.staticfiles import StaticFiles
 
-from .db import connect, migrate
+from .db import connect, ensure_identity, migrate
 from .driver import Driver
 from .launcher import NoopLauncher, SubprocessLauncher
 from .routes import router
@@ -49,22 +49,35 @@ async def _lifespan(app: FastAPI):
     # Restore: resume any Pond Runs that were in flight when the Catchment last stopped.
     driver.resume_incomplete()
 
+    from .poller import run_poller
+
+    # The poller wakes immediately when a Draw acquires demand (so it solicits its upstream at once),
+    # instead of waiting for its next cycle. The driver signals across threads via the running loop.
+    wake = asyncio.Event()
+    loop = asyncio.get_running_loop()
+    driver.set_notify(lambda: loop.call_soon_threadsafe(wake.set))
+
     scheduler = asyncio.create_task(_scheduler(driver))
+    poller = asyncio.create_task(run_poller(driver, app.state.root, wake))
     try:
         yield
     finally:
-        scheduler.cancel()
-        try:
-            await scheduler
-        except asyncio.CancelledError:
-            pass
+        for task in (scheduler, poller):
+            task.cancel()
+            try:
+                await task
+            except asyncio.CancelledError:
+                pass
         launcher.shutdown_all()
 
 
-def create_app(root: Path, api_key: str | None = None, base_url: str | None = None) -> FastAPI:
+def create_app(
+    root: Path, api_key: str | None = None, base_url: str | None = None, name: str | None = None
+) -> FastAPI:
     root.mkdir(parents=True, exist_ok=True)
     con = connect(root / "duck.db")
     migrate(con)
+    ensure_identity(con, name or os.environ.get("DUCKSTRING_CATCHMENT_NAME"))
 
     app = FastAPI(title="Duckstring Catchment", lifespan=_lifespan)
     app.state.root = root

{duckstring-0.2.0 → duckstring-0.3.0}/src/duckstring/catchment/db.py

@@ -1,12 +1,30 @@
 import sqlite3
+import uuid
 from datetime import datetime, timezone
 from pathlib import Path
 
 _SCHEMA_DIR = Path(__file__).parent / "schema"
 
 
+def ensure_identity(con: sqlite3.Connection, name: str | None = None) -> None:
+    """Mint this Catchment's stable id on first start (never changes), and set/refresh its optional
+    display name. Call after migrate()."""
+    row = con.execute("SELECT value FROM catchment_meta WHERE key = 'id'").fetchone()
+    if row is None:
+        con.execute("INSERT INTO catchment_meta (key, value) VALUES ('id', ?)", (str(uuid.uuid4()),))
+    if name:
+        con.execute(
+            "INSERT INTO catchment_meta (key, value) VALUES ('name', ?) "
+            "ON CONFLICT(key) DO UPDATE SET value = excluded.value",
+            (name,),
+        )
+    con.commit()
+
+
 def connect(path: Path) -> sqlite3.Connection:
     con = sqlite3.connect(path, check_same_thread=False)
+    if path != Path(":memory:") and path.exists():
+        path.chmod(0o600)  # may hold duct credentials (auth headers for upstream Catchments)
     con.execute("PRAGMA foreign_keys = ON")
     con.execute("PRAGMA journal_mode = WAL")
     con.execute("PRAGMA busy_timeout = 5000")  # queue on a locked DB (up to 5 s) instead of erroring