dataenginex 0.4.5__tar.gz → 0.4.6__tar.gz

{dataenginex-0.4.5 → dataenginex-0.4.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataenginex
-Version: 0.4.5
+Version: 0.4.6
 Summary: DataEngineX — open-source, self-hosted, local-first Data + ML + AI workbench library
 Author-email: Jay <jayapal.myaka99@gmail.com>
 License: MIT License

{dataenginex-0.4.5 → dataenginex-0.4.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dataenginex"
-version = "0.4.5"
+version = "0.4.6"
 description = "DataEngineX — open-source, self-hosted, local-first Data + ML + AI workbench library"
 authors = [{ name = "Jay", email = "jayapal.myaka99@gmail.com" }]
 readme = "src/dataenginex/README.md"

{dataenginex-0.4.5 → dataenginex-0.4.6}/src/dataenginex/ai/tools/builtin.py

@@ -29,13 +29,30 @@ logger = structlog.get_logger()
 
 # ── SQL / lakehouse ────────────────────────────────────────────────────────────
 
+# This tool is reachable from HTTP-exposed agent/native-call routes, so the SQL
+# it runs must be treated as untrusted input. DuckDB's enable_external_access
+# blocks filesystem/network access (read_csv/read_parquet/httpfs/ATTACH/etc.)
+# for the connection it's set on; the statement-shape check below is a second,
+# cheap layer against non-SELECT statements (COPY/INSTALL/PRAGMA/...).
+_READ_ONLY_PREFIXES = ("SELECT", "WITH")
+
+
+def _require_read_only(sql: str) -> None:
+    stripped = sql.strip().lstrip("(").strip()
+    if ";" in stripped[:-1]:  # allow a single optional trailing semicolon
+        raise ValueError("query tool only allows a single statement")
+    if not stripped.upper().startswith(_READ_ONLY_PREFIXES):
+        raise ValueError("query tool only allows read-only SELECT/WITH statements")
+
 
 def _query_sql(sql: str, database: str = ":memory:") -> list[dict[str, Any]]:
-    """Execute a SQL query via DuckDB and return results."""
+    """Execute a read-only SQL query via DuckDB and return results."""
     import duckdb
 
+    _require_read_only(sql)
     conn = duckdb.connect(database)
     try:
+        conn.execute("SET enable_external_access=false")
         result = conn.execute(sql)
         description = result.description or []
         if not description:
@@ -52,6 +69,7 @@ def _make_lakehouse_query(lakehouse_dir: Path) -> Callable[[str], list[dict[str,
     def _query_with_lakehouse(sql: str) -> list[dict[str, Any]]:
         import duckdb
 
+        _require_read_only(sql)
         conn = duckdb.connect(":memory:")
         try:
             for layer in ("bronze", "silver", "gold"):
@@ -65,6 +83,9 @@ def _make_lakehouse_query(lakehouse_dir: Path) -> Callable[[str], list[dict[str,
                             f"CREATE OR REPLACE VIEW {pf.stem}"
                             f" AS SELECT * FROM read_parquet('{safe}')"
                         )
+            # Lock the connection down to the views already registered above —
+            # untrusted `sql` below can no longer touch the filesystem/network.
+            conn.execute("SET enable_external_access=false")
             result = conn.execute(sql)
             description = result.description or []
             if not description:

{dataenginex-0.4.5 → dataenginex-0.4.6}/src/dataenginex/ai/vectorstore.py

@@ -23,8 +23,8 @@ Example::
 from __future__ import annotations
 
 import abc
+import hashlib
 import math
-import uuid
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any
 
@@ -55,11 +55,20 @@ __all__ = [
 class Document:
     """A text document with optional metadata and embedding."""
 
-    id: str = field(default_factory=lambda: uuid.uuid4().hex[:16])
+    id: str = ""
     text: str = ""
     metadata: dict[str, Any] = field(default_factory=dict)
     embedding: list[float] = field(default_factory=list)
 
+    def __post_init__(self) -> None:
+        if not self.id:
+            # Content-hash id, not random: re-ingesting the same text upserts
+            # (overwrites) the existing entry instead of piling up a new
+            # random-id duplicate. InMemoryBackend has no eviction/cap, so a
+            # random id here means unbounded growth across repeated scheduled
+            # pipeline runs that re-ingest the same rows.
+            self.id = hashlib.sha256(self.text.encode("utf-8")).hexdigest()[:16]
+
 
 @dataclass
 class SearchResult:
@@ -447,7 +456,10 @@ class RAGPipeline:
             Number of documents stored.
         """
         meta = metadata or [{} for _ in texts]
-        doc_ids = ids or [uuid.uuid4().hex[:16] for _ in texts]
+        # Content-hash ids by default (same scheme as Document.__post_init__)
+        # so re-ingesting identical text upserts instead of duplicating —
+        # see Document's docstring for why a random id here matters.
+        doc_ids = ids or [hashlib.sha256(t.encode("utf-8")).hexdigest()[:16] for t in texts]
 
         docs: list[Document] = []
         for doc_id, text, m in zip(doc_ids, texts, meta, strict=True):

{dataenginex-0.4.5 → dataenginex-0.4.6}/src/dataenginex/data/connectors/dbt.py

@@ -73,6 +73,7 @@ class DbtConnector(BaseConnector):
         target_database: str | None = None,
         profiles_dir: str | None = None,
         target: str = "dev",
+        timeout_s: float = 600,
         **kwargs: Any,
     ) -> None:
         if not _DBT_AVAILABLE:
@@ -84,6 +85,7 @@ class DbtConnector(BaseConnector):
         )
         self._profiles_dir = Path(profiles_dir) if profiles_dir else self._project_dir
         self._target = target
+        self._timeout_s = timeout_s
 
     def connect(self) -> None:
         project_file = self._project_dir / "dbt_project.yml"
@@ -131,7 +133,13 @@ class DbtConnector(BaseConnector):
             "--target",
             self._target,
         ]
-        proc = subprocess.run(cmd, capture_output=True, text=True)  # noqa: S603,S607
+        try:
+            proc = subprocess.run(  # noqa: S603,S607
+                cmd, capture_output=True, text=True, timeout=self._timeout_s
+            )
+        except subprocess.TimeoutExpired as exc:
+            msg = f"dbt run timed out for model '{model}' after {self._timeout_s}s"
+            raise RuntimeError(msg) from exc
         if proc.returncode != 0:
             msg = f"dbt run failed for model '{model}':\n{proc.stdout}\n{proc.stderr}"
             raise RuntimeError(msg)

{dataenginex-0.4.5 → dataenginex-0.4.6}/tests/unit/test_vectorstore_extended.py

@@ -20,10 +20,17 @@ class TestDocument:
         d = Document(text="hello")
         assert len(d.id) > 0
 
-    def test_ids_unique(self) -> None:
-        ids = {Document().id for _ in range(50)}
+    def test_ids_deterministic_by_content(self) -> None:
+        """Ids are content-hash derived, not random — re-ingesting the same
+        text must upsert (same id), and distinct texts must not collide."""
+        assert Document(text="hello").id == Document(text="hello").id
+        ids = {Document(text=f"doc-{i}").id for i in range(50)}
         assert len(ids) == 50
 
+    def test_explicit_id_not_overridden(self) -> None:
+        d = Document(id="custom-id", text="hello")
+        assert d.id == "custom-id"
+
     def test_metadata_default_empty(self) -> None:
         d = Document(text="x")
         assert d.metadata == {}

{dataenginex-0.4.5 → dataenginex-0.4.6}/uv.lock

@@ -830,7 +830,7 @@ wheels = [
 
 [[package]]
 name = "dataenginex"
-version = "0.4.5"
+version = "0.4.6"
 source = { editable = "." }
 dependencies = [
     { name = "click" },