data-designer-engine 0.9.3__tar.gz → 0.9.4__tar.gz

{data_designer_engine-0.9.3 → data_designer_engine-0.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: data-designer-engine
-Version: 0.9.3
+Version: 0.9.4
 Summary: Generation engine for DataDesigner synthetic data generation
 License-Expression: Apache-2.0
 Classifier: Development Status :: 4 - Beta
@@ -15,10 +15,11 @@ Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.10
 Requires-Dist: anyascii<1,>=0.3.3
+Requires-Dist: anyio<5,>=4.14.2
 Requires-Dist: asciichartpy<2,>=1.5.25
 Requires-Dist: chardet<6,>=3.0.2
-Requires-Dist: cryptography<=49,>=48.0.1
-Requires-Dist: data-designer-config==0.9.3
+Requires-Dist: cryptography<51,>=50.0.0
+Requires-Dist: data-designer-config==0.9.4
 Requires-Dist: duckdb<2,>=1.5.0
 Requires-Dist: faker<21,>=20.1.0
 Requires-Dist: fsspec<2026,>=2025.3.0
@@ -34,10 +35,10 @@ Requires-Dist: marko<3,>=2.1.2
 Requires-Dist: mcp<2,>=1.29.0
 Requires-Dist: networkx<4,>=3.0
 Requires-Dist: numpy<3,>=1.23.5
-Requires-Dist: pandas<3,>=2.3.3
+Requires-Dist: pandas<4,>=2.3.3
 Requires-Dist: pyarrow<25,>=24
 Requires-Dist: pydantic<3,>=2.9.2
-Requires-Dist: pyjwt[crypto]<3,>=2.13.0
+Requires-Dist: pyjwt[crypto]<3,>=2.15.0
 Requires-Dist: python-multipart<1,>=0.0.32
 Requires-Dist: rich<15,>=13.7.1
 Requires-Dist: ruff<1,>=0.14.10

{data_designer_engine-0.9.3 → data_designer_engine-0.9.4}/pyproject.toml

@@ -34,9 +34,10 @@ bump = true
 [tool.hatch.metadata.hooks.uv-dynamic-versioning]
 dependencies = [
     "anyascii>=0.3.3,<1",
+    "anyio>=4.14.2,<5", # 4.14.2 fixes AnyIO security advisories pulled in by HTTP clients
     "asciichartpy>=1.5.25,<2",
     "chardet>=3.0.2,<6",  # Pulled in by sqlfluff
-    "cryptography>=48.0.1,<=49",
+    "cryptography>=50.0.0,<51", # 50.0.0 fixes certificate and PKCS#7 security advisories
     "data-designer-config=={{ version }}",
     "duckdb>=1.5.0,<2",
     "faker>=20.1.0,<21",
@@ -53,10 +54,10 @@ dependencies = [
     "mcp>=1.29.0,<2", # 1.29.0 fixes MCP security advisories
     "networkx>=3.0,<4",
     "numpy>=1.23.5,<3",
-    "pandas>=2.3.3,<3",
+    "pandas>=2.3.3,<4",
     "pyarrow>=24,<25", # 24.0.0 fixes PyArrow security advisories
     "pydantic>=2.9.2,<3",
-    "pyjwt[crypto]>=2.13.0,<3", # 2.13.0 fixes PyJWT security advisories pulled in by mcp
+    "pyjwt[crypto]>=2.15.0,<3", # 2.15.0 fixes PyJWT security advisories pulled in by mcp
     "python-multipart>=0.0.32,<1", # 0.0.32 addresses multipart security advisories pulled in by mcp
     "rich>=13.7.1,<15",
     "ruff>=0.14.10,<1",

{data_designer_engine-0.9.3 → data_designer_engine-0.9.4}/src/data_designer/engine/column_generators/generators/validation.py

@@ -119,7 +119,7 @@ class ValidationColumnGenerator(ColumnGeneratorFullColumn[ValidationColumnConfig
             column_name=self.config.name,
             result_callback=result_callback,
             error_callback=error_callback,
-            shutdown_error_rate=settings.shutdown_error_rate,
+            shutdown_error_rate=settings.effective_shutdown_error_rate,
             shutdown_error_window=settings.shutdown_error_window,
             disable_early_shutdown=settings.disable_early_shutdown,
         ) as executor:

{data_designer_engine-0.9.3 → data_designer_engine-0.9.4}/src/data_designer/engine/dataset_builders/dataset_builder.py

@@ -843,7 +843,7 @@ class DatasetBuilder:
                 num_records,
                 buffer_size,
                 on_finalize_row_group=finalize_row_group,
-                shutdown_error_rate=settings.shutdown_error_rate,
+                shutdown_error_rate=settings.effective_shutdown_error_rate,
                 shutdown_error_window=settings.shutdown_error_window,
                 disable_early_shutdown=settings.disable_early_shutdown,
                 trace=trace_enabled,

{data_designer_engine-0.9.3 → data_designer_engine-0.9.4}/src/data_designer/engine/models/clients/adapters/http_model_client.py

@@ -3,6 +3,8 @@
 
 from __future__ import annotations
 
+import asyncio
+import importlib
 import threading
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Any
@@ -15,10 +17,11 @@ from data_designer.engine.models.clients.adapters.http_helpers import (
     wrap_transport_error,
 )
 from data_designer.engine.models.clients.errors import SyncClientUnavailableError, map_http_error_to_provider_error
-from data_designer.engine.models.clients.retry import RetryConfig, RetryTransport, create_retry_transport
+from data_designer.engine.models.clients.retry import RetryConfig, create_retry_transport
 
 if TYPE_CHECKING:
     import httpx
+    from httpx_retries import RetryTransport
 
 
 class ClientConcurrencyMode(StrEnum):
@@ -66,6 +69,7 @@ class HttpModelClient(ABC):
         self._timeout_s = timeout_s
         self._retry_config = retry_config
         self._mode: ClientConcurrencyMode = concurrency_mode
+        self._shard_count = min(16, max(1, max_parallel_requests))
 
         pool_max = max(_MIN_MAX_CONNECTIONS, _POOL_MAX_MULTIPLIER * max_parallel_requests)
         pool_keepalive = max(_MIN_KEEPALIVE_CONNECTIONS, max_parallel_requests)
@@ -78,6 +82,7 @@ class HttpModelClient(ABC):
         self._aclient: httpx.AsyncClient | None = async_client
         self._init_lock = threading.Lock()
         self._closed = False
+        self._close_future: asyncio.Future[list[None | BaseException]] | None = None
 
     @property
     def concurrency_mode(self) -> ClientConcurrencyMode:
@@ -120,7 +125,8 @@ class HttpModelClient(ABC):
                 raise RuntimeError("Model client is closed.")
             if self._aclient is None:
                 if self._transport is None:
-                    inner = lazy.httpx.AsyncHTTPTransport(limits=self._limits)
+                    sharding = importlib.import_module("data_designer.engine.models.clients.adapters.httpx_sharding")
+                    inner = sharding.ShardedAsyncHTTPTransport(limits=self._limits, shard_count=self._shard_count)
                     self._transport = create_retry_transport(
                         self._retry_config, strip_rate_limit_codes=True, transport=inner
                     )
@@ -152,15 +158,18 @@ class HttpModelClient(ABC):
         if self._mode != ClientConcurrencyMode.ASYNC:
             return
         with self._init_lock:
-            async_client = self._aclient
-            transport = self._transport
             self._closed = True
+            resource = self._aclient if self._aclient is not None else self._transport
+            if self._close_future is None and resource is not None:
+                self._close_future = asyncio.gather(resource.aclose(), return_exceptions=True)
+            elif self._close_future is None:
+                return
             self._aclient = None
             self._transport = None
-        if async_client is not None:
-            await async_client.aclose()
-        elif transport is not None:
-            await transport.aclose()
+        results = await asyncio.shield(self._close_future)
+        for result in results:
+            if isinstance(result, BaseException):
+                raise result
 
     # --- HTTP helpers ---
 

data_designer_engine-0.9.4/src/data_designer/engine/models/clients/adapters/httpx_sharding.py

@@ -0,0 +1,82 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Public-API HTTPX transport sharding."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Callable
+from typing import TYPE_CHECKING
+
+# RetryTransport requires concrete AsyncBaseTransport inheritance.
+# This module is loaded only when constructing an async transport.
+from httpx import AsyncBaseTransport
+
+import data_designer.lazy_heavy_imports as lazy
+
+if TYPE_CHECKING:
+    import httpx
+
+_DEFAULT_SHARD_COUNT = 16
+
+
+def _split_limit(total: int, shard_count: int) -> list[int]:
+    base, remainder = divmod(total, shard_count)
+    return [base + (index < remainder) for index in range(shard_count)]
+
+
+class ShardedAsyncHTTPTransport(AsyncBaseTransport):
+    """Distribute requests round-robin across ordinary HTTPX transports."""
+
+    def __init__(
+        self,
+        *,
+        limits: httpx.Limits,
+        shard_count: int = _DEFAULT_SHARD_COUNT,
+        transport_factory: Callable[[httpx.Limits], httpx.AsyncBaseTransport] | None = None,
+    ) -> None:
+        if shard_count <= 0:
+            raise ValueError("shard_count must be positive")
+        if limits.max_connections is None or limits.max_keepalive_connections is None:
+            raise ValueError("sharded transport requires finite connection limits")
+        if limits.max_connections < shard_count or limits.max_keepalive_connections < shard_count:
+            raise ValueError("connection limits must provide at least one connection per shard")
+
+        factory = transport_factory or (lambda shard_limits: lazy.httpx.AsyncHTTPTransport(limits=shard_limits))
+        maximums = _split_limit(limits.max_connections, shard_count)
+        keepalives = _split_limit(limits.max_keepalive_connections, shard_count)
+        self._shard_limits = tuple(
+            lazy.httpx.Limits(
+                max_connections=maximum,
+                max_keepalive_connections=keepalive,
+                keepalive_expiry=limits.keepalive_expiry,
+            )
+            for maximum, keepalive in zip(maximums, keepalives, strict=True)
+        )
+        self._transports = tuple(factory(shard_limits) for shard_limits in self._shard_limits)
+        self._next_transport = 0
+        self._closed = False
+        self._close_future: asyncio.Future[list[None | BaseException]] | None = None
+
+    @property
+    def shard_limits(self) -> tuple[httpx.Limits, ...]:
+        return self._shard_limits
+
+    async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
+        if self._closed:
+            raise RuntimeError("Transport is closed.")
+        transport = self._transports[self._next_transport]
+        self._next_transport = (self._next_transport + 1) % len(self._transports)
+        return await transport.handle_async_request(request)
+
+    async def aclose(self) -> None:
+        self._closed = True
+        if self._close_future is None:
+            self._close_future = asyncio.gather(
+                *(transport.aclose() for transport in self._transports), return_exceptions=True
+            )
+        results = await asyncio.shield(self._close_future)
+        for result in results:
+            if isinstance(result, BaseException):
+                raise result

{data_designer_engine-0.9.3 → data_designer_engine-0.9.4}/src/data_designer/engine/models/clients/retry.py

@@ -7,10 +7,11 @@ import logging
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 
-from httpx_retries import Retry, RetryTransport
+import data_designer.lazy_heavy_imports as lazy
 
 if TYPE_CHECKING:
     import httpx
+    from httpx_retries import RetryTransport
 
 logger = logging.getLogger(__name__)
 
@@ -72,13 +73,13 @@ def create_retry_transport(
                 sorted(reserved_overlap),
             )
             status_codes = status_codes - _RESERVED_STATUS_CODES
-    retry = Retry(
+    retry = lazy.httpx_retries.Retry(
         total=cfg.max_retries,
         backoff_factor=cfg.backoff_factor,
         backoff_jitter=cfg.backoff_jitter,
         max_backoff_wait=cfg.max_backoff_wait,
         status_forcelist=status_codes,
         respect_retry_after_header=True,
-        allowed_methods=Retry.RETRYABLE_METHODS | frozenset(["POST"]),
+        allowed_methods=lazy.httpx_retries.Retry.RETRYABLE_METHODS | frozenset(["POST"]),
     )
-    return RetryTransport(transport=transport, retry=retry)
+    return lazy.httpx_retries.RetryTransport(transport=transport, retry=retry)

{data_designer_engine-0.9.3 → data_designer_engine-0.9.4}/src/data_designer/engine/models/parsers/postprocessors.py

@@ -3,6 +3,9 @@
 
 from __future__ import annotations
 
+import json
+from typing import NoReturn
+
 import json_repair
 
 from data_designer.engine.models.parsers.types import (
@@ -41,6 +44,7 @@ def merge_text_blocks(
 def deserialize_json_code(
     structured_response: LLMStructuredResponse,
 ) -> LLMStructuredResponse:
+    """Deserialize fenced JSON, falling back to a complete bare JSON response."""
     processed_response = structured_response.model_copy()
     processed_response.parsed = []
 
@@ -54,4 +58,15 @@ def deserialize_json_code(
         else:
             processed_response.parsed.append(block)
 
+    if not any(isinstance(block, StructuredDataBlock) for block in processed_response.parsed):
+        try:
+            deserialized = json.loads(structured_response.response, parse_constant=_reject_json_constant)
+        except ValueError:
+            return processed_response
+        processed_response.parsed = [StructuredDataBlock(serialized=structured_response.response, obj=deserialized)]
+
     return processed_response
+
+
+def _reject_json_constant(value: str) -> NoReturn:
+    raise ValueError(f"Non-standard JSON constant: {value}")

{data_designer_engine-0.9.3 → data_designer_engine-0.9.4}/src/data_designer/engine/models/recipes/response_recipes.py

@@ -82,7 +82,7 @@ class StructuredResponseRecipe(ResponseRecipe[dict]):
             pruning (bool): If `True`, then any extra fields in the returned
                 JSON object will be removed. Otherwise, they are retained,
                 which could raise validation errors. Default=True
-            no_extra_properties (bool) If `True`, then validation will fail
+            no_extra_properties (bool): If `True`, then validation will fail
                 if extra properties are encountered in the returned JSON response.
                 Default=True.
         """
@@ -139,7 +139,7 @@ class StructuredResponseRecipe(ResponseRecipe[dict]):
                 return validate(obj, **self._validate_args)
             except IndexError:
                 raise ParserException(
-                    "No parsable JSON structure within ```json markdown fence.",
+                    "No parsable JSON found: expected a ```json markdown fence or a complete bare JSON response.",
                     source=response,
                 ) from None
             except JSONSchemaValidationError as exc:
@@ -219,7 +219,7 @@ class PydanticResponseRecipe(ResponseRecipe[BaseModel]):
                 return self.data_type.model_validate(obj)
             except IndexError:
                 raise ParserException(
-                    "No parsable JSON structure within ```json markdown fence.",
+                    "No parsable JSON found: expected a ```json markdown fence or a complete bare JSON response.",
                     source=response,
                 ) from None
             except Exception as exc:

{data_designer_engine-0.9.3 → data_designer_engine-0.9.4}/src/data_designer/engine/models/request_admission/queue.py

@@ -51,7 +51,6 @@ class RequestFairQueue:
         self._queued: dict[str, RequestWaiter] = {}
         self._waiter_groups: dict[str, RequestResourceKey] = {}
         self._group_finish: dict[RequestResourceKey, float] = {}
-        self._heap: list[tuple[float, int, RequestResourceKey]] = []
         self._active_heap_entries: dict[RequestResourceKey, tuple[float, int]] = {}
         self._sequence = 0
         self._sequence_version = 0
@@ -83,7 +82,11 @@ class RequestFairQueue:
         waiter = self._queued.pop(waiter_id, None)
         if waiter is None:
             return None
-        self._waiter_groups.pop(waiter_id, None)
+        key = self._waiter_groups.pop(waiter_id, None)
+        if key is not None:
+            self._purge_queue_head(key)
+            if not self._queues.get(key):
+                self._active_heap_entries.pop(key, None)
         self._sequence_version += 1
         return waiter
 
@@ -91,7 +94,7 @@ class RequestFairQueue:
         self, is_eligible: Callable[[RequestWaiter, RequestQueueView], bool]
     ) -> RequestQueueSelection | None:
         view = self.view()
-        heap_copy = list(self._heap)
+        heap_copy = [(finish, sequence, key) for key, (finish, sequence) in self._active_heap_entries.items()]
         heapq.heapify(heap_copy)
         active_seen: set[RequestResourceKey] = set()
         while heap_copy:
@@ -165,7 +168,6 @@ class RequestFairQueue:
             return
         self._sequence += 1
         finish = self._group_finish.get(key, self._virtual_time)
-        heapq.heappush(self._heap, (finish, self._sequence, key))
         self._active_heap_entries[key] = (finish, self._sequence)
 
     def _first_valid_waiter(self, key: RequestResourceKey) -> RequestWaiter | None:

{data_designer_engine-0.9.3 → data_designer_engine-0.9.4}/src/data_designer/engine/processing/ginja/ast.py

@@ -62,6 +62,7 @@ def ast_count_name_references(ast: j_nodes.Node, name: str) -> int:
 
     Args:
         ast (jinja2.nodes.Node): The starting Jinja2 AST node
+        name (str): The variable name to count references to.
 
     Returns:
         int: The number of nodes descended from the provided node whose

{data_designer_engine-0.9.3 → data_designer_engine-0.9.4}/src/data_designer/engine/validation.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import re
 from enum import Enum
 from fnmatch import fnmatch
 from string import Formatter
@@ -451,9 +452,28 @@ def validate_skip_references(
     return violations
 
 
+_FORMAT_FIELD_REFERENCE_PATTERN = re.compile(
+    r"(?<!\{)\{"
+    r"([^{}!:]+)"
+    r"(?:![sra])?"
+    r"(?::[^{}]*(?:\{[^{}]*\}[^{}]*)*)?"
+    r"\}"
+)
+
+
 def _get_string_formatter_references(template: str, allowed_references: list[str]) -> list[str]:
-    return [
-        k[1].strip()
-        for k in Formatter().parse(template)
-        if len(k) > 1 and k[1] is not None and k[1].strip() in allowed_references
-    ]
+    try:
+        return [
+            k[1].strip()
+            for k in Formatter().parse(template)
+            if len(k) > 1 and k[1] is not None and k[1].strip() in allowed_references
+        ]
+    except ValueError:
+        # Unmatched literal braces (e.g. JSON examples like 'output format: }') are
+        # invalid f-string syntax but valid Jinja text, so ``Formatter().parse`` can
+        # raise ``ValueError``. Fall back to a tolerant regex scan that skips Jinja
+        # ``{{ ... }}`` expressions so the advisory check still detects ``{column}``
+        # references without crashing validation.
+        return [
+            ref for m in _FORMAT_FIELD_REFERENCE_PATTERN.findall(template) if (ref := m.strip()) in allowed_references
+        ]

{data_designer_engine-0.9.3 → data_designer_engine-0.9.4}/tests/engine/column_generators/generators/test_validation.py

@@ -9,6 +9,7 @@ import pytest
 
 import data_designer.lazy_heavy_imports as lazy
 from data_designer.config.column_configs import ValidationColumnConfig
+from data_designer.config.run_config import RunConfig
 from data_designer.config.utils.code_lang import CodeLang
 from data_designer.config.validator_params import (
     CodeValidatorParams,
@@ -203,8 +204,16 @@ def test_validation_column_generator_generate_with_different_strategies(
     assert len(result["validation_column"]) == len(df)
 
 
+@pytest.mark.parametrize(
+    ("disable", "rate", "expected_rate"),
+    [(True, 0.2, 1.0), (False, 0.2, 0.2)],
+    ids=["disabled-passes-effective-rate", "enabled-passes-configured-rate"],
+)
 @patch("data_designer.engine.column_generators.generators.validation.get_validator_from_params", autospec=True)
-def test_validation_column_generator_validate_in_parallel_failure(mock_get_validator, stub_resource_provider):
+def test_validation_column_generator_validate_in_parallel_failure(
+    mock_get_validator: Mock, stub_resource_provider: Mock, disable: bool, rate: float, expected_rate: float
+) -> None:
+    stub_resource_provider.run_config = RunConfig(disable_early_shutdown=disable, shutdown_error_rate=rate)
     mock_validator = Mock()
     mock_validator.run_validation.return_value = ValidationResult(data=[ValidationOutput(is_valid=True)])
     mock_get_validator.return_value = mock_validator
@@ -242,3 +251,4 @@ def test_validation_column_generator_validate_in_parallel_failure(mock_get_valid
 
         call_kwargs = mock_executor_class.call_args[1]
         assert call_kwargs["disable_early_shutdown"] == stub_resource_provider.run_config.disable_early_shutdown
+        assert call_kwargs["shutdown_error_rate"] == expected_rate

{data_designer_engine-0.9.3 → data_designer_engine-0.9.4}/tests/engine/dataset_builders/test_async_scheduler.py

@@ -4989,7 +4989,6 @@ async def test_scheduler_request_resource_admission_avoids_creating_waiters() ->
     request_admission = AdaptiveRequestAdmissionController(
         RequestAdmissionConfig(
             initial_limits={resource: 4},
-            default_queue_wait_timeout_seconds=0.02,
         ),
         event_sink=sink,
     )