vgi-python 0.36.1__tar.gz → 0.36.2__tar.gz

{vgi_python-0.36.1 → vgi_python-0.36.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vgi-python
-Version: 0.36.1
+Version: 0.36.2
 Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
 Project-URL: Homepage, https://query.farm
 Project-URL: Repository, https://github.com/Query-farm/vgi-python

vgi_python-0.36.2/conformance/filter-v2/cases/positive/boolean-column-predicate.json

@@ -0,0 +1,32 @@
+{
+  "encoding": "vgi.filters.v2",
+  "semantics": "vgi.duckdb.standard.v1",
+  "kind": "snapshot",
+  "predicates": [
+    {
+      "id": "query:bare-boolean-column",
+      "revision": 0,
+      "mode": "required",
+      "source": "query",
+      "expression": {
+        "node": "column_ref",
+        "column_index": 0,
+        "column_name": "flag"
+      }
+    },
+    {
+      "id": "query:negated-boolean-column",
+      "revision": 0,
+      "mode": "required",
+      "source": "query",
+      "expression": {
+        "node": "not",
+        "expression": {
+          "node": "column_ref",
+          "column_index": 1,
+          "column_name": "other_flag"
+        }
+      }
+    }
+  ]
+}

{vgi_python-0.36.1 → vgi_python-0.36.2}/conformance/filter-v2/manifest.json

@@ -30,6 +30,11 @@
       "document": "cases/positive/runtime-filter-snapshot.json",
       "expected": "valid"
     },
+    {
+      "id": "structural.boolean-column-predicate",
+      "document": "cases/positive/boolean-column-predicate.json",
+      "expected": "valid"
+    },
     {
       "id": "structural.and-too-few-children",
       "document": "cases/negative/and-too-few-children.json",

{vgi_python-0.36.1 → vgi_python-0.36.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vgi-python"
-version = "0.36.1"
+version = "0.36.2"
 description = "Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow"
 readme = "README.md"
 keywords = [

{vgi_python-0.36.1 → vgi_python-0.36.2}/tests/test_filter_v2.py

@@ -6,7 +6,7 @@ from __future__ import annotations
 
 import json
 from collections.abc import Mapping
-from typing import cast
+from typing import Any, cast
 
 import pyarrow as pa
 import pytest
@@ -531,3 +531,168 @@ def test_unknown_arrow_extension_metadata_is_rejected() -> None:
             _batch(_document([_predicate(_comparison())]), (extension_field, b"opaque")),
             output_schema=pa.schema([("n", pa.int64())]),
         )
+
+
+# ---------------------------------------------------------------------------
+# A boolean column is a predicate on its own
+#
+# `WHERE flag` / `WHERE NOT flag` is idiomatic SQL, and DuckDB pushes it down as
+# a bare `column_ref` rather than rewriting it to `flag = true`. The schema
+# admits that -- `coreExpression` lists `columnRef` first -- but the decoder's
+# boolean gate used to enumerate node *kinds* instead of asking for the resolved
+# *type*, so it refused every one of them: "predicate root must resolve to
+# BOOLEAN", on a node whose type it had already resolved to BOOLEAN and stored.
+# ---------------------------------------------------------------------------
+
+_BOOL_FIELDS: list[pa.Field[Any]] = [pa.field("flag", pa.bool_()), pa.field("n", pa.int64())]
+_BOOL_SCHEMA = pa.schema(_BOOL_FIELDS)
+
+
+def _flag(index: int = 0) -> dict[str, object]:
+    return {"node": "column_ref", "column_index": index, "column_name": "flag"}
+
+
+def test_bare_boolean_column_is_a_valid_predicate_root() -> None:
+    """`WHERE flag` -- the column itself is the predicate."""
+    filters = deserialize_filters(
+        _batch(_document([_predicate(_flag())])),
+        output_schema=_BOOL_SCHEMA,
+    )
+    batch = pa.RecordBatch.from_pydict(
+        {"flag": [True, False, None, True], "n": [1, 2, 3, 4]},
+        schema=_BOOL_SCHEMA,
+    )
+    # evaluate() is the raw three-valued predicate: NULL stays NULL.
+    assert filters.evaluate(batch).to_pylist() == [True, False, None, True]
+    # apply() is what selects rows, and there a NULL predicate is not satisfied
+    # -- the same rows `flag = true` would keep, which is what makes the SQL
+    # rewrite below a projection rather than a change of meaning.
+    assert filters.apply(batch).to_pydict()["n"] == [1, 4]
+
+
+def test_negated_boolean_column_is_a_valid_predicate_root() -> None:
+    """`WHERE NOT flag` -- `not` over a bare column, which the gate also refused."""
+    filters = deserialize_filters(
+        _batch(_document([_predicate({"node": "not", "expression": _flag()})])),
+        output_schema=_BOOL_SCHEMA,
+    )
+    batch = pa.RecordBatch.from_pydict(
+        {"flag": [True, False, None, True], "n": [1, 2, 3, 4]},
+        schema=_BOOL_SCHEMA,
+    )
+    assert filters.evaluate(batch).to_pylist() == [False, True, None, False]
+    # NOT NULL is NULL, so row 3 is not kept -- exactly `flag = false`.
+    assert filters.apply(batch).to_pydict()["n"] == [2]
+
+
+def test_boolean_column_predicates_render_as_sql() -> None:
+    """They must reach SQL, not just the evaluator.
+
+    Left as a generic v2 expression a bare column renders no SQL at all, so a
+    SQL-backed worker loses pushdown on the commonest boolean predicate there
+    is -- and DuckDB does not re-apply what it pushed down, so that is a wrong
+    answer, not a slow one.
+    """
+    positive = deserialize_filters(_batch(_document([_predicate(_flag())])), output_schema=_BOOL_SCHEMA)
+    where, params = positive.to_sql()
+    assert where == '"flag" = ?'
+    assert params == [True]
+
+    negative = deserialize_filters(
+        _batch(_document([_predicate({"node": "not", "expression": _flag()})])),
+        output_schema=_BOOL_SCHEMA,
+    )
+    where, params = negative.to_sql()
+    assert where == '"flag" = ?'
+    assert params == [False]
+
+
+def test_boolean_column_inside_a_conjunction_still_pushes_down() -> None:
+    """The shape that actually showed up: `WHERE other = x AND NOT flag`.
+
+    One unrenderable child used to cost the whole conjunction its pushdown.
+    """
+    document = _document(
+        [
+            _predicate(
+                {
+                    "node": "and",
+                    "children": [
+                        {
+                            "node": "comparison",
+                            "op": "gt",
+                            "left": _column(name="n", index=1),
+                            "right": {"node": "literal", "value_ref": 0},
+                        },
+                        {"node": "not", "expression": _flag()},
+                    ],
+                }
+            )
+        ]
+    )
+    filters = deserialize_filters(
+        _batch(document, (pa.field("value_0", pa.int64()), 2)),
+        output_schema=_BOOL_SCHEMA,
+    )
+    where, params = filters.to_sql()
+    assert where == '("n" > ? AND "flag" = ?)'
+    assert params == [2, False]
+
+
+def test_an_unresolved_column_is_not_assumed_boolean() -> None:
+    """Without a bind schema the column's type is unknown -- refuse, don't guess."""
+    with pytest.raises(FilterDeserializationError, match="predicate root"):
+        deserialize_filters(_batch(_document([_predicate(_flag())])))
+
+
+def test_a_non_boolean_column_is_still_refused() -> None:
+    """`WHERE n` where n is BIGINT is not a predicate."""
+    with pytest.raises(FilterDeserializationError, match="predicate root"):
+        deserialize_filters(
+            _batch(_document([_predicate(_column(name="n", index=1))])),
+            output_schema=_BOOL_SCHEMA,
+        )
+
+
+def test_an_unrenderable_filter_raises_rather_than_vanishing() -> None:
+    """`to_sql()` must never quietly drop a predicate it cannot render.
+
+    DuckDB does not re-apply a filter it pushed into a table function, so the
+    worker's WHERE clause is the only thing standing between the caller and
+    rows the predicate excludes. A skipped filter is therefore a wrong answer,
+    not a slow one -- which is why this raises instead of degrading.
+    """
+    from vgi.table_filter_pushdown import FilterSQLUnsupportedError
+
+    # `n + 1 > 2` -- arithmetic has no legacy leaf shape and no SQL rendering.
+    document = _document(
+        [
+            _predicate(
+                {
+                    "node": "comparison",
+                    "op": "gt",
+                    "left": {
+                        "node": "arithmetic",
+                        "op": "add",
+                        "left": _column(name="n", index=1),
+                        "right": {"node": "literal", "value_ref": 0},
+                    },
+                    "right": {"node": "literal", "value_ref": 1},
+                }
+            )
+        ]
+    )
+    filters = deserialize_filters(
+        _batch(
+            document,
+            (pa.field("value_0", pa.int64()), 1),
+            (pa.field("value_1", pa.int64()), 2),
+        ),
+        output_schema=_BOOL_SCHEMA,
+    )
+    # It evaluates correctly -- there is simply no SQL for it.
+    batch = pa.RecordBatch.from_pydict({"flag": [True, True, True], "n": [0, 2, 5]}, schema=_BOOL_SCHEMA)
+    assert filters.apply(batch).to_pydict()["n"] == [2, 5]
+
+    with pytest.raises(FilterSQLUnsupportedError, match="no SQL rendering"):
+        filters.to_sql()

{vgi_python-0.36.1 → vgi_python-0.36.2}/uv.lock

@@ -2580,7 +2580,7 @@ requires-dist = [
 
 [[package]]
 name = "vgi-python"
-version = "0.36.1"
+version = "0.36.2"
 source = { editable = "." }
 dependencies = [
     { name = "click" },

{vgi_python-0.36.1 → vgi_python-0.36.2}/vgi/_test_fixtures/table/__init__.py

@@ -65,6 +65,7 @@ from vgi._test_fixtures.table.catalog_scans import (
     ProjectsScanFunction,
 )
 from vgi._test_fixtures.table.filters import (
+    BoolFilterEchoFunction,
     DictFilterEchoFunction,
     DynamicFilterEchoFunction,
     ExpressionFilterTestFunction,
@@ -216,6 +217,7 @@ __all__ = [
     "ConstantColumnsFunction",
     "CountryPartitionedSalesFunction",
     "DepartmentsScanFunction",
+    "BoolFilterEchoFunction",
     "DictFilterEchoFunction",
     "DisjointRangePartitionedFunction",
     "DoubleSequenceFunction",

{vgi_python-0.36.1 → vgi_python-0.36.2}/vgi/_test_fixtures/table/filters.py

@@ -191,6 +191,129 @@ class FilterEchoFunction(TableFunctionGenerator[FilterEchoFunctionArgs, FilterEc
         state.remaining -= size
 
 
+# ============================================================================
+# BoolFilterEchoFunction — a table function with a BOOLEAN column, so the
+# bare-column predicate shapes (`WHERE flag`, `WHERE NOT flag`) can be driven
+# end to end.
+#
+# filter_echo cannot cover these: its schema has no boolean column, so the
+# predicate cannot even be written. The table-in-out `echo` path cannot either
+# — DuckDB does not hand it a bare boolean column as a pushed predicate, so a
+# test written there passes whether or not the worker can decode one. It takes
+# a TABLE FUNCTION with a boolean column to see the shape at all, which is why
+# a decoder that refused every one of them ("predicate root must resolve to
+# BOOLEAN", vgi-python < 0.36.2) went unnoticed by a filter-pushdown suite of
+# 348 files including one dedicated to booleans.
+#
+# `pushed_filters` echoes the SQL rendering, so this covers both halves: the
+# decode (the rows are auto-applied and must be right) and `to_sql()` (a shape
+# that parses but renders nothing would show up here as "(none)").
+# ============================================================================
+
+
+@dataclass(slots=True, frozen=True)
+class _BoolFilterEchoArgs:
+    """Arguments for BoolFilterEchoFunction."""
+
+    count: Annotated[int, Arg(0, doc="Number of rows to generate", ge=0)]
+
+
+@dataclass(kw_only=True)
+class _BoolFilterEchoState(ArrowSerializableDataclass):
+    """Remaining rows plus the echoed SQL rendering of the pushed filters."""
+
+    remaining: int
+    current_index: int = 0
+    filter_str: str = "(none)"
+
+
+@init_single_worker
+@bind_fixed_schema
+@_cardinality_from_count
+class BoolFilterEchoFunction(TableFunctionGenerator[_BoolFilterEchoArgs, _BoolFilterEchoState]):
+    """Rows with a nullable BOOLEAN column, echoing the pushed-down filters.
+
+    SCHEMA
+    ------
+    Output: {"n": int64, "flag": bool, "pushed_filters": string}
+
+    ``flag`` cycles TRUE, FALSE, NULL so every predicate is exercised against
+    all three truth values -- the NULL row is what distinguishes ``WHERE flag``
+    from ``WHERE flag IS NOT FALSE``, and ``WHERE NOT flag`` from
+    ``WHERE flag IS NOT TRUE``.
+
+    Attributes:
+        FIXED_SCHEMA: The fixed Arrow output schema this function always produces.
+
+    """
+
+    class Meta:
+        """Metadata for BoolFilterEchoFunction."""
+
+        name = "bool_filter_echo"
+        description = "Rows with a nullable BOOLEAN column, echoing pushed-down filters"
+        categories = ["generator", "diagnostic"]
+        filter_pushdown = True
+        auto_apply_filters = True
+        projection_pushdown = True
+        examples = [
+            FunctionExample(
+                sql="SELECT * FROM bool_filter_echo(6) WHERE flag",
+                description="A boolean column is a predicate on its own",
+            ),
+        ]
+
+    FIXED_SCHEMA: ClassVar[pa.Schema] = schema({"n": pa.int64(), "flag": pa.bool_(), "pushed_filters": pa.utf8()})
+
+    @classmethod
+    def initial_state(cls, params: ProcessParams[_BoolFilterEchoArgs]) -> _BoolFilterEchoState:
+        """Create initial state, caching the SQL rendering of the pushed filters."""
+        assert params.init_call is not None
+        pf = params.init_call.pushdown_filters
+        filters = (
+            cls.pushdown_filters(
+                pf,
+                join_keys=params.init_call.join_keys,
+                output_schema=params.init_call.output_schema,
+            )
+            if pf is not None
+            else None
+        )
+        return _BoolFilterEchoState(
+            remaining=params.args.count,
+            filter_str=_format_pushed_filters(filters),
+        )
+
+    @classmethod
+    def process(
+        cls,
+        params: ProcessParams[_BoolFilterEchoArgs],
+        state: _BoolFilterEchoState,
+        out: OutputCollector,
+    ) -> None:
+        """Emit rows whose ``flag`` cycles TRUE, FALSE, NULL."""
+        if state.remaining <= 0:
+            out.finish()
+            return
+
+        n_values = list(range(state.current_index, state.current_index + state.remaining))
+        flags: list[bool | None] = [(True, False, None)[i % 3] for i in n_values]
+
+        out.emit(
+            pa.RecordBatch.from_pydict(
+                {
+                    "n": n_values,
+                    "flag": flags,
+                    "pushed_filters": [state.filter_str] * len(n_values),
+                },
+                schema=params.output_schema,
+            )
+        )
+
+        state.current_index += state.remaining
+        state.remaining = 0
+
+
 # ============================================================================
 # ValuePruneFunction — exercises PushdownFilters.get_column_values('n'), the
 # partition-pruning idiom (resolve the discrete value set up front, fetch only

{vgi_python-0.36.1 → vgi_python-0.36.2}/vgi/_test_fixtures/worker.py

@@ -140,6 +140,7 @@ from vgi._test_fixtures.table import (
     RFF_SIMPLE_COLUMNS,
     RFF_STRUCT_COLUMNS,
     BatchIndexOverflowFunction,
+    BoolFilterEchoFunction,
     BrokenMissingPartitionValuesFunction,
     BrokenPartitionColumnAbsentFromBatchFunction,
     BrokenPartitionMinNeqMaxFunction,
@@ -486,6 +487,7 @@ _EXAMPLE_CATALOG = Catalog(
                 # TableFunctionGenerator - generate output without input
                 ConstantColumnsFunction,
                 SlowCancellableFunction,
+                BoolFilterEchoFunction,
                 FilterEchoFunction,
                 FilterEchoPartitionedFunction,
                 FilterEchoTableScanFunction,

{vgi_python-0.36.1 → vgi_python-0.36.2}/vgi/filter_v2.py

@@ -740,11 +740,24 @@ class _Parser:
 
     @staticmethod
     def _is_boolean(expression: FilterExpression) -> bool:
-        if isinstance(expression, Literal):
-            return pa.types.is_boolean(expression.field.type)
-        if isinstance(expression, (Comparison, BooleanExpression, Not, IsNull, In, RuntimeFilter)):
-            return True
-        return isinstance(expression, Call)
+        """Whether *expression* resolves to BOOLEAN.
+
+        Ask the type, never the node kind. The two agree for every node whose
+        type is structurally fixed -- a comparison is BOOLEAN whatever its
+        operands -- and that is exactly why enumerating kinds looked sufficient.
+        It is not: ``column_ref``, ``field_ref``, ``cast`` and ``arithmetic``
+        are BOOLEAN or not depending on a type resolved from elsewhere (the
+        bind output schema, or a parent expression), and the schema admits all
+        four as a predicate root -- ``coreExpression`` lists ``columnRef``
+        first. Enumerating kinds rejected the lot, so the idiomatic
+        ``WHERE flag`` / ``WHERE NOT flag`` -- which DuckDB pushes down as a
+        bare column reference rather than rewriting to ``flag = true`` -- was
+        refused as "predicate root must resolve to BOOLEAN" even though the
+        parser had already resolved the column to BOOLEAN and stored it on the
+        node.
+        """
+        data_type = expression_type(expression)
+        return data_type is not None and pa.types.is_boolean(data_type)
 
     def _payload(self, prefix: str, ref: object) -> tuple[pa.Field[Any], pa.Scalar[Any]]:
         index = _uint(ref, f"{prefix}_ref")

{vgi_python-0.36.1 → vgi_python-0.36.2}/vgi/table_filter_pushdown.py

@@ -63,6 +63,9 @@ from vgi.filter_v2 import (
 from vgi.filter_v2 import (
     Literal as V2Literal,
 )
+from vgi.filter_v2 import (
+    Not as V2Not,
+)
 from vgi.filter_v2 import (
     RuntimeFilter as V2RuntimeFilter,
 )
@@ -72,6 +75,9 @@ from vgi.filter_v2 import (
 from vgi.filter_v2 import (
     expression_columns as v2_expression_columns,
 )
+from vgi.filter_v2 import (
+    expression_type as v2_expression_type,
+)
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterator
@@ -207,6 +213,7 @@ __all__ = [
     "OrFilter",
     "StructFilter",
     "ExpressionFilter",
+    "FilterSQLUnsupportedError",
     "V2ExpressionFilter",
     # Expression node classes
     "ExpressionNode",
@@ -1638,6 +1645,22 @@ class PushdownFilters:
 # =============================================================================
 
 
+class FilterSQLUnsupportedError(ValueError):
+    """A filter that `to_sql()` cannot render as a SQL fragment.
+
+    Raised rather than skipped, and that is the whole point: DuckDB does **not**
+    re-apply a predicate it pushed into a table function, so a WHERE clause that
+    silently omits one does not return the same rows more slowly -- it returns
+    the wrong rows. Degrading quietly here would turn an unrenderable filter
+    into a wrong answer with no error anywhere.
+
+    A worker that catches this has two honest options: evaluate the filters in
+    memory instead (`PushdownFilters.apply()` / `evaluate()` handle every v2
+    expression), or decline the scan. Subclasses `ValueError` so existing
+    callers that catch the old error keep working.
+    """
+
+
 def _filter_to_sql(
     f: Filter,
     quote: Callable[[str], str],
@@ -1713,8 +1736,19 @@ def _filter_to_sql(
             # Constants are embedded directly in the SQL string.
             return expr.to_sql(f.column_name), []
 
+        case V2ExpressionFilter(expression=expression):
+            # A v2 expression the legacy projection could not reduce to a leaf
+            # shape (arithmetic, casts, extension calls, ...). It evaluates
+            # correctly -- `V2ExpressionFilter.evaluate()` runs the v2 reference
+            # evaluator -- there is simply no SQL rendering for it yet.
+            raise FilterSQLUnsupportedError(
+                f"filter on {f.column_name!r} is a v2 expression with no SQL rendering "
+                f"({type(expression).__name__}); evaluate it with PushdownFilters.apply() "
+                f"instead, or the scan will return rows the predicate excludes"
+            )
+
         case _:
-            raise ValueError(f"Unknown filter type: {type(f)}")
+            raise FilterSQLUnsupportedError(f"Unknown filter type: {type(f)}")
 
 
 # =============================================================================
@@ -1787,6 +1821,48 @@ def _pushdown_from_v2_state(state: V2FilterState) -> PushdownFilters:
     )
 
 
+def _v2_boolean_column_view(expression: V2FilterExpression) -> Filter | None:
+    """Project ``WHERE flag`` / ``WHERE NOT flag`` onto the legacy equality leaf.
+
+    DuckDB pushes a predicate that *is* a boolean column down as a bare
+    ``column_ref``, not as ``flag = true``. Left as a generic v2 expression it
+    becomes a `V2ExpressionFilter`, which evaluates fine but renders no SQL --
+    so every SQL-backed worker loses pushdown on the most ordinary boolean
+    predicate there is, and on the commonest shape of all, a conjunction with
+    one, it loses pushdown for the whole conjunction.
+
+    The rewrite is exact rather than approximate, including under NULLs:
+    ``WHERE flag`` keeps only TRUE (a NULL predicate is not satisfied), and so
+    does ``flag = true``; ``WHERE NOT flag`` keeps only FALSE, and so does
+    ``flag = false``. Three-valued logic makes the two pairs agree on every
+    input, which is what lets this be a projection and not a semantic change.
+
+    Args:
+        expression: A v2 expression that may be a bare boolean column or its
+            negation.
+
+    Returns:
+        The equivalent `ConstantFilter`, or None when *expression* is neither.
+
+    """
+
+    def boolean_column(node: V2FilterExpression) -> V2ColumnRef | None:
+        if not isinstance(node, V2ColumnRef):
+            return None
+        node_type = v2_expression_type(node)
+        # An unresolved column (no bind schema) is not known to be boolean, and
+        # guessing here would rewrite a non-boolean predicate into `= true`.
+        if node_type is None or not pa.types.is_boolean(node_type):
+            return None
+        return node
+
+    if ref := boolean_column(expression):
+        return ConstantFilter(ref.column_name, ref.column_index, ComparisonOp.EQ, pa.scalar(True))
+    if isinstance(expression, V2Not) and (ref := boolean_column(expression.expression)):
+        return ConstantFilter(ref.column_name, ref.column_index, ComparisonOp.EQ, pa.scalar(False))
+    return None
+
+
 def _v2_filter_view(expression: V2FilterExpression, context: EvaluationContext) -> Filter | None:
     """Project common v2 shapes into the longstanding convenience classes."""
     if isinstance(expression, V2Comparison):
@@ -1802,6 +1878,8 @@ def _v2_filter_view(expression: V2FilterExpression, context: EvaluationContext)
             }
             if op := mapping.get(expression.op):
                 return ConstantFilter(operands[0].column_name, operands[0].column_index, op, operands[1].value)
+    if boolean_leaf := _v2_boolean_column_view(expression):
+        return boolean_leaf
     if isinstance(expression, V2IsNull) and isinstance(expression.expression, V2ColumnRef):
         ref = expression.expression
         if expression.negated: