vgi-python 0.36.1__py3-none-any.whl → 0.36.2__py3-none-any.whl

vgi/_test_fixtures/table/__init__.py

@@ -65,6 +65,7 @@ from vgi._test_fixtures.table.catalog_scans import (
     ProjectsScanFunction,
 )
 from vgi._test_fixtures.table.filters import (
+    BoolFilterEchoFunction,
     DictFilterEchoFunction,
     DynamicFilterEchoFunction,
     ExpressionFilterTestFunction,
@@ -216,6 +217,7 @@ __all__ = [
     "ConstantColumnsFunction",
     "CountryPartitionedSalesFunction",
     "DepartmentsScanFunction",
+    "BoolFilterEchoFunction",
     "DictFilterEchoFunction",
     "DisjointRangePartitionedFunction",
     "DoubleSequenceFunction",

vgi/_test_fixtures/table/filters.py

@@ -191,6 +191,129 @@ class FilterEchoFunction(TableFunctionGenerator[FilterEchoFunctionArgs, FilterEc
         state.remaining -= size
 
 
+# ============================================================================
+# BoolFilterEchoFunction — a table function with a BOOLEAN column, so the
+# bare-column predicate shapes (`WHERE flag`, `WHERE NOT flag`) can be driven
+# end to end.
+#
+# filter_echo cannot cover these: its schema has no boolean column, so the
+# predicate cannot even be written. The table-in-out `echo` path cannot either
+# — DuckDB does not hand it a bare boolean column as a pushed predicate, so a
+# test written there passes whether or not the worker can decode one. It takes
+# a TABLE FUNCTION with a boolean column to see the shape at all, which is why
+# a decoder that refused every one of them ("predicate root must resolve to
+# BOOLEAN", vgi-python < 0.36.2) went unnoticed by a filter-pushdown suite of
+# 348 files including one dedicated to booleans.
+#
+# `pushed_filters` echoes the SQL rendering, so this covers both halves: the
+# decode (the rows are auto-applied and must be right) and `to_sql()` (a shape
+# that parses but renders nothing would show up here as "(none)").
+# ============================================================================
+
+
+@dataclass(slots=True, frozen=True)
+class _BoolFilterEchoArgs:
+    """Arguments for BoolFilterEchoFunction."""
+
+    count: Annotated[int, Arg(0, doc="Number of rows to generate", ge=0)]
+
+
+@dataclass(kw_only=True)
+class _BoolFilterEchoState(ArrowSerializableDataclass):
+    """Remaining rows plus the echoed SQL rendering of the pushed filters."""
+
+    remaining: int
+    current_index: int = 0
+    filter_str: str = "(none)"
+
+
+@init_single_worker
+@bind_fixed_schema
+@_cardinality_from_count
+class BoolFilterEchoFunction(TableFunctionGenerator[_BoolFilterEchoArgs, _BoolFilterEchoState]):
+    """Rows with a nullable BOOLEAN column, echoing the pushed-down filters.
+
+    SCHEMA
+    ------
+    Output: {"n": int64, "flag": bool, "pushed_filters": string}
+
+    ``flag`` cycles TRUE, FALSE, NULL so every predicate is exercised against
+    all three truth values -- the NULL row is what distinguishes ``WHERE flag``
+    from ``WHERE flag IS NOT FALSE``, and ``WHERE NOT flag`` from
+    ``WHERE flag IS NOT TRUE``.
+
+    Attributes:
+        FIXED_SCHEMA: The fixed Arrow output schema this function always produces.
+
+    """
+
+    class Meta:
+        """Metadata for BoolFilterEchoFunction."""
+
+        name = "bool_filter_echo"
+        description = "Rows with a nullable BOOLEAN column, echoing pushed-down filters"
+        categories = ["generator", "diagnostic"]
+        filter_pushdown = True
+        auto_apply_filters = True
+        projection_pushdown = True
+        examples = [
+            FunctionExample(
+                sql="SELECT * FROM bool_filter_echo(6) WHERE flag",
+                description="A boolean column is a predicate on its own",
+            ),
+        ]
+
+    FIXED_SCHEMA: ClassVar[pa.Schema] = schema({"n": pa.int64(), "flag": pa.bool_(), "pushed_filters": pa.utf8()})
+
+    @classmethod
+    def initial_state(cls, params: ProcessParams[_BoolFilterEchoArgs]) -> _BoolFilterEchoState:
+        """Create initial state, caching the SQL rendering of the pushed filters."""
+        assert params.init_call is not None
+        pf = params.init_call.pushdown_filters
+        filters = (
+            cls.pushdown_filters(
+                pf,
+                join_keys=params.init_call.join_keys,
+                output_schema=params.init_call.output_schema,
+            )
+            if pf is not None
+            else None
+        )
+        return _BoolFilterEchoState(
+            remaining=params.args.count,
+            filter_str=_format_pushed_filters(filters),
+        )
+
+    @classmethod
+    def process(
+        cls,
+        params: ProcessParams[_BoolFilterEchoArgs],
+        state: _BoolFilterEchoState,
+        out: OutputCollector,
+    ) -> None:
+        """Emit rows whose ``flag`` cycles TRUE, FALSE, NULL."""
+        if state.remaining <= 0:
+            out.finish()
+            return
+
+        n_values = list(range(state.current_index, state.current_index + state.remaining))
+        flags: list[bool | None] = [(True, False, None)[i % 3] for i in n_values]
+
+        out.emit(
+            pa.RecordBatch.from_pydict(
+                {
+                    "n": n_values,
+                    "flag": flags,
+                    "pushed_filters": [state.filter_str] * len(n_values),
+                },
+                schema=params.output_schema,
+            )
+        )
+
+        state.current_index += state.remaining
+        state.remaining = 0
+
+
 # ============================================================================
 # ValuePruneFunction — exercises PushdownFilters.get_column_values('n'), the
 # partition-pruning idiom (resolve the discrete value set up front, fetch only

vgi/_test_fixtures/worker.py

@@ -140,6 +140,7 @@ from vgi._test_fixtures.table import (
     RFF_SIMPLE_COLUMNS,
     RFF_STRUCT_COLUMNS,
     BatchIndexOverflowFunction,
+    BoolFilterEchoFunction,
     BrokenMissingPartitionValuesFunction,
     BrokenPartitionColumnAbsentFromBatchFunction,
     BrokenPartitionMinNeqMaxFunction,
@@ -486,6 +487,7 @@ _EXAMPLE_CATALOG = Catalog(
                 # TableFunctionGenerator - generate output without input
                 ConstantColumnsFunction,
                 SlowCancellableFunction,
+                BoolFilterEchoFunction,
                 FilterEchoFunction,
                 FilterEchoPartitionedFunction,
                 FilterEchoTableScanFunction,

vgi/filter_v2.py

@@ -740,11 +740,24 @@ class _Parser:
 
     @staticmethod
     def _is_boolean(expression: FilterExpression) -> bool:
-        if isinstance(expression, Literal):
-            return pa.types.is_boolean(expression.field.type)
-        if isinstance(expression, (Comparison, BooleanExpression, Not, IsNull, In, RuntimeFilter)):
-            return True
-        return isinstance(expression, Call)
+        """Whether *expression* resolves to BOOLEAN.
+
+        Ask the type, never the node kind. The two agree for every node whose
+        type is structurally fixed -- a comparison is BOOLEAN whatever its
+        operands -- and that is exactly why enumerating kinds looked sufficient.
+        It is not: ``column_ref``, ``field_ref``, ``cast`` and ``arithmetic``
+        are BOOLEAN or not depending on a type resolved from elsewhere (the
+        bind output schema, or a parent expression), and the schema admits all
+        four as a predicate root -- ``coreExpression`` lists ``columnRef``
+        first. Enumerating kinds rejected the lot, so the idiomatic
+        ``WHERE flag`` / ``WHERE NOT flag`` -- which DuckDB pushes down as a
+        bare column reference rather than rewriting to ``flag = true`` -- was
+        refused as "predicate root must resolve to BOOLEAN" even though the
+        parser had already resolved the column to BOOLEAN and stored it on the
+        node.
+        """
+        data_type = expression_type(expression)
+        return data_type is not None and pa.types.is_boolean(data_type)
 
     def _payload(self, prefix: str, ref: object) -> tuple[pa.Field[Any], pa.Scalar[Any]]:
         index = _uint(ref, f"{prefix}_ref")

vgi/table_filter_pushdown.py

@@ -63,6 +63,9 @@ from vgi.filter_v2 import (
 from vgi.filter_v2 import (
     Literal as V2Literal,
 )
+from vgi.filter_v2 import (
+    Not as V2Not,
+)
 from vgi.filter_v2 import (
     RuntimeFilter as V2RuntimeFilter,
 )
@@ -72,6 +75,9 @@ from vgi.filter_v2 import (
 from vgi.filter_v2 import (
     expression_columns as v2_expression_columns,
 )
+from vgi.filter_v2 import (
+    expression_type as v2_expression_type,
+)
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterator
@@ -207,6 +213,7 @@ __all__ = [
     "OrFilter",
     "StructFilter",
     "ExpressionFilter",
+    "FilterSQLUnsupportedError",
     "V2ExpressionFilter",
     # Expression node classes
     "ExpressionNode",
@@ -1638,6 +1645,22 @@ class PushdownFilters:
 # =============================================================================
 
 
+class FilterSQLUnsupportedError(ValueError):
+    """A filter that `to_sql()` cannot render as a SQL fragment.
+
+    Raised rather than skipped, and that is the whole point: DuckDB does **not**
+    re-apply a predicate it pushed into a table function, so a WHERE clause that
+    silently omits one does not return the same rows more slowly -- it returns
+    the wrong rows. Degrading quietly here would turn an unrenderable filter
+    into a wrong answer with no error anywhere.
+
+    A worker that catches this has two honest options: evaluate the filters in
+    memory instead (`PushdownFilters.apply()` / `evaluate()` handle every v2
+    expression), or decline the scan. Subclasses `ValueError` so existing
+    callers that catch the old error keep working.
+    """
+
+
 def _filter_to_sql(
     f: Filter,
     quote: Callable[[str], str],
@@ -1713,8 +1736,19 @@ def _filter_to_sql(
             # Constants are embedded directly in the SQL string.
             return expr.to_sql(f.column_name), []
 
+        case V2ExpressionFilter(expression=expression):
+            # A v2 expression the legacy projection could not reduce to a leaf
+            # shape (arithmetic, casts, extension calls, ...). It evaluates
+            # correctly -- `V2ExpressionFilter.evaluate()` runs the v2 reference
+            # evaluator -- there is simply no SQL rendering for it yet.
+            raise FilterSQLUnsupportedError(
+                f"filter on {f.column_name!r} is a v2 expression with no SQL rendering "
+                f"({type(expression).__name__}); evaluate it with PushdownFilters.apply() "
+                f"instead, or the scan will return rows the predicate excludes"
+            )
+
         case _:
-            raise ValueError(f"Unknown filter type: {type(f)}")
+            raise FilterSQLUnsupportedError(f"Unknown filter type: {type(f)}")
 
 
 # =============================================================================
@@ -1787,6 +1821,48 @@ def _pushdown_from_v2_state(state: V2FilterState) -> PushdownFilters:
     )
 
 
+def _v2_boolean_column_view(expression: V2FilterExpression) -> Filter | None:
+    """Project ``WHERE flag`` / ``WHERE NOT flag`` onto the legacy equality leaf.
+
+    DuckDB pushes a predicate that *is* a boolean column down as a bare
+    ``column_ref``, not as ``flag = true``. Left as a generic v2 expression it
+    becomes a `V2ExpressionFilter`, which evaluates fine but renders no SQL --
+    so every SQL-backed worker loses pushdown on the most ordinary boolean
+    predicate there is, and on the commonest shape of all, a conjunction with
+    one, it loses pushdown for the whole conjunction.
+
+    The rewrite is exact rather than approximate, including under NULLs:
+    ``WHERE flag`` keeps only TRUE (a NULL predicate is not satisfied), and so
+    does ``flag = true``; ``WHERE NOT flag`` keeps only FALSE, and so does
+    ``flag = false``. Three-valued logic makes the two pairs agree on every
+    input, which is what lets this be a projection and not a semantic change.
+
+    Args:
+        expression: A v2 expression that may be a bare boolean column or its
+            negation.
+
+    Returns:
+        The equivalent `ConstantFilter`, or None when *expression* is neither.
+
+    """
+
+    def boolean_column(node: V2FilterExpression) -> V2ColumnRef | None:
+        if not isinstance(node, V2ColumnRef):
+            return None
+        node_type = v2_expression_type(node)
+        # An unresolved column (no bind schema) is not known to be boolean, and
+        # guessing here would rewrite a non-boolean predicate into `= true`.
+        if node_type is None or not pa.types.is_boolean(node_type):
+            return None
+        return node
+
+    if ref := boolean_column(expression):
+        return ConstantFilter(ref.column_name, ref.column_index, ComparisonOp.EQ, pa.scalar(True))
+    if isinstance(expression, V2Not) and (ref := boolean_column(expression.expression)):
+        return ConstantFilter(ref.column_name, ref.column_index, ComparisonOp.EQ, pa.scalar(False))
+    return None
+
+
 def _v2_filter_view(expression: V2FilterExpression, context: EvaluationContext) -> Filter | None:
     """Project common v2 shapes into the longstanding convenience classes."""
     if isinstance(expression, V2Comparison):
@@ -1802,6 +1878,8 @@ def _v2_filter_view(expression: V2FilterExpression, context: EvaluationContext)
             }
             if op := mapping.get(expression.op):
                 return ConstantFilter(operands[0].column_name, operands[0].column_index, op, operands[1].value)
+    if boolean_leaf := _v2_boolean_column_view(expression):
+        return boolean_leaf
     if isinstance(expression, V2IsNull) and isinstance(expression.expression, V2ColumnRef):
         ref = expression.expression
         if expression.negated:

{vgi_python-0.36.1.dist-info → vgi_python-0.36.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vgi-python
-Version: 0.36.1
+Version: 0.36.2
 Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
 Project-URL: Homepage, https://query.farm
 Project-URL: Repository, https://github.com/Query-farm/vgi-python

{vgi_python-0.36.1.dist-info → vgi_python-0.36.2.dist-info}/RECORD

@@ -10,7 +10,7 @@ vgi/cache_control.py,sha256=IyBp9gZTCc4Y-4V7Rs05_o5lSEjbR2jt6cddXPU0dKw,7812
 vgi/copy_from_function.py,sha256=oroQe5qYzZTJ-ppzQTQ-OruEF1ZlwTBNLBoXYfN-P2o,7133
 vgi/copy_to_function.py,sha256=8l0v_DyEF1EnFwinStrZDQ-yNFfGJsv3Mr5o2jG30f0,9279
 vgi/exceptions.py,sha256=oX_sZc9xGWi7Xf8cJQf89fX19i3ocDEj_V_76GINgBQ,7294
-vgi/filter_v2.py,sha256=pSdmZF4HgSr9ZuM_mU3NMwUuAw0X4sZ7KonTMeGnjUw,56243
+vgi/filter_v2.py,sha256=FXuYOrm7uA-c0hYBXd5ScCRfjJmR5I9KfTT78e3o0vM,57089
 vgi/filter_v2_builder.py,sha256=8OmyzDI6GDdH4pYm2sJFMidzKusi0I8IGNipolPpjX0,5535
 vgi/function.py,sha256=KIfvKk1nzBvRDPXViHNX89tv6gvIMiDj4e8fl6ANqQg,11626
 vgi/function_storage.py,sha256=FjVwxc1BIjWo-82j8YE5qcU8w66sGjwyGsTmCUhn-Kw,67576
@@ -33,7 +33,7 @@ vgi/secret_service.py,sha256=r7qCzfnOMirf9wSCUe-mgFe3z6v9_cDRzixuSAZG_bs,8973
 vgi/serve.py,sha256=5UmvRAGcHj5RKCOUQMXG9RFVYAbLZsHwwk1qVI9fipY,63844
 vgi/split_token.py,sha256=tutcksj-hgWrLY7UW3sSqkyoaD-cMqNTOeff40MGl_g,10850
 vgi/table_buffering_function.py,sha256=O5TcIdqk8-YViCQCkb8bOevO-zhe3IoyzLgwXDO3Cjc,19531
-vgi/table_filter_pushdown.py,sha256=agzbDOz_-64E2uNZdDOQs8woP1CBAE_SU5rv2N4fqWk,63435
+vgi/table_filter_pushdown.py,sha256=4Qo4OAOLjTYQ5pinNTOjLCDnb96OHqpcxJlbSJo3cJU,67144
 vgi/table_function.py,sha256=8Pti3uFnsRaPCVG-CGYl9r3GHJF146Fp_62oXXwHKKw,73502
 vgi/table_in_out_function.py,sha256=F2QEtsbGEYHDVyQ4Cf9OCW7HlWXeqekAyx607foB-bs,19393
 vgi/worker.py,sha256=MGMQmy6PPKCqo5q_9fBS3z3zf4y6-UKdJAJiHGfnk50,249553
@@ -57,7 +57,7 @@ vgi/_test_fixtures/table_in_out_same_name.py,sha256=jdsQNdKn64T54W_TcLB-rhydI69Y
 vgi/_test_fixtures/twin_catalogs.py,sha256=UFsFuZ_nFpe62HatGnOMru36vk35Xt3Ta1Hkr1aWN9w,4450
 vgi/_test_fixtures/versioned.py,sha256=gnxNejc4cNGPKl_DxJk1SU8Ielj-rwmQYGkiShXzxK4,5786
 vgi/_test_fixtures/versioned_tables.py,sha256=_JHPe6LaiO0x2We28Ion8hh6pSATmHZYvAczB4uwBsA,22272
-vgi/_test_fixtures/worker.py,sha256=tAdCcL9x3LwgtY0OG56bapmBCwGZJ-M7paQY9hXjgB8,96440
+vgi/_test_fixtures/worker.py,sha256=ybrkhnYchWhbdMbUYKHHsnB6W0K5y9-QplyJSx2Bs7Y,96508
 vgi/_test_fixtures/accumulate/__init__.py,sha256=4hYT8jqRoVHSjV9TB7v0Z1CMJtdLuPaDWSz4J2fvMDs,868
 vgi/_test_fixtures/accumulate/worker.py,sha256=4a2VaCoOIXPEoJpkNume7FoBlQzs9kRl8b-_ZJMHvnQ,30165
 vgi/_test_fixtures/aggregate/__init__.py,sha256=2GCp80FNrsWFOA2SV3lCffzLjoN5AKwrf57t2YRg8O0,2393
@@ -90,13 +90,13 @@ vgi/_test_fixtures/scalar/settings_secrets.py,sha256=ZOGkGY0CoD3ous_ij0--ss5byB4
 vgi/_test_fixtures/scalar/type_info.py,sha256=2WeTxakT-_tcWybPfkCrAHVAMOadFN3tb8E3_EmqIyw,6304
 vgi/_test_fixtures/schema_reconcile/__init__.py,sha256=rCCtM5bd67-PTPeIYg9SCJaKUSglA6YeXsedQBEUlmA,1324
 vgi/_test_fixtures/schema_reconcile/worker.py,sha256=_r-cCQC3r48F0yTEdZC5zQYN_vuCx1HIkoP8_idBJHk,23680
-vgi/_test_fixtures/table/__init__.py,sha256=qE3gpMgrbeersJ8Lw-tX-ivYR1nYeqm4gZSrEjnDzF4,9887
+vgi/_test_fixtures/table/__init__.py,sha256=qwTsNJLlsWgD4wEJav8e6pU1MQl8jV8Gvw6wR0v9crM,9945
 vgi/_test_fixtures/table/_common.py,sha256=m_99LRrDlULKGg8W8gJQQ29-_RxFINFlCMi3JtM0Y08,7753
 vgi/_test_fixtures/table/batch_index.py,sha256=P5ds0xgikuEQanSEWVWKMLbdvIzUeJraI-GuSoPdb6U,11641
 vgi/_test_fixtures/table/batch_index_broken.py,sha256=kZOGrLL7ZW1rmwPmNEYRmiF_vqIfHsfXioq5vKPWHk0,7314
 vgi/_test_fixtures/table/cache.py,sha256=7oohKpFJ_si7LoKE7QINkz5oIyj1sx8Kiobpwo4Sn7w,72322
 vgi/_test_fixtures/table/catalog_scans.py,sha256=5j1Sx02-HWK7bFurDu4e9HiS3Q9BmBukA3sAErH4GHE,5080
-vgi/_test_fixtures/table/filters.py,sha256=pOvuBFr0Rj7PXUWILUdfGf735sJda-iphJ_aQyiw_VY,42197
+vgi/_test_fixtures/table/filters.py,sha256=1FCD_rTs8ssen7QZ_J3p0izKO09jzWEJ3j5L37sqkc4,46725
 vgi/_test_fixtures/table/late_materialization.py,sha256=BT1FOpQNeP4IXBS3hUvAmO-5jscl5iiQ11fvxUfN6DM,10355
 vgi/_test_fixtures/table/make_series.py,sha256=K-G_YNq25Kb7I5bp6XK4rCZzwMYNTxgKH3M_O-AQAlA,8812
 vgi/_test_fixtures/table/misc.py,sha256=71WOIFqk5ntnEIqsG-57rZ9DY7ShQqMKHi7yluNAlM4,16250
@@ -152,8 +152,8 @@ vgi/transactor/_duckdb_compat.py,sha256=sXVZ9JLKAQyGR1BjWczSwdQEavtr-TcZPoVZZnTr
 vgi/transactor/client.py,sha256=7DTeMksogsw6ANjQjGOPpKYrV76rg4_kGjktMJf54jg,4486
 vgi/transactor/protocol.py,sha256=v59IvrKnuvwOXvn_HEcBAbCzDVcx0akgKN0R1mChXg0,5034
 vgi/transactor/server.py,sha256=nzsZQxJZdgappdYX8okrFIrjFtUbdM_Hhskl1fZ2nDY,32553
-vgi_python-0.36.1.dist-info/METADATA,sha256=KiNZHizy9p1n8zOX0mFjOPot5qz_HCfD-qmipkpSBdc,25617
-vgi_python-0.36.1.dist-info/WHEEL,sha256=THafob7ofN-NsuMN7Mg4qZyHaQI7KkD-QlcQatYhXPo,87
-vgi_python-0.36.1.dist-info/entry_points.txt,sha256=3Kz1vgodw3pOL_xjtSyDB55-ZRy-U2X-X_Bdr582x0Q,165
-vgi_python-0.36.1.dist-info/licenses/LICENSE,sha256=pbJb4zZasP6n5ifEV81wFu017TarjydaYVmGbHcehtY,6103
-vgi_python-0.36.1.dist-info/RECORD,,
+vgi_python-0.36.2.dist-info/METADATA,sha256=kwE2GNB9zbRBOopkbMvkpJoV6dW7sbz8p6f5CGUtU-s,25617
+vgi_python-0.36.2.dist-info/WHEEL,sha256=THafob7ofN-NsuMN7Mg4qZyHaQI7KkD-QlcQatYhXPo,87
+vgi_python-0.36.2.dist-info/entry_points.txt,sha256=3Kz1vgodw3pOL_xjtSyDB55-ZRy-U2X-X_Bdr582x0Q,165
+vgi_python-0.36.2.dist-info/licenses/LICENSE,sha256=pbJb4zZasP6n5ifEV81wFu017TarjydaYVmGbHcehtY,6103
+vgi_python-0.36.2.dist-info/RECORD,,