opteryx-catalog 0.4.13__py3-none-any.whl

opteryx_catalog-0.4.13.dist-info/licenses/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2025 Justin Joyce (@joocer)
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

opteryx_catalog-0.4.13.dist-info/top_level.txt

@@ -0,0 +1,3 @@
+opteryx_catalog
+scripts
+tests

scripts/create_dataset.py

@@ -0,0 +1,201 @@
+"""Create test tables using the `FirestoreCatalog` and write Parquet-only
+manifests/files (no Avro). This script creates a small deterministic set of
+tables under `tests_temp`, writes a single Parquet data file per table, writes
+a Parquet manifest for each snapshot, and records a minimal snapshot document
+in Firestore so planners can discover the Parquet manifest.
+
+Run locally with valid GCP credentials set in `GOOGLE_APPLICATION_CREDENTIALS`.
+"""
+
+import os
+import sys
+import time
+import traceback
+
+import pyarrow as pa
+
+from opteryx_catalog import OpteryxCatalog
+
+sys.path.insert(0, os.path.join(sys.path[0], ".."))  # Add parent dir for pyiceberg_firestore_gcs
+sys.path.insert(1, os.path.join(sys.path[0], "../opteryx-core"))
+
+
+def create_parquet_only_tables(count: int = 2) -> list:
+    workspace = os.environ.get("OPTERYX_WORKSPACE", "opteryx")
+    catalog = OpteryxCatalog(
+        workspace=workspace,
+        firestore_project=os.environ.get("GCP_PROJECT_ID"),
+        firestore_database=os.environ.get("FIRESTORE_DATABASE"),
+        gcs_bucket=os.environ.get("GCS_BUCKET"),
+    )
+
+    collection = "tests_temp"
+    created = []
+
+    for i in range(count):
+        table_name = f"test_table_{i}_{int(time.time())}"
+        # location = f"gs://{os.environ.get('GCS_BUCKET')}/{workspace}/{collection}/{table_name}"
+
+        # Use fixed test author
+        author = "me"
+
+        # Create dataset metadata (create_collection is handled inside create_dataset)
+        try:
+            tbl = catalog.create_dataset(
+                f"{collection}.{table_name}",
+                pa.schema([pa.field("id", pa.int64()), pa.field("name", pa.string())]),
+                author=author,
+            )
+            print("Created dataset metadata:", f"{collection}.{table_name}")
+        except Exception:
+            tbl = catalog.load_dataset(f"{collection}.{table_name}")
+            print("Loaded existing dataset metadata:", f"{collection}.{table_name}")
+
+        # Add an example sort order to the table metadata and persist it so
+        # we can inspect `sort-orders` in Firestore.
+        try:
+            tbl.metadata.sort_orders = [
+                {"order-id": 1, "fields": [{"name": "id", "direction": "asc"}]}
+            ]
+            if hasattr(catalog, "save_dataset_metadata"):
+                catalog.save_dataset_metadata(f"{collection}.{table_name}", tbl.metadata)
+        except Exception:
+            pass
+
+        # Create a deterministic planets dataset so read_table tests can validate
+        planets = [
+            (1, "Mercury"),
+            (2, "Venus"),
+            (3, "Earth"),
+            (4, "Mars"),
+            (5, "Jupiter"),
+            (6, "Saturn"),
+            (7, "Uranus"),
+            (8, "Neptune"),
+        ]
+
+        data = pa.table({"id": [p[0] for p in planets], "name": [p[1] for p in planets]})
+
+        # Append data using the new SimpleDataset.append() which writes the data
+        # file, creates a Parquet manifest and persists snapshot metadata.
+        try:
+            tbl.append(data, author=author)
+            print("Appended data via Dataset.append() for", f"{collection}.{table_name}")
+        except Exception as e:
+            print("Dataset.append() failed:", e)
+            raise
+
+        # Also append a second dataset (an edit) so we have multiple snapshots/files
+        try:
+            # small additional rows to simulate an update
+            extra = [(9, "Pluto"), (10, "Eris")]
+            data2 = pa.table({"id": [p[0] for p in extra], "name": [p[1] for p in extra]})
+            tbl.append(data2, author=author, commit_message="append extra planets")
+            print("Appended second dataset via Dataset.append() for", f"{collection}.{table_name}")
+        except Exception as e:
+            print("Second Dataset.append() failed:", e)
+            raise
+
+        # Inspect snapshot and Firestore snapshot doc for parquet-manifest
+        parquet_manifest_path = None
+        try:
+            snapshot = tbl.snapshot()
+            snapshot_id = snapshot.snapshot_id if snapshot else None
+            print("Current snapshot id:", snapshot_id)
+            if snapshot_id is not None:
+                from google.cloud import firestore
+
+                db = firestore.Client(
+                    project=os.environ.get("GCP_PROJECT_ID"),
+                    database=os.environ.get("FIRESTORE_DATABASE"),
+                )
+                snap_doc = (
+                    db.collection(workspace)
+                    .document(collection)
+                    .collection("datasets")
+                    .document(table_name)
+                    .collection("snapshots")
+                    .document(str(snapshot_id))
+                    .get()
+                )
+                if snap_doc.exists:
+                    d = snap_doc.to_dict() or {}
+                    parquet_manifest_path = d.get("manifest")
+                    print("Snapshot doc keys:", list(d.keys()))
+                    print("manifest:", parquet_manifest_path)
+                else:
+                    print("Snapshot document not found in Firestore for", snapshot_id)
+        except Exception as e:
+            print("Failed to read snapshot/Firestore doc:", e)
+
+        created.append(
+            {
+                "collection": collection,
+                "dataset": table_name,
+                "location": tbl.metadata.location,
+                "manifest": parquet_manifest_path,
+            }
+        )
+
+        # Create a simple view document in Firestore under `views` subcollection
+        try:
+            from google.cloud import firestore
+
+            db = firestore.Client(
+                project=os.environ.get("GCP_PROJECT_ID"),
+                database=os.environ.get("FIRESTORE_DATABASE"),
+            )
+            view_name = f"view_{table_name}"
+            view_doc_ref = (
+                db.collection(workspace)
+                .document(collection)
+                .collection("views")
+                .document(view_name)
+            )
+            now_ms = int(time.time() * 1000)
+            author = "me"
+            # Include optional last-execution metrics (populated later by the view runner)
+            # Store the SQL text in a `statement` subcollection so we can version/track changes
+            statement_id = str(now_ms)
+            statement_coll = view_doc_ref.collection("statement")
+            statement_coll.document(statement_id).set(
+                {
+                    "sql": f"SELECT * FROM {workspace}.{collection}.{table_name}",
+                    "timestamp-ms": now_ms,
+                    "author": author,
+                    "sequence-number": 1,
+                }
+            )
+
+            # Root view doc references the statement doc via `statement-id` (no inline query)
+            view_doc_ref.set(
+                {
+                    "name": view_name,
+                    "collection": collection,
+                    "workspace": workspace,
+                    "timestamp-ms": now_ms,
+                    "author": author,
+                    "description": f"View over {table_name}",
+                    "describer": author,
+                    "last-execution-ms": None,
+                    "last-execution-data-size": None,
+                    "last-execution-records": None,
+                    "statement-id": statement_id,
+                }
+            )
+            print("Created view doc:", f"{collection}.{view_name}")
+        except Exception as e:
+            print("Failed to create view doc:", e)
+
+    return created
+
+
+if __name__ == "__main__":
+    try:
+        created = create_parquet_only_tables(2)
+        print("\nCreated datasets summary:")
+        for c in created:
+            print(c)
+    except Exception:
+        traceback.print_exc()
+        raise

scripts/read_dataset.py

@@ -0,0 +1,268 @@
+import datetime
+import os
+import sys
+import time
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from opteryx_catalog.opteryx_catalog import OpteryxCatalog
+
+# Add local paths to sys.path to use local code instead of installed packages
+sys.path.insert(0, os.path.join(sys.path[0], ".."))  # Add parent dir for pyiceberg_firestore_gcs
+sys.path.insert(1, os.path.join(sys.path[0], "../opteryx-core"))
+sys.path.insert(1, os.path.join(sys.path[0], "../pyiceberg-firestore-gcs"))
+
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = (
+    "/Users/justin/Nextcloud/mabel/mabeldev-b37f651c2916.json"
+)
+os.environ["GCP_PROJECT_ID"] = "mabeldev"
+os.environ["FIRESTORE_DATABASE"] = "catalogs"
+os.environ["GCS_BUCKET"] = "opteryx_data"
+
+
+# from opteryx.connectors.iceberg_connector import IcebergConnector
+# Using opteryx_catalog for manifest discovery + simple in-Python filters
+
+workspace = "opteryx"
+collection_name = "tests_temp"
+
+
+# Step 1: Create a local catalog
+catalog = OpteryxCatalog(
+    workspace,
+    firestore_project="mabeldev",
+    firestore_database="catalogs",
+    gcs_bucket="opteryx_data",
+)
+
+# Choose the latest dataset in the collection to test against
+datasets = list(catalog.list_datasets(collection_name))
+if not datasets:
+    raise RuntimeError(f"No datasets found in collection {collection_name}")
+datasets = sorted(datasets)
+table = datasets[-1]
+
+table = "test_table_0_1767300842"
+
+
+# opteryx.register_store(
+#    prefix="_default",
+#    connector=IcebergConnector,
+#    remove_prefix=True,
+#    catalog=FirestoreCatalog,
+#    firestore_project="mabeldev",
+#    firestore_database="catalogs",
+#    gcs_bucket="opteryx_data",
+# )
+
+# catalog.create_collection_if_not_exists(schema_name, properties={"iceberg_compatible": "false"})
+
+# df = opteryx.query_to_arrow("SELECT * FROM $planets")
+
+# Drop table if it exists
+# try:
+#    catalog.drop_dataset(f"{schema_name}.{table}")
+# except Exception:
+#    pass
+
+# s = catalog.create_dataset(f"{schema_name}.{table}", df.schema, properties={"iceberg_compatible": "false"})
+
+# Load table metadata using the new catalog
+# Attempt to load the requested table; if it has no manifest, fall back to
+# the most recently created table in the namespace (useful when running the
+# paired `create_dataset.py` script which creates new deterministic datasets).
+s = catalog.load_dataset(f"{collection_name}.{table}")
+
+
+def _read_parquet_manifest(io, manifest_path: str) -> list:
+    """Read a Parquet manifest produced by `FirestoreCatalog.write_parquet_manifest`.
+
+    Returns a list of entry dicts (the original pyarrow.from_pylist rows).
+    """
+    if not manifest_path:
+        return []
+    inp = io.new_input(manifest_path)
+    try:
+        buf = inp.open().read()
+    except FileNotFoundError:
+        return []
+    tbl = pq.read_table(pa.BufferReader(buf))
+    return tbl.to_pylist()
+
+
+def _read_data_file(io, path: str) -> list:
+    inp = io.new_input(path)
+    buf = inp.open().read()
+    tbl = pq.read_table(pa.BufferReader(buf))
+    return tbl.to_pylist()
+
+
+# s.append(df)
+
+ts = s.snapshots()[-1].timestamp_ms
+print(f"Table last updated: {datetime.datetime.fromtimestamp(ts / 1000)}")
+ts = s.snapshots()[0].timestamp_ms
+print(f"Table last created: {datetime.datetime.fromtimestamp(ts / 1000)}")
+
+
+print(f"Table format version: {s.metadata.format_version}")
+print(f"Table location: {s.metadata.location}")
+
+# Discover planned files from the Parquet manifest
+print("\n=== Test 1: No filter (baseline) ===")
+entries = []
+# Read manifests from all snapshots so edits (additional snapshots) are included
+for snap in s.metadata.snapshots:
+    if snap.manifest_list:
+        entries.extend(_read_parquet_manifest(catalog.io, snap.manifest_list))
+if not entries:
+    # Try the most recently listed dataset in the namespace
+    try:
+        datasets = list(catalog.list_datasets(collection_name))
+        if datasets:
+            last = datasets[-1]
+            print(f"Falling back to dataset: {last}")
+            s = catalog.load_dataset(f"{collection_name}.{last}")
+            # read manifests from all snapshots
+            entries = []
+            for snap in s.metadata.snapshots:
+                if snap.manifest_list:
+                    entries.extend(_read_parquet_manifest(catalog.io, snap.manifest_list))
+    except Exception:
+        pass
+
+files = []
+seen = set()
+for e in entries:
+    fp = e.get("file_path")
+    if fp and fp not in seen:
+        seen.add(fp)
+        files.append(fp)
+print(f"✓ Planned {len(files)} files (from manifest)")
+assert len(files) > 0, "Should have at least 1 file without filter"
+baseline_file_count = len(files)
+
+# Test 2: EqualTo filter on 'name' = 'Earth'
+print("\n=== Test 2: EqualTo filter (name = 'Earth') ===")
+t = time.monotonic_ns()
+files_eq = files
+print(f"✓ Planned {len(files_eq)} files (from manifest)")
+# Read and verify data by scanning files and applying the filter in Python
+rows_eq = []
+for fpath in files_eq:
+    rows = _read_data_file(catalog.io, fpath)
+    for r in rows:
+        if r.get("name") == "Earth":
+            rows_eq.append(r)
+print(f"  Found {len(rows_eq)} rows")
+assert len(rows_eq) == 1, f"Expected 1 row for Earth, got {len(rows_eq)}"
+assert rows_eq[0]["name"] == "Earth", f"Expected Earth, got {rows_eq[0]['name']}"
+print("  ✓ Verified: Only 'Earth' returned")
+
+
+# Test 2a: EqualTo filter on 'name' = 'Xenon'
+print("\n=== Test 2a: EqualTo filter (name = 'Xenon') ===")
+t = time.monotonic_ns()
+rows_eq = []
+for fpath in files:
+    rows = _read_data_file(catalog.io, fpath)
+    for r in rows:
+        if r.get("name") == "Xenon":
+            rows_eq.append(r)
+print(f"  Found {len(rows_eq)} rows")
+assert len(rows_eq) == 0, f"Expected 0 rows for Xenon, got {len(rows_eq)}"
+print("  ✓ Verified: No rows returned")
+
+# Test 3: In filter with multiple values
+print("\n=== Test 3: In filter (name IN ['Earth', 'Mars']) ===")
+t = time.monotonic_ns()
+rows_in = []
+for fpath in files:
+    rows = _read_data_file(catalog.io, fpath)
+    for r in rows:
+        if r.get("name") in {"Earth", "Mars"}:
+            rows_in.append(r)
+print(f"  Found {len(rows_in)} rows")
+assert len(rows_in) == 2, f"Expected 2 rows for Earth and Mars, got {len(rows_in)}"
+names = {row["name"] for row in rows_in}
+assert names == {"Earth", "Mars"}, f"Expected Earth and Mars, got {names}"
+print("  ✓ Verified: Only 'Earth' and 'Mars' returned")
+
+# Test 4: In filter that should return no rows
+print("\n=== Test 4: In filter (name IN ['NonExistent1', 'NonExistent2']) ===")
+t = time.monotonic_ns()
+rows_empty = []
+for fpath in files:
+    rows = _read_data_file(catalog.io, fpath)
+    for r in rows:
+        if r.get("name") in {"NonExistent1", "NonExistent2"}:
+            rows_empty.append(r)
+print(f"  Found {len(rows_empty)} rows")
+assert len(rows_empty) == 0, f"Expected 0 rows for non-existent names, got {len(rows_empty)}"
+print("  ✓ Verified: No rows returned (filter works at row level)")
+if baseline_file_count == len(files):
+    print(f"  ⚠ Note: File pruning not working - still scanned {len(files)} files (expected 0)")
+
+# Test 5: EqualTo on a different value
+print("\n=== Test 5: EqualTo filter (name = 'Jupiter') ===")
+t = time.monotonic_ns()
+rows_jupiter = []
+for fpath in files:
+    rows = _read_data_file(catalog.io, fpath)
+    for r in rows:
+        if r.get("name") == "Jupiter":
+            rows_jupiter.append(r)
+print(f"  Found {len(rows_jupiter)} rows")
+assert len(rows_jupiter) == 1, f"Expected 1 row for Jupiter, got {len(rows_jupiter)}"
+assert rows_jupiter[0]["name"] == "Jupiter", f"Expected Jupiter, got {rows_jupiter[0]['name']}"
+print("  ✓ Verified: Only 'Jupiter' returned")
+
+# Test 6: In filter with single value (should behave like EqualTo)
+print("\n=== Test 6: In filter (name IN ['Venus']) ===")
+t = time.monotonic_ns()
+rows_single = []
+for fpath in files:
+    rows = _read_data_file(catalog.io, fpath)
+    for r in rows:
+        if r.get("name") == "Venus":
+            rows_single.append(r)
+print(f"  Found {len(rows_single)} rows")
+assert len(rows_single) == 1, f"Expected 1 row for Venus, got {len(rows_single)}"
+assert rows_single[0]["name"] == "Venus", f"Expected Venus, got {rows_single[0]['name']}"
+print("  ✓ Verified: Only 'Venus' returned")
+
+# Test 7: In filter with all planets
+print("\n=== Test 7: In filter with all planet names ===")
+all_planets = ["Mercury", "Venus", "Earth", "Mars", "Jupiter", "Saturn", "Uranus", "Neptune"]
+t = time.monotonic_ns()
+rows_all = []
+for fpath in files:
+    rows = _read_data_file(catalog.io, fpath)
+    rows_all.extend(rows)
+print(f"  Found {len(rows_all)} rows")
+assert len(rows_all) == 10, f"Expected 10 rows for all planets, got {len(rows_all)}"
+print("  ✓ Verified: All rows returned (including appended rows)")
+
+# Test 8: Empty In filter
+print("\n=== Test 8: In filter with empty list (name IN []) ===")
+t = time.monotonic_ns()
+rows_empty_list = []
+for fpath in files:
+    rows = _read_data_file(catalog.io, fpath)
+    for r in rows:
+        if r.get("name") in []:
+            rows_empty_list.append(r)
+print(f"  Found {len(rows_empty_list)} rows")
+assert len(rows_empty_list) == 0, f"Expected 0 rows for empty IN list, got {len(rows_empty_list)}"
+print("  ✓ Verified: No rows returned for empty IN list")
+
+print("\n" + "=" * 60)
+print("SUMMARY OF FILTER PUSHDOWN TESTS")
+print("=" * 60)
+print("✅ All filter checks passed (performed in-Python against data files)")
+print("✅ Filters work correctly at row level")
+print(f"File counts: baseline={baseline_file_count}, planned_files={len(files)}")
+print("⚠️  File-level pruning not implemented in this test (manifest-driven planning)")
+print("=" * 60)