xlr8 0.1.7b2__cp313-cp313-macosx_10_12_x86_64.whl

xlr8/__init__.py

@@ -0,0 +1,113 @@
+# XLR8: High-Performance MongoDB Acceleration Layer
+
+"""
+XLR8 - Accelerate MongoDB analytical queries with Parquet caching.
+
+XLR8 is a high-performance wrapper for PyMongo that:
+- Decomposes queries into brackets for parallel execution
+- Executes parallel async MongoDB fetches with Rust backend
+- Caches results in compressed Parquet files
+- Reconstructs pandas/Polars DataFrames efficiently using Rust backend.
+
+Quick Start:
+```python
+from pymongo import MongoClient
+from xlr8 import accelerate, Schema, Types
+
+# Define your schema with time field and typed fields
+schema = Schema(
+    time_field="recordedAt",
+    fields={
+        "recordedAt": Types.Timestamp("ms", tz="UTC"),
+        "metadata.device_id": Types.ObjectId(),
+        "metadata.sensor_id": Types.ObjectId(),
+        "value": Types.Any(),  # Polymorphic - int, float, str, etc.
+    },
+    avg_doc_size_bytes=250,
+)
+
+# Connect to MongoDB and get collection
+client = MongoClient("mongodb://localhost:27017")
+collection = client["main"]["sensorData"]
+
+# Wrap collection with schema for acceleration
+xlr8_collection = accelerate(
+    collection,
+    schema=schema,
+    mongo_uri="mongodb://localhost:27017"
+)
+
+# Use like normal PyMongo - find() returns XLR8Cursor
+cursor = xlr8_collection.find({
+    "recordedAt": {"$gte": start_date, "$lt": end_date}
+}).sort("recordedAt", 1)
+
+# Accelerated DataFrame construction
+df = cursor.to_dataframe()
+
+# Clean up
+client.close()
+```
+"""
+
+from __future__ import annotations
+
+from importlib.metadata import version as _get_version
+from typing import TYPE_CHECKING
+
+from .schema import types as Types
+from .schema.schema import Schema
+
+try:
+    __version__ = _get_version("xlr8")
+except Exception:
+    __version__ = "0.0.0.dev"  # Fallback for editable installs without metadata
+
+# Lazy loader for rust_backend to avoid import errors when Rust isn't built yet
+# This allows `from xlr8 import rust_backend` to work without importing at module load
+_rust_backend_cached = None
+_collection_exports_cached: dict[str, object] | None = None
+
+
+if TYPE_CHECKING:
+    from .collection.cursor import XLR8Cursor as XLR8Cursor
+    from .collection.wrapper import XLR8Collection as XLR8Collection
+    from .collection.wrapper import accelerate as accelerate
+
+
+def __getattr__(name: str):
+    global _rust_backend_cached
+    global _collection_exports_cached
+    if name == "rust_backend":
+        if _rust_backend_cached is None:
+            # Import the module directly to avoid recursion
+            import importlib
+
+            _rust_backend_cached = importlib.import_module(
+                ".rust_backend", package="xlr8"
+            )
+        return _rust_backend_cached
+
+    if name in {"XLR8Cursor", "XLR8Collection", "accelerate"}:
+        if _collection_exports_cached is None:
+            from .collection.cursor import XLR8Cursor
+            from .collection.wrapper import XLR8Collection, accelerate
+
+            _collection_exports_cached = {
+                "XLR8Cursor": XLR8Cursor,
+                "XLR8Collection": XLR8Collection,
+                "accelerate": accelerate,
+            }
+        return _collection_exports_cached[name]
+
+    raise AttributeError(f"module 'xlr8' has no attribute '{name}'")
+
+
+__all__ = [
+    "Schema",
+    "Types",
+    "rust_backend",
+    "accelerate",
+    "XLR8Collection",
+    "XLR8Cursor",
+]

xlr8/_xlr8_rust.pyi

@@ -0,0 +1,71 @@
+"""Type stubs for _xlr8_rust native module.
+
+This is the compiled Rust extension. Type hints enable intellisense.
+"""
+
+from typing import Any, Dict, List
+
+def fetch_chunks_bson(
+    mongodb_uri: str,
+    db_name: str,
+    collection_name: str,
+    chunks_bson: bytes,
+    schema_json: str,
+    cache_dir: str,
+    num_workers: int,
+    batch_size: int,
+    flush_trigger_mb: int,
+    avg_doc_size_bytes: int,
+    sort_spec_json: str,
+    time_field: str,
+    projection_json: str,
+    row_group_size: int | None = None,
+) -> Dict[str, Any]:
+    """Fetch MongoDB documents in parallel chunks and write to Parquet.
+
+    Args:
+        mongodb_uri: MongoDB connection string
+        db_name: Database name
+        collection_name: Collection name
+        chunks_bson: BSON-encoded chunk definitions
+        schema_json: JSON string describing Arrow schema
+        cache_dir: Directory where Parquet files will be written
+        num_workers: Number of parallel workers
+        batch_size: Documents per MongoDB batch
+        flush_trigger_mb: Memory threshold for flushing to disk (MB)
+        avg_doc_size_bytes: Average document size for memory estimation
+        sort_spec_json: JSON sort specification
+        time_field: Field name containing timestamps
+        projection_json: MongoDB projection as JSON
+        row_group_size: Parquet row group size (None = use Arrow default)
+
+    Returns:
+        Dictionary with total_docs, total_files, duration_secs
+    """
+    ...
+
+def decode_any_struct_arrow(
+    arrow_array: Any,  # pyarrow.StructArray
+) -> List[Any]:
+    """Decode PyArrow StructArray (Any type) to Python values.
+
+    Args:
+        arrow_array: PyArrow StructArray with 13-field Any encoding
+
+    Returns:
+        List of decoded Python values
+    """
+    ...
+
+def encode_any_values_to_arrow(
+    values: List[Any],
+) -> Any:  # pyarrow.StructArray
+    """Encode Python values to PyArrow StructArray (Any type).
+
+    Args:
+        values: List of Python values to encode
+
+    Returns:
+        PyArrow StructArray with 13-field Any encoding
+    """
+    ...

xlr8/analysis/__init__.py

@@ -0,0 +1,58 @@
+"""
+Query analysis and execution planning.
+
+This module provides utilities for analyzing MongoDB queries and
+creating optimal execution plans for parallel processing.
+"""
+
+from .brackets import (
+    build_brackets_for_find,
+)
+from .chunker import (
+    chunk_time_range,
+)
+from .inspector import (
+    # Operator classification sets
+    ALWAYS_ALLOWED,
+    CONDITIONAL,
+    NEVER_ALLOWED,
+    # Validation
+    ValidationResult,
+    check_conditional_operators,
+    extract_time_bounds_recursive,
+    generate_sort_sql,
+    get_sort_field_info,
+    has_forbidden_ops,
+    is_chunkable_query,
+    normalize_datetime,
+    # Query analysis
+    or_depth,
+    split_global_and,
+    validate_query_for_chunking,
+    validate_sort_field,
+)
+
+__all__ = [
+    # inspector - operator sets
+    "ALWAYS_ALLOWED",
+    "CONDITIONAL",
+    "NEVER_ALLOWED",
+    # inspector - validation
+    "ValidationResult",
+    "has_forbidden_ops",
+    "validate_query_for_chunking",
+    "validate_sort_field",
+    "get_sort_field_info",
+    "generate_sort_sql",
+    "check_conditional_operators",
+    # inspector - analysis
+    "or_depth",
+    "split_global_and",
+    "normalize_datetime",
+    "extract_time_bounds_recursive",
+    "is_chunkable_query",
+    # brackets
+    "build_brackets_for_find",
+    # chunker
+    "chunk_time_range",
+]