vgi-python 0.8.0__py3-none-any.whl

vgi/exceptions.py

@@ -0,0 +1,205 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""Exception classes for VGI.
+
+This module defines custom exceptions used throughout the VGI framework.
+
+Classes:
+    InitIdentifierError: Raised when execution_identifier is required but not set.
+    SchemaValidationError: Raised when a batch schema doesn't match expected schema.
+
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import pyarrow as pa
+
+__all__ = [
+    "BindStateNotFoundError",
+    "CatalogReadOnlyError",
+    "ExecutionIdentifierError",
+    "SchemaValidationError",
+]
+
+
+class BindStateNotFoundError(Exception):
+    """Raised when init is called with invalid or missing bind_data.
+
+    This exception is raised when:
+    - INIT invocation is missing bind_data field
+    - The bind_data is corrupted or cannot be deserialized
+    - The function_name in bind state doesn't match the invocation
+
+    The client should catch this and provide a clear error message indicating
+    that a BIND call must be made before INIT.
+
+    """
+
+
+class CatalogReadOnlyError(Exception):
+    """Raised when a DDL operation is attempted on a read-only catalog.
+
+    This exception is raised by ReadOnlyCatalogInterface when any
+    create, drop, rename, or modify operation is attempted.
+
+    Read-only catalogs only support:
+    - catalogs() - list catalogs
+    - catalog_attach/detach - attach to/detach from catalogs
+    - schemas() - list schemas
+    - schema_get() - get schema info
+    - schema_contents() - list schema contents
+    - table_get(), view_get() - get table/view info
+    - table_scan_function_get() - get scan function for tables
+
+    """
+
+
+class ExecutionIdentifierError(ValueError):
+    """Raised when an operation requires an execution_identifier that hasn't been set.
+
+    This typically occurs when:
+    - store_state() is called before initialize_global_state() or load_global_state()
+    - collect_states() is called before initialize_global_state() or load_global_state()
+    - Work queue operations are attempted before initialization
+
+    The execution_identifier is automatically set during:
+    - initialize_global_state() for the primary worker
+    - load_global_state() for secondary workers
+
+    Resolution:
+    - Ensure your function calls super().initialize_global_state()
+    - Ensure the worker correctly calls load_global_state() for secondary workers
+
+    """
+
+
+class SchemaValidationError(Exception):
+    """Raised when a batch schema doesn't match the expected schema.
+
+    This error is raised by the framework during input/output validation.
+    It indicates a programming error where a batch doesn't conform to the
+    declared schema.
+
+    The error message includes detailed information about what differs:
+    - Missing fields (in expected but not in actual)
+    - Extra fields (in actual but not in expected)
+    - Type mismatches (same field name, different types)
+    - Field order differences
+
+    Attributes:
+        expected: The expected Arrow schema.
+        actual: The actual Arrow schema that was received.
+        context: Description of where the validation occurred.
+
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        expected: pa.Schema | None = None,
+        actual: pa.Schema | None = None,
+        context: str = "",
+    ) -> None:
+        """Initialize with schema comparison details.
+
+        Args:
+            message: Base error message.
+            expected: The expected Arrow schema.
+            actual: The actual Arrow schema.
+            context: Where the error occurred (e.g., "output from transform()").
+
+        """
+        self.expected = expected
+        self.actual = actual
+        self.context = context
+
+        if expected is not None and actual is not None:
+            full_message = self._build_detailed_message(message, expected, actual)
+        else:
+            full_message = message
+
+        super().__init__(full_message)
+
+    def _build_detailed_message(self, base_message: str, expected: pa.Schema, actual: pa.Schema) -> str:
+        """Build a detailed message showing exactly what differs."""
+        lines = [base_message, ""]
+
+        if self.context:
+            lines.append(f"  Context: {self.context}")
+            lines.append("")
+
+        # Build field maps for comparison
+        expected_fields = {f.name: f for f in expected}
+        actual_fields = {f.name: f for f in actual}
+
+        expected_names = set(expected_fields.keys())
+        actual_names = set(actual_fields.keys())
+
+        # Find differences
+        missing = expected_names - actual_names
+        extra = actual_names - expected_names
+        common = expected_names & actual_names
+
+        # Check for type mismatches in common fields
+        type_mismatches = []
+        for name in common:
+            exp_field = expected_fields[name]
+            act_field = actual_fields[name]
+            if exp_field.type != act_field.type:
+                type_mismatches.append((name, exp_field.type, act_field.type))
+            elif exp_field.nullable != act_field.nullable:
+                exp_null = "nullable" if exp_field.nullable else "non-nullable"
+                act_null = "nullable" if act_field.nullable else "non-nullable"
+                type_mismatches.append((name, exp_null, act_null))
+
+        # Check for order differences (only if names match but order differs)
+        order_differs = False
+        if not missing and not extra and not type_mismatches:
+            expected_order = [f.name for f in expected]
+            actual_order = [f.name for f in actual]
+            if expected_order != actual_order:
+                order_differs = True
+
+        # Report missing fields
+        if missing:
+            lines.append("  Missing fields (expected but not found):")
+            for name in sorted(missing):
+                field = expected_fields[name]
+                lines.append(f"    - {name}: {field.type}")
+
+        # Report extra fields
+        if extra:
+            lines.append("  Extra fields (found but not expected):")
+            for name in sorted(extra):
+                field = actual_fields[name]
+                lines.append(f"    - {name}: {field.type}")
+
+        # Report type mismatches
+        if type_mismatches:
+            lines.append("  Type mismatches:")
+            for name, exp_type, act_type in type_mismatches:
+                lines.append(f"    - {name}: expected {exp_type}, got {act_type}")
+
+        # Report order differences
+        if order_differs:
+            lines.append("  Field order differs:")
+            lines.append(f"    Expected: {[f.name for f in expected]}")
+            lines.append(f"    Actual:   {[f.name for f in actual]}")
+
+        # Summary of schemas
+        lines.append("")
+        lines.append("  Expected schema:")
+        for field in expected:
+            nullable = " (nullable)" if field.nullable else ""
+            lines.append(f"    {field.name}: {field.type}{nullable}")
+
+        lines.append("  Actual schema:")
+        for field in actual:
+            nullable = " (nullable)" if field.nullable else ""
+            lines.append(f"    {field.name}: {field.type}{nullable}")
+
+        return "\n".join(lines)

vgi/function.py

@@ -0,0 +1,245 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""Core data structures for VGI function calls and bind results.
+
+This module defines the foundational classes used during function binding
+in the VGI protocol. When a client invokes a function, it sends the
+function name, arguments, input schema, and function type.
+
+Classes:
+    Function: Base class for all VGI functions.
+
+See Also:
+    vgi.scalar_function: Scalar functions with 1:1 row transforms.
+    vgi.table_function: Table functions with cardinality hints.
+    vgi.table_in_out_function: Streaming table functions for batch transforms.
+    vgi_rpc.log: Level and Message for in-band function diagnostics.
+
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from abc import ABC
+from typing import (
+    Annotated,
+    Any,
+    ClassVar,
+    final,
+    get_args,
+    get_origin,
+)
+
+import pyarrow as pa
+
+from vgi.exceptions import SchemaValidationError
+from vgi.function_storage import FunctionStorage, FunctionStorageSqlite
+from vgi.metadata import MetadataMixin, ResolvedMetadata
+
+
+def _resolve_storage() -> FunctionStorage:
+    """Resolve the default FunctionStorage backend from environment."""
+    backend = os.environ.get("VGI_WORKER_SHARED_STORAGE", "sqlite").lower()
+    if backend == "memory":
+        # In-process tier: SQLite at ":memory:" (shared-cache, process-local).
+        # Ignores VGI_WORKER_SQLITE_PATH by design — "memory" always means a
+        # process-local store with no cross-process coordination. Correct only
+        # for single-process deployments; use "sqlite" (file) for multi-process
+        # on one machine, "cloudflare-do" for cross-machine.
+        return FunctionStorageSqlite(db_path=":memory:")
+    if backend == "sqlite":
+        # VGI_WORKER_SQLITE_PATH=":memory:" picks the in-process shared-cache
+        # in-memory backend. Used by single-process test fixtures (notably
+        # the test fixture HTTP server) to avoid per-op WAL fsync cost.
+        db_path = os.environ.get("VGI_WORKER_SQLITE_PATH") or None
+        if os.environ.get("VGI_SQLITE_SHARD") == "1":
+            # Debug: partition sqlite by shard_key to reproduce cloudflare-do
+            # per-DO isolation locally (surfaces shard-routing bugs sqlite hides).
+            from vgi.function_storage import ShardedSqliteStorage
+
+            return ShardedSqliteStorage(db_path)
+        return FunctionStorageSqlite(db_path=db_path)
+    if backend == "azure-sql":
+        from vgi.function_storage_azure_sql import FunctionStorageAzureSql
+
+        return FunctionStorageAzureSql.from_env()
+    if backend == "cloudflare-do":
+        from vgi.function_storage_cf_do import FunctionStorageCfDo
+
+        return FunctionStorageCfDo.from_env()
+    raise ValueError(
+        f"Unknown VGI_WORKER_SHARED_STORAGE backend: {backend!r}. "
+        "Supported: 'memory', 'sqlite', 'azure-sql', 'cloudflare-do'"
+    )
+
+
+class _DefaultStorageDescriptor:
+    """Resolve FunctionStorage lazily on first attribute access.
+
+    This avoids evaluating environment variables at import time. When a
+    subclass explicitly sets ``storage = SomeStorage(...)``, the plain
+    attribute shadows this descriptor — no interference.
+    """
+
+    _resolved: FunctionStorage | None = None
+
+    def __get__(self, obj: object | None, objtype: type | None = None) -> FunctionStorage:
+        if self._resolved is None:
+            self._resolved = _resolve_storage()
+        return self._resolved
+
+
+# Default max_workers when not explicitly specified (effectively unlimited)
+DEFAULT_MAX_WORKERS = 99999
+
+__all__ = [
+    "Function",
+    "DEFAULT_MAX_WORKERS",
+]
+
+
+class Function(ABC, MetadataMixin):
+    """Base class for all VGI functions.
+
+    Provides shared infrastructure (metadata, storage, Arg descriptor extraction,
+    schema validation) for all function types. Since the child classes have very
+    different APIs, there are not many standard methods here.
+
+    Subclasses can define a nested Meta class to provide metadata.
+
+    Available Meta attributes:
+        name: Function name for registration (default: class name to snake_case)
+        description: Human-readable description (default: docstring first line)
+        categories: Classification tags
+        examples: List of SQL examples
+        See vgi.metadata for all available attributes.
+
+    Attributes:
+        logger: Structured logger for function diagnostics.
+
+    See Also:
+        vgi.scalar_function.ScalarFunction: Scalar 1:1 row transforms.
+        vgi.table_function.TableFunctionGenerator: Table functions.
+        vgi.table_in_out_function.TableInOutFunction: Table-in-out batch transforms.
+        vgi.metadata: Metadata documentation for functions.
+
+    """
+
+    storage: ClassVar[FunctionStorage] = _DefaultStorageDescriptor()  # type: ignore[assignment]
+
+    # Cache for resolved metadata
+    _metadata_cache: ClassVar[ResolvedMetadata | None] = None
+
+    def __init_subclass__(cls, **kwargs: Any) -> None:
+        """Extract Arg descriptors from Annotated type hints.
+
+        The Arg is extracted from the annotation metadata and installed
+        as a class attribute (descriptor).
+        """
+        super().__init_subclass__(**kwargs)
+
+        # Import here to avoid circular imports
+        from vgi.arguments import AnyArrowValue, Arg
+
+        # Get type hints with include_extras=True to access Annotated metadata
+        # We only look at the class's own annotations (not inherited) to avoid
+        # issues with forward references that can't be resolved in this module
+        annotations = getattr(cls, "__annotations__", {})
+        if not annotations:
+            return
+
+        # Build evaluation namespace from module globals
+        module = __import__(cls.__module__, fromlist=[""])
+        globalns = getattr(module, "__dict__", {})
+        # Add common typing imports that might be needed
+        globalns.setdefault("Annotated", Annotated)
+
+        for attr_name, annotation in annotations.items():
+            # Evaluate string annotation if needed (from __future__ import annotations)
+            if isinstance(annotation, str):
+                try:
+                    hint = eval(annotation, globalns)  # noqa: S307
+                except Exception:
+                    # Can't evaluate this annotation, skip it
+                    continue
+            else:
+                hint = annotation
+            # Skip if not Annotated
+            if get_origin(hint) is not Annotated:
+                continue
+
+            # Get the base type and metadata from Annotated[BaseType, metadata...]
+            args = get_args(hint)
+            if not args:
+                continue
+
+            base_type = args[0]
+            metadata = args[1:]
+
+            # Look for Arg in the metadata
+            for meta in metadata:
+                if isinstance(meta, Arg):
+                    # Check if an Arg descriptor already exists for this name
+                    # (could be from a parent class or explicit assignment)
+                    existing = getattr(cls, attr_name, None)
+                    if isinstance(existing, Arg):
+                        continue
+
+                    # Set the name on the Arg (normally done by __set_name__)
+                    meta._name = attr_name
+
+                    # Set _returns_any_arrow_value based on the annotated type
+                    meta._returns_any_arrow_value = base_type is AnyArrowValue
+
+                    # Infer _type_param from the base type for metadata extraction
+                    # and type_bound validation
+                    if base_type is AnyArrowValue or meta.type_bound is not None:
+                        # AnyArrowValue or type_bound means this is an AnyArrow arg
+                        from vgi.arguments import AnyArrow
+
+                        meta._type_param = AnyArrow
+                    elif meta._type_param is None:
+                        # Use the annotation type as the type param
+                        meta._type_param = base_type
+
+                    # Install the Arg as a class attribute
+                    setattr(cls, attr_name, meta)
+                    break
+
+    def __init__(
+        self,
+        *,
+        logger: logging.Logger,
+    ):
+        """Initialize the function with invocation data and logger.
+
+        Args:
+            logger: Logger for function diagnostics.
+
+        """
+        self.logger = logger
+
+    @final
+    @classmethod
+    def _validate_output_schema(cls, batch: pa.RecordBatch, output_schema: pa.Schema) -> None:
+        """Validate that a batch conforms to the expected output schema."""
+        if batch.schema != output_schema:
+            raise SchemaValidationError(
+                "Output batch schema does not match expected output_schema.",
+                expected=output_schema,
+                actual=batch.schema,
+                context=f"output from {cls.__name__}",
+            )
+
+    @final
+    @classmethod
+    def _validate_input_schema(cls, batch: pa.RecordBatch, input_schema: pa.Schema) -> None:
+        """Validate that a batch conforms to the expected input schema."""
+        if batch.schema != input_schema:
+            raise SchemaValidationError(
+                "Input batch schema does not match expected input_schema.",
+                expected=input_schema,
+                actual=batch.schema,
+                context=f"input to {cls.__name__}",
+            )