opteryx-core 0.5.0__cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

opteryx/__init__.py

@@ -0,0 +1,344 @@
+# isort: skip_file
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# See the License at http://www.apache.org/licenses/LICENSE-2.0
+# Distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND.
+
+"""
+Opteryx is a SQL query engine optimized for speed and efficiency.
+
+To get started:
+    import opteryx
+    results = opteryx.query("SELECT * FROM my_table")
+
+Opteryx handles parsing, planning, and execution of SQL queries with a focus on low-latency analytics over
+local or remote data sources.
+
+For more information check out https://opteryx.dev.
+"""
+
+import datetime
+import os
+import platform
+import secrets
+import time
+import warnings
+from pathlib import Path
+
+from decimal import getcontext
+from typing import Optional, Union, Dict, Any, List, TYPE_CHECKING, Iterable
+
+if TYPE_CHECKING:  # pragma: no cover - only for type checkers
+    import pyarrow
+
+# Set Decimal precision to 28 globally
+getcontext().prec = 28
+
+
+# end-of-stream marker
+def _generate_eos_marker() -> int:
+    """Generate a random 64-bit signed end-of-stream marker."""
+    return secrets.randbits(64) - (1 << 63)
+
+
+EOS: int = _generate_eos_marker()
+
+
+def is_mac() -> bool:  # pragma: no cover
+    """
+    Check if the current platform is macOS.
+
+    Returns:
+        bool: True if the platform is macOS, False otherwise.
+    """
+    return platform.system().lower() == "darwin"
+
+
+# we do a separate check for debug mode here so we don't load the config module just yet
+OPTERYX_DEBUG = os.environ.get("OPTERYX_DEBUG") is not None
+
+
+# python-dotenv allows us to create an environment file to store secrets.
+# Only try to import dotenv if a .env file exists to avoid paying the
+# import cost when no environment file is present.
+_env_path = Path.cwd() / ".env"
+if _env_path.exists():
+    try:
+        import dotenv  # type:ignore
+
+        dotenv.load_dotenv(dotenv_path=_env_path)
+        if OPTERYX_DEBUG:
+            print(f"{datetime.datetime.now()} [LOADER] Loading `.env` file.")
+    except ImportError:  # pragma: no cover
+        # dotenv is optional; if it's not installed, just continue.
+        pass
+
+
+if OPTERYX_DEBUG:  # pragma: no cover
+    from opteryx.debugging import OpteryxOrsoImportFinder
+
+from opteryx import config
+
+from opteryx.connectors import register_workspace
+from opteryx.connectors import set_default_connector
+
+from opteryx.__version__ import __author__
+from opteryx.__version__ import __build__
+from opteryx.__version__ import __version__
+from opteryx.__version__ import __lib__
+
+
+__all__ = [
+    "analyze_query",
+    "connect",
+    "Connection",
+    "query",
+    "query_to_arrow",
+    "register_workspace",
+    "set_default_connector",
+    "__author__",
+    "__build__",
+    "__version__",
+    "__lib__",
+    "OPTERYX_DEBUG",
+]
+
+
+def connect(*args, **kwargs) -> "Connection":
+    """
+    Establish a new database connection and return a Connection object.
+
+    Note: This function is designed to comply with the 'connect' method
+    described in PEP0249 for Python Database API Specification v2.0.
+    """
+    # Lazy import Connection
+    from opteryx.connection import Connection
+
+    # Create and return a Connection object
+    return Connection(*args, **kwargs)
+
+
+def query(
+    operation: str,
+    params: Union[list, Dict, None] = None,
+    visibility_filters: Optional[Dict[str, Any]] = None,
+    **kwargs,
+):
+    """
+    Helper function to execute a query and return a cursor.
+
+    This function is designed to be similar to the DuckDB function of the same name.
+    It simplifies the process of executing queries by abstracting away the connection
+    and cursor creation steps.
+
+    Parameters:
+        operation: SQL query string
+        params: list of parameters to bind into the SQL query
+        kwargs: additional arguments for creating the Connection
+
+    Returns:
+        Executed cursor
+    """
+    # Lazy import Connection
+    from opteryx.connection import Connection
+
+    # Create a new database connection
+    conn = Connection(**kwargs)
+
+    # Create a new cursor object using the connection
+    curr = conn.cursor()
+    curr._owns_connection = True
+
+    # Execute the SQL query using the cursor
+    curr.execute(operation=operation, params=params, visibility_filters=visibility_filters)
+
+    # Return the executed cursor
+    return curr
+
+
+def query_to_arrow(
+    operation: str,
+    params: Union[List, Dict, None] = None,
+    visibility_filters: Optional[Dict[str, Any]] = None,
+    limit: int = None,
+    **kwargs,
+) -> "pyarrow.Table":
+    """
+    Helper function to execute a query and return a pyarrow Table.
+
+    This is the fastest way to get a pyarrow table from Opteryx, it bypasses needing
+    orso to create a Dataframe and converting from the Dataframe. This is fast, but
+    not doing it is faster.
+
+    Parameters:
+        operation: SQL query string
+        params: list of parameters to bind into the SQL query (optional)
+        limit: stop after this many rows (optional)
+        kwargs: additional arguments for creating the Connection
+
+    Returns:
+        pyarrow Table
+    """
+    # Lazy import Connection
+    from opteryx.connection import Connection
+
+    # Create a new database connection
+    conn = Connection(**kwargs)
+
+    # Create a new cursor object using the connection
+    curr = conn.cursor()
+    curr._owns_connection = True
+
+    # Execute the SQL query using the cursor
+    return curr.execute_to_arrow(
+        operation=operation, params=params, visibility_filters=visibility_filters, limit=limit
+    )
+
+
+def query_to_arrow_batches(
+    operation: str,
+    params: Union[List, Dict, None] = None,
+    batch_size: int = 1024,
+    limit: int = None,
+    visibility_filters: Optional[Dict[str, Any]] = None,
+    **kwargs,
+) -> "Iterable[pyarrow.RecordBatch]":
+    """
+    Helper function to execute a query and stream pyarrow RecordBatch objects.
+
+    Parameters:
+        operation: SQL query string
+        params: list of parameters to bind into the SQL query
+        batch_size: Number of rows per arrow record batch
+        limit: stop after this many rows (optional)
+        kwargs: additional arguments for creating the Connection
+    Returns:
+        Iterable over pyarrow.RecordBatch
+    """
+    # Lazy import Connection
+    from opteryx.connection import Connection
+
+    # Create a new database connection
+    conn = Connection(**kwargs)
+
+    # Create a new cursor object using the connection
+    curr = conn.cursor()
+    curr._owns_connection = True
+
+    # Execute the SQL query using the cursor
+    return curr.execute_to_arrow_batches(
+        operation=operation,
+        params=params,
+        batch_size=batch_size,
+        limit=limit,
+        visibility_filters=visibility_filters,
+    )
+
+
+def analyze_query(sql: str) -> Dict[str, Any]:
+    """
+    Parse a SQL query and extract metadata without executing it.
+
+    This function analyzes the SQL query structure to extract information such as:
+    - Query type (SELECT, INSERT, UPDATE, DELETE, etc.)
+    - Tables being queried
+    - Other metadata available from the SQL syntax alone
+
+    This is useful for:
+    - Pre-flight permission checks
+    - Query validation before queueing
+    - Resource planning
+    - Query analysis
+
+    Parameters:
+        sql: SQL query string to parse
+
+    Returns:
+        Dictionary containing:
+        - query_type: Type of query (e.g., "Query", "Insert", "Update")
+        - tables: List of table names referenced in the query
+        - is_select: True if this is a SELECT query
+        - is_mutation: True if this modifies data (INSERT, UPDATE, DELETE)
+
+    Example:
+        >>> info = opteryx.parse_query_info("SELECT * FROM users WHERE id = 1")
+        >>> print(info["query_type"])
+        'Query'
+        >>> print(info["tables"])
+        ['users']
+    """
+    from opteryx.utils.query_parser import parse_query_info as _parse_query_info
+
+    return _parse_query_info(sql)
+
+
+# Try to increase the priority of the application
+if not config.DISABLE_HIGH_PRIORITY and hasattr(os, "nice"):  # pragma: no cover
+    nice_value = 0
+    try:
+        nice_value = os.nice(0)
+        if not is_mac():
+            os.nice(-20 + nice_value)
+    except PermissionError:
+        display_nice = f"{nice_value} (normal)" if nice_value == 0 else str(nice_value)
+        if OPTERYX_DEBUG:
+            print(
+                f"{datetime.datetime.now()} [LOADER] Cannot update process priority. Currently set to {display_nice}."
+            )
+
+# Enable all warnings, including DeprecationWarning
+warnings.simplefilter("once", DeprecationWarning)
+
+# Lazy initialization of system_telemetry
+_system_telemetry = None
+
+
+def _get_system_telemetry():
+    """
+    Lazy getter for system telemetry.
+
+    System telemetry are only created when first accessed, which avoids
+    importing the QueryTelemetry model (and its dependencies) during the
+    initial import of the opteryx module.
+
+    Returns:
+        QueryTelemetry: The global system telemetry object
+    """
+    global _system_telemetry
+    if _system_telemetry is None:
+        from opteryx.models import QueryTelemetry
+
+        _system_telemetry = QueryTelemetry("system")
+        _system_telemetry.start_time = time.time_ns()
+    return _system_telemetry
+
+
+# Provide access via module attribute
+def __getattr__(name):
+    """
+    Lazy load module attributes to improve import performance.
+
+    This function intercepts attribute access on the opteryx module to
+    implement lazy loading of heavy components like Connection and
+    system_telemetry. This reduces initial import time from ~500ms to ~130ms.
+
+    Supported lazy attributes:
+    - Connection: The main connection class
+    - system_telemetry: Global query telemetry object
+
+    Args:
+        name: The attribute name being accessed
+
+    Returns:
+        The requested attribute
+
+    Raises:
+        AttributeError: If the attribute doesn't exist
+    """
+    if name == "Connection":
+        from opteryx.connection import Connection
+
+        return Connection
+    elif name == "system_telemetry":
+        return _get_system_telemetry()
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

opteryx/__main__.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python
+# pragma: no cover
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# See the License at http://www.apache.org/licenses/LICENSE-2.0
+# Distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND.
+
+"""
+A command line interface for Opteryx
+"""
+
+import argparse
+import os
+import sys
+import threading
+import time
+
+sys.path.insert(1, os.path.join(sys.path[0], ".."))
+
+if True:
+    import opteryx
+    from opteryx.exceptions import MissingSqlStatement
+    from opteryx.utils.sql import clean_statement
+    from opteryx.utils.sql import remove_comments
+
+# Define ANSI color codes
+ANSI_RED = "\u001b[31m"
+ANSI_RESET = "\u001b[0m"
+
+
+def print_dots(stop_event):
+    """
+    Prints dots with pauses to indicate processing activity until the stop_event is set.
+    """
+    while not stop_event.is_set():  # pragma: no cover
+        print(".", end="", flush=True)
+        time.sleep(0.5)
+        if not stop_event.is_set():
+            print(".", end="", flush=True)
+            time.sleep(0.5)
+        if not stop_event.is_set():
+            print(".", end="", flush=True)
+            time.sleep(0.5)
+        if not stop_event.is_set():
+            print(".", end="", flush=True)
+            time.sleep(0.5)
+        if not stop_event.is_set():
+            print(".", end="", flush=True)
+            time.sleep(0.5)
+        if not stop_event.is_set():
+            print("\r     \r", end="", flush=True)
+            time.sleep(0.5)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="A command line interface for Opteryx")
+
+    parser.add_argument(
+        "--o", type=str, default="console", help="Output location (ignored by REPL)", dest="output"
+    )
+
+    # Mutually exclusive group for `--color` and `--no-color`
+    color_group = parser.add_mutually_exclusive_group()
+    color_group.add_argument(
+        "--color", dest="color", action="store_true", default=True, help="Colorize the table."
+    )
+    color_group.add_argument(
+        "--no-color", dest="color", action="store_false", help="Disable colorized output."
+    )
+
+    parser.add_argument(
+        "--table_width",
+        action="store_true",
+        default=True,
+        help="Limit console display to the screen width.",
+    )
+    parser.add_argument("--max_col_width", type=int, default=64, help="Maximum column width")
+
+    # Mutually exclusive group for `--stats` and `--no-stats`
+    stats_group = parser.add_mutually_exclusive_group()
+    stats_group.add_argument(
+        "--stats", dest="stats", action="store_true", default=True, help="Report statistics."
+    )
+    stats_group.add_argument(
+        "--no-stats", dest="stats", action="store_false", help="Disable report statistics."
+    )
+
+    parser.add_argument("--cycles", type=int, default=1, help="Repeat Execution.")
+    parser.add_argument("sql", type=str, nargs="?", help="Execute SQL statement and quit.")
+
+    args = parser.parse_args()
+
+    # Run in REPL mode if no SQL is provided
+    if args.sql is None:  # pragma: no cover
+        if args.output != "console":
+            raise ValueError("Cannot specify output location and not provide a SQL statement.")
+        print(f"Opteryx version {opteryx.__version__}")
+        print("  Enter '.help' for usage hints")
+        print("  Enter '.exit' to exit this program")
+
+        while True:  # REPL loop
+            print()
+            statement = input("opteryx> ")
+            if statement in {".exit", ".quit"}:
+                break
+            if statement == ".help":
+                print("  .exit        Exit this program")
+                print("  .help        Show help text")
+                continue
+
+            stop_event = threading.Event()
+            dot_thread = threading.Thread(target=print_dots, args=(stop_event,))
+            dot_thread.start()
+            try:
+                start = time.monotonic_ns()
+                result = opteryx.query(statement, memberships=["opteryx"])
+                result.materialize()
+                stop_event.set()
+                duration = time.monotonic_ns() - start
+                print("\r     \r", end="", flush=True)
+                print(
+                    result.display(
+                        limit=-1,
+                        display_width=args.table_width,
+                        colorize=args.color,
+                        max_column_width=args.max_col_width,
+                    )
+                )
+                if args.stats:
+                    print(
+                        f"[ {result.rowcount} rows x {result.columncount} columns ] ( {duration / 1e9} seconds )"
+                    )
+            except MissingSqlStatement:
+                print(
+                    f"{ANSI_RED}Error{ANSI_RESET}: Expected SQL statement or dot command missing."
+                )
+            except Exception as e:
+                print(f"{ANSI_RED}Error{ANSI_RESET}: {e}")
+            finally:
+                stop_event.set()
+                dot_thread.join()
+        quit()
+
+    # Process the SQL query
+    sql = clean_statement(remove_comments(args.sql))
+
+    if args.cycles > 1:  # Benchmarking mode
+        print("[", end="")
+        for i in range(args.cycles):
+            start = time.monotonic_ns()
+            result = opteryx.query_to_arrow(sql)
+            print(
+                f"{(time.monotonic_ns() - start) / 1e9:.3f}",
+                flush=True,
+                end=("," if (i + 1) < args.cycles else "]\n"),
+            )
+            sys.stdout.flush()
+        return
+
+    start = time.monotonic_ns()
+    result = opteryx.query(sql)
+    result.materialize()
+    duration = time.monotonic_ns() - start
+
+    if args.output == "console":
+        print(
+            result.display(
+                limit=-1,
+                display_width=args.table_width,
+                colorize=args.color,
+                max_column_width=args.max_col_width,
+            )
+        )
+        if args.stats:
+            print(
+                f"[ {result.rowcount} rows x {result.columncount} columns ] ( {duration / 1e9} seconds )"
+            )
+    else:
+        table = result.arrow()
+        ext = args.output.lower().split(".")[-1]
+
+        if ext == "parquet":
+            from pyarrow import parquet
+
+            parquet.write_table(table, args.output)
+        elif ext == "csv":
+            from pyarrow import csv
+
+            csv.write_csv(table, args.output)
+        elif ext == "jsonl":
+            with open(args.output, mode="wb") as file:
+                for row in result:
+                    file.write(row.as_json + b"\n")
+        elif ext == "md":
+            with open(args.output, mode="w") as file:
+                file.write(result.markdown(limit=-1))
+        else:
+            raise ValueError(f"Unknown output format '{ext}'")
+        print(
+            f"[ {result.rowcount} rows x {result.columncount} columns ] ( {duration / 1e9} seconds )"
+        )
+        print(f"Written result to '{args.output}'")
+
+
+if __name__ == "__main__":
+    import readline  # pragma: no cover
+
+    try:
+        main()
+    except Exception as e:
+        print(f"{ANSI_RED}Error{ANSI_RESET}: {e}")

opteryx/__version__.py

@@ -0,0 +1,13 @@
+# THIS FILE IS AUTOMATICALLY UPDATED DURING THE BUILD PROCESS
+# DO NOT EDIT THIS FILE DIRECTLY
+
+__build__ = 113
+__author__ = "@joocer"
+__version__ = "0.5.0"
+__lib__ = "opteryx-core"
+__build_date__ = "2025-12-30T15:40:08.207992+00:00Z"
+
+# Store the version here so:
+# 1) we don't load dependencies by storing it in __init__.py
+# 2) we can import it in setup.py for the same reason
+# 3) we can import it in the CLI for the same reason

opteryx/command.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# pragma: no cover
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# See the License at http://www.apache.org/licenses/LICENSE-2.0
+# Distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND.
+
+from opteryx.__main__ import main
+
+__all__ = ("main",)

opteryx/compiled/__init__.py

@@ -0,0 +1 @@
+