polars-runtime-compat 1.34.0b2__cp39-abi3-macosx_11_0_arm64.whl

polars/lazyframe/in_process.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from polars._utils.wrap import wrap_df
+
+if TYPE_CHECKING:
+    from polars import DataFrame
+    from polars._plr import PyInProcessQuery
+
+
+class InProcessQuery:
+    """
+    A placeholder for an in process query.
+
+    This can be used to do something else while a query is running.
+    The queries can be cancelled. You can peek if the query is finished,
+    or you can await the result.
+    """
+
+    def __init__(self, ipq: PyInProcessQuery) -> None:
+        self._inner = ipq
+
+    def cancel(self) -> None:
+        """Cancel the query at earliest convenience."""
+        self._inner.cancel()
+
+    def fetch(self) -> DataFrame | None:
+        """
+        Fetch the result.
+
+        If it is ready, a materialized DataFrame is returned.
+        If it is not ready it will return `None`.
+        """
+        if (out := self._inner.fetch()) is not None:
+            return wrap_df(out)
+        else:
+            return None
+
+    def fetch_blocking(self) -> DataFrame:
+        """Await the result synchronously."""
+        return wrap_df(self._inner.fetch_blocking())

polars/lazyframe/opt_flags.py

@@ -0,0 +1,333 @@
+from __future__ import annotations
+
+import contextlib
+
+from polars._utils.deprecation import issue_deprecation_warning
+
+with contextlib.suppress(ImportError):  # Module not available when building docs
+    from polars._plr import PyOptFlags
+
+import inspect
+from functools import wraps
+from typing import TYPE_CHECKING, Callable, TypeVar
+
+if TYPE_CHECKING:
+    import sys
+
+    if sys.version_info >= (3, 10):
+        from typing import ParamSpec
+    else:
+        from typing_extensions import ParamSpec
+
+    P = ParamSpec("P")
+    T = TypeVar("T")
+
+
+class QueryOptFlags:
+    """
+    The set of the optimizations considered during query optimization.
+
+    .. warning::
+        This functionality is considered **unstable**. It may be changed
+        at any point without it being considered a breaking change.
+    """
+
+    def __init__(
+        self,
+        *,
+        predicate_pushdown: None | bool = None,
+        projection_pushdown: None | bool = None,
+        simplify_expression: None | bool = None,
+        slice_pushdown: None | bool = None,
+        comm_subplan_elim: None | bool = None,
+        comm_subexpr_elim: None | bool = None,
+        cluster_with_columns: None | bool = None,
+        collapse_joins: None | bool = None,
+        check_order_observe: None | bool = None,
+        fast_projection: None | bool = None,
+    ) -> None:
+        self._pyoptflags = PyOptFlags.default()
+        self.update(
+            predicate_pushdown=predicate_pushdown,
+            projection_pushdown=projection_pushdown,
+            simplify_expression=simplify_expression,
+            slice_pushdown=slice_pushdown,
+            comm_subplan_elim=comm_subplan_elim,
+            comm_subexpr_elim=comm_subexpr_elim,
+            cluster_with_columns=cluster_with_columns,
+            collapse_joins=collapse_joins,
+            check_order_observe=check_order_observe,
+            fast_projection=fast_projection,
+        )
+
+    @classmethod
+    def _from_pyoptflags(self, pyoptflags: PyOptFlags) -> QueryOptFlags:
+        optflags = self.__new__(self)
+        optflags._pyoptflags = pyoptflags
+        return optflags
+
+    @staticmethod
+    def none(
+        *,
+        predicate_pushdown: None | bool = None,
+        projection_pushdown: None | bool = None,
+        simplify_expression: None | bool = None,
+        slice_pushdown: None | bool = None,
+        comm_subplan_elim: None | bool = None,
+        comm_subexpr_elim: None | bool = None,
+        cluster_with_columns: None | bool = None,
+        collapse_joins: None | bool = None,
+        check_order_observe: None | bool = None,
+        fast_projection: None | bool = None,
+    ) -> QueryOptFlags:
+        """Create new empty set off optimizations."""
+        optflags = QueryOptFlags()
+        optflags.no_optimizations()
+        return optflags.update(
+            predicate_pushdown=predicate_pushdown,
+            projection_pushdown=projection_pushdown,
+            simplify_expression=simplify_expression,
+            slice_pushdown=slice_pushdown,
+            comm_subplan_elim=comm_subplan_elim,
+            comm_subexpr_elim=comm_subexpr_elim,
+            cluster_with_columns=cluster_with_columns,
+            collapse_joins=collapse_joins,
+            check_order_observe=check_order_observe,
+            fast_projection=fast_projection,
+        )
+
+    def update(
+        self,
+        *,
+        predicate_pushdown: None | bool = None,
+        projection_pushdown: None | bool = None,
+        simplify_expression: None | bool = None,
+        slice_pushdown: None | bool = None,
+        comm_subplan_elim: None | bool = None,
+        comm_subexpr_elim: None | bool = None,
+        cluster_with_columns: None | bool = None,
+        collapse_joins: None | bool = None,
+        check_order_observe: None | bool = None,
+        fast_projection: None | bool = None,
+    ) -> QueryOptFlags:
+        """Update the current optimization flags."""
+        if predicate_pushdown is not None:
+            self.predicate_pushdown = predicate_pushdown
+        if projection_pushdown is not None:
+            self.projection_pushdown = projection_pushdown
+        if simplify_expression is not None:
+            self.simplify_expression = simplify_expression
+        if slice_pushdown is not None:
+            self.slice_pushdown = slice_pushdown
+        if comm_subplan_elim is not None:
+            self.comm_subplan_elim = comm_subplan_elim
+        if comm_subexpr_elim is not None:
+            self.comm_subexpr_elim = comm_subexpr_elim
+        if cluster_with_columns is not None:
+            self.cluster_with_columns = cluster_with_columns
+        if collapse_joins is not None:
+            issue_deprecation_warning(
+                "the `collapse_joins` parameter for `QueryOptFlags` is deprecated. "
+                "Use `predicate_pushdown` instead.",
+                version="1.33.1",
+            )
+            if not collapse_joins:
+                self.predicate_pushdown = False
+        if check_order_observe is not None:
+            self.check_order_observe = check_order_observe
+        if fast_projection is not None:
+            self.fast_projection = fast_projection
+
+        return self
+
+    @staticmethod
+    def _eager() -> QueryOptFlags:
+        """Create new empty set off optimizations."""
+        optflags = QueryOptFlags()
+        optflags.no_optimizations()
+        optflags._pyoptflags.eager = True
+        optflags.simplify_expression = True
+        return optflags
+
+    def __copy__(self) -> QueryOptFlags:
+        return QueryOptFlags._from_pyoptflags(self._pyoptflags.copy())
+
+    def __deepcopy__(self) -> QueryOptFlags:
+        return QueryOptFlags._from_pyoptflags(self._pyoptflags.copy())
+
+    def no_optimizations(self) -> None:
+        """Remove selected optimizations."""
+        self._pyoptflags.no_optimizations()
+
+    @property
+    def projection_pushdown(self) -> bool:
+        """Only read columns that are used later in the query."""
+        return self._pyoptflags.projection_pushdown
+
+    @projection_pushdown.setter
+    def projection_pushdown(self, value: bool) -> None:
+        self._pyoptflags.projection_pushdown = value
+
+    @property
+    def predicate_pushdown(self) -> bool:
+        """Apply predicates/filters as early as possible."""
+        return self._pyoptflags.predicate_pushdown
+
+    @predicate_pushdown.setter
+    def predicate_pushdown(self, value: bool) -> None:
+        self._pyoptflags.predicate_pushdown = value
+
+    @property
+    def cluster_with_columns(self) -> bool:
+        """Cluster sequential `with_columns` calls to independent calls."""
+        return self._pyoptflags.cluster_with_columns
+
+    @cluster_with_columns.setter
+    def cluster_with_columns(self, value: bool) -> None:
+        self._pyoptflags.cluster_with_columns = value
+
+    @property
+    def simplify_expression(self) -> bool:
+        """Run many expression optimization rules until fixed point."""
+        return self._pyoptflags.simplify_expression
+
+    @simplify_expression.setter
+    def simplify_expression(self, value: bool) -> None:
+        self._pyoptflags.simplify_expression = value
+
+    @property
+    def slice_pushdown(self) -> bool:
+        """Pushdown slices/limits."""
+        return self._pyoptflags.slice_pushdown
+
+    @slice_pushdown.setter
+    def slice_pushdown(self, value: bool) -> None:
+        self._pyoptflags.slice_pushdown = value
+
+    @property
+    def comm_subplan_elim(self) -> bool:
+        """Elide duplicate plans and caches their outputs."""
+        return self._pyoptflags.comm_subplan_elim
+
+    @comm_subplan_elim.setter
+    def comm_subplan_elim(self, value: bool) -> None:
+        self._pyoptflags.comm_subplan_elim = value
+
+    @property
+    def comm_subexpr_elim(self) -> bool:
+        """Elide duplicate expressions and caches their outputs."""
+        return self._pyoptflags.comm_subexpr_elim
+
+    @comm_subexpr_elim.setter
+    def comm_subexpr_elim(self, value: bool) -> None:
+        self._pyoptflags.comm_subexpr_elim = value
+
+    @property
+    def check_order_observe(self) -> bool:
+        """Do not maintain order if the order would not be observed."""
+        return self._pyoptflags.check_order_observe
+
+    @check_order_observe.setter
+    def check_order_observe(self, value: bool) -> None:
+        self._pyoptflags.check_order_observe = value
+
+    @property
+    def fast_projection(self) -> bool:
+        """Replace simple projections with a faster inlined projection that skips the expression engine."""  # noqa: W505
+        return self._pyoptflags.fast_projection
+
+    @fast_projection.setter
+    def fast_projection(self, value: bool) -> None:
+        self._pyoptflags.fast_projection = value
+
+    def __str__(self) -> str:
+        return f"""
+QueryOptFlags {{
+    type_coercion: {self._pyoptflags.type_coercion}
+    type_check: {self._pyoptflags.type_check}
+
+    predicate_pushdown: {self.predicate_pushdown}
+    projection_pushdown: {self.projection_pushdown}
+    simplify_expression: {self.simplify_expression}
+    slice_pushdown: {self.slice_pushdown}
+    comm_subplan_elim: {self.comm_subplan_elim}
+    comm_subexpr_elim: {self.comm_subexpr_elim}
+    cluster_with_columns: {self.cluster_with_columns}
+    check_order_observe: {self.check_order_observe}
+    fast_projection: {self.fast_projection}
+
+    eager: {self._pyoptflags.eager}
+    streaming: {self._pyoptflags.streaming}
+}}
+        """.strip()
+
+
+DEFAULT_QUERY_OPT_FLAGS: QueryOptFlags
+try:  # Module not available when building docs
+    DEFAULT_QUERY_OPT_FLAGS = QueryOptFlags()
+except (ImportError, NameError) as _:
+    DEFAULT_QUERY_OPT_FLAGS = ()  # type: ignore[assignment]
+
+
+def forward_old_opt_flags() -> Callable[[Callable[P, T]], Callable[P, T]]:
+    """Decorator to mark to forward the old optimization flags."""
+
+    def helper(f: QueryOptFlags, field_name: str, value: bool) -> QueryOptFlags:  # noqa: FBT001
+        setattr(f, field_name, value)
+        return f
+
+    def helper_hidden(f: QueryOptFlags, field_name: str, value: bool) -> QueryOptFlags:  # noqa: FBT001
+        setattr(f._pyoptflags, field_name, value)
+        return f
+
+    def clear_optimizations(f: QueryOptFlags, value: bool) -> QueryOptFlags:  # noqa: FBT001
+        if value:
+            return QueryOptFlags.none()
+        else:
+            return f
+
+    def eager(f: QueryOptFlags, value: bool) -> QueryOptFlags:  # noqa: FBT001
+        if value:
+            return QueryOptFlags._eager()
+        else:
+            return f
+
+    OLD_OPT_PARAMETERS_MAPPING = {
+        "no_optimization": lambda f, v: clear_optimizations(f, v),
+        "_eager": lambda f, v: eager(f, v),
+        "type_coercion": lambda f, v: helper_hidden(f, "type_coercion", v),
+        "_type_check": lambda f, v: helper_hidden(f, "type_check", v),
+        "predicate_pushdown": lambda f, v: helper(f, "predicate_pushdown", v),
+        "projection_pushdown": lambda f, v: helper(f, "projection_pushdown", v),
+        "simplify_expression": lambda f, v: helper(f, "simplify_expression", v),
+        "slice_pushdown": lambda f, v: helper(f, "slice_pushdown", v),
+        "comm_subplan_elim": lambda f, v: helper(f, "comm_subplan_elim", v),
+        "comm_subexpr_elim": lambda f, v: helper(f, "comm_subexpr_elim", v),
+        "cluster_with_columns": lambda f, v: helper(f, "cluster_with_columns", v),
+        "collapse_joins": lambda f, v: helper(f, "collapse_joins", v),
+        "_check_order": lambda f, v: helper(f, "check_order_observe", v),
+    }
+
+    def decorate(function: Callable[P, T]) -> Callable[P, T]:
+        @wraps(function)
+        def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+            optflags: QueryOptFlags = kwargs.get(
+                "optimizations", DEFAULT_QUERY_OPT_FLAGS
+            )  # type: ignore[assignment]
+            optflags = optflags.__copy__()
+            for key in list(kwargs.keys()):
+                cb = OLD_OPT_PARAMETERS_MAPPING.get(key)
+                if cb is not None:
+                    from polars._utils.various import issue_warning
+
+                    message = f"optimization flag `{key}` is deprecated. Please use `optimizations` parameter\n(Deprecated in version 1.30.0)"
+                    issue_warning(message, DeprecationWarning)
+                    optflags = cb(optflags, kwargs.pop(key))  # type: ignore[no-untyped-call,unused-ignore]
+
+            kwargs["optimizations"] = optflags
+            return function(*args, **kwargs)
+
+        wrapper.__signature__ = inspect.signature(function)  # type: ignore[attr-defined]
+        return wrapper
+
+    return decorate

polars/meta/__init__.py

@@ -0,0 +1,14 @@
+"""Public functions that provide information about the Polars package or the environment it runs in."""  # noqa: W505
+
+from polars.meta.build import build_info
+from polars.meta.index_type import get_index_type
+from polars.meta.thread_pool import thread_pool_size, threadpool_size
+from polars.meta.versions import show_versions
+
+__all__ = [
+    "build_info",
+    "get_index_type",
+    "show_versions",
+    "thread_pool_size",
+    "threadpool_size",
+]

polars/meta/build.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from typing import Any
+
+from polars._utils.polars_version import get_polars_version
+
+try:
+    from polars._plr import __build__
+except ImportError:
+    __build__ = {}
+
+__build__["version"] = get_polars_version() or "<missing>"
+
+
+def build_info() -> dict[str, Any]:
+    """
+    Return detailed Polars build information.
+
+    The dictionary with build information contains the following keys:
+
+    - `"compiler"`
+    - `"time"`
+    - `"dependencies"`
+    - `"features"`
+    - `"host"`
+    - `"target"`
+    - `"git"`
+    - `"version"`
+
+    If Polars was compiled without the `build_info` feature flag, only the `"version"`
+    key is included.
+    """
+    return __build__

polars/meta/index_type.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import contextlib
+from typing import TYPE_CHECKING
+
+with contextlib.suppress(ImportError):  # Module not available when building docs
+    import polars._plr as plr
+
+if TYPE_CHECKING:
+    from polars._typing import PolarsIntegerType
+
+
+def get_index_type() -> PolarsIntegerType:
+    """
+    Return the data type used for Polars indexing.
+
+    Returns
+    -------
+    PolarsIntegerType
+        :class:`UInt32` in regular Polars, :class:`UInt64` in bigidx Polars.
+
+    Examples
+    --------
+    >>> pl.get_index_type()
+    UInt32
+    """
+    return plr.get_index_type()

polars/meta/thread_pool.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import contextlib
+
+from polars._utils.deprecation import deprecated
+
+with contextlib.suppress(ImportError):  # Module not available when building docs
+    import polars._plr as plr
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import sys
+
+    if sys.version_info >= (3, 13):
+        from warnings import deprecated
+    else:
+        from typing_extensions import deprecated  # noqa: TC004
+
+
+def thread_pool_size() -> int:
+    """
+    Return the number of threads in the Polars thread pool.
+
+    Notes
+    -----
+    The thread pool size can be overridden by setting the `POLARS_MAX_THREADS`
+    environment variable before process start. The thread pool is not behind a
+    lock, so it cannot be modified once set. A reasonable use case for this might
+    be temporarily limiting the number of threads before importing Polars in a
+    PySpark UDF or similar context. Otherwise, it is strongly recommended not to
+    override this value as it will be set automatically by the engine.
+
+    Examples
+    --------
+    >>> pl.thread_pool_size()  # doctest: +SKIP
+    16
+    """
+    return plr.thread_pool_size()
+
+
+@deprecated("`threadpool_size` was renamed; use `thread_pool_size` instead.")
+def threadpool_size() -> int:
+    """
+    Return the number of threads in the Polars thread pool.
+
+    .. deprecated:: 0.20.7
+        This function has been renamed to :func:`thread_pool_size`.
+    """
+    return thread_pool_size()

polars/meta/versions.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import sys
+
+from polars._cpu_check import get_runtime_repr
+from polars._utils.polars_version import get_polars_version
+from polars.meta.index_type import get_index_type
+
+
+def show_versions() -> None:
+    """
+    Print out the version of Polars and its optional dependencies.
+
+    Examples
+    --------
+    >>> pl.show_versions()  # doctest: +SKIP
+    --------Version info---------
+    Polars:               0.20.22
+    Index type:           UInt32
+    Platform:             macOS-14.4.1-arm64-arm-64bit
+    Python:               3.11.8 (main, Feb  6 2024, 21:21:21) [Clang 15.0.0 (clang-1500.1.0.2.5)]
+    LTS CPU:              False
+    ----Optional dependencies----
+    adbc_driver_manager:  0.11.0
+    altair:               5.4.0
+    cloudpickle:          3.0.0
+    connectorx:           0.3.2
+    deltalake:            0.17.1
+    fastexcel:            0.10.4
+    fsspec:               2023.12.2
+    gevent:               24.2.1
+    matplotlib:           3.8.4
+    numpy:                1.26.4
+    openpyxl:             3.1.2
+    pandas:               2.2.2
+    pyarrow:              16.0.0
+    pydantic:             2.7.1
+    pyiceberg:            0.7.1
+    sqlalchemy:           2.0.29
+    torch:                2.2.2
+    xlsx2csv:             0.8.2
+    xlsxwriter:           3.2.0
+    """  # noqa: W505
+    # Note: we import 'platform' here (rather than at the top of the
+    # module) as a micro-optimization for polars' initial import
+    import platform
+
+    deps = _get_dependency_list()
+    core_properties = ("Polars", "Index type", "Platform", "Python", "Runtime")
+    keylen = max(len(x) for x in [*core_properties, "Azure CLI", *deps]) + 1
+
+    print("--------Version info---------")
+    print(f"{'Polars:':{keylen}s} {get_polars_version()}")
+    print(f"{'Index type:':{keylen}s} {get_index_type()}")
+    print(f"{'Platform:':{keylen}s} {platform.platform()}")
+    print(f"{'Python:':{keylen}s} {sys.version}")
+    print(f"{'Runtime:':{keylen}s} {get_runtime_repr()}")
+
+    print("\n----Optional dependencies----")
+
+    from polars.io.cloud.credential_provider import CredentialProviderAzure
+
+    print(f"{'Azure CLI':{keylen}s} ", end="", flush=True)
+    print(CredentialProviderAzure._azcli_version() or "<not installed>")
+
+    for name in deps:
+        print(f"{name:{keylen}s} ", end="", flush=True)
+        print(_get_dependency_version(name))
+
+
+# See the list of dependencies in pyproject.toml.
+def _get_dependency_list() -> list[str]:
+    return [
+        "adbc_driver_manager",
+        "altair",
+        "azure.identity",
+        "boto3",
+        "cloudpickle",
+        "connectorx",
+        "deltalake",
+        "fastexcel",
+        "fsspec",
+        "gevent",
+        "google.auth",
+        "great_tables",
+        "matplotlib",
+        "numpy",
+        "openpyxl",
+        "pandas",
+        "polars_cloud",
+        "pyarrow",
+        "pydantic",
+        "pyiceberg",
+        "sqlalchemy",
+        "torch",
+        "xlsx2csv",
+        "xlsxwriter",
+    ]
+
+
+def _get_dependency_version(dep_name: str) -> str:
+    # note: we import 'importlib' inside the function as an
+    # optimisation for initial polars module import
+    import importlib
+    import importlib.metadata
+
+    try:
+        module = importlib.import_module(dep_name)
+    except ImportError:
+        return "<not installed>"
+
+    if hasattr(module, "__version__"):
+        module_version = module.__version__
+    else:
+        try:
+            module_version = importlib.metadata.version(dep_name)  # pragma: no cover
+        except Exception:
+            return "<invalid install>"
+
+    return module_version