polars-runtime-compat 1.34.0b2__cp39-abi3-win_amd64.whl

polars/convert/normalize.py

@@ -0,0 +1,261 @@
+# This code is partially forked and adapted from pandas.
+# Some parts are distributed under: https://github.com/pandas-dev/pandas/blob/main/LICENSE
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Mapping, Sequence
+from typing import TYPE_CHECKING, Any
+
+from polars._utils.unstable import unstable
+from polars.dataframe import DataFrame
+from polars.datatypes.constants import N_INFER_DEFAULT
+
+if TYPE_CHECKING:
+    from polars._typing import JSONEncoder
+    from polars.schema import Schema
+
+
+def _simple_json_normalize(
+    data: dict[Any, Any] | Sequence[dict[Any, Any] | Any],
+    separator: str,
+    max_level: int,
+    encoder: JSONEncoder,
+) -> dict[Any, Any] | list[dict[Any, Any]] | Any:
+    if max_level > 0:
+        # expect dict or list (both are valid JSON objects)
+        normalized_json_object = {}
+        if isinstance(data, dict):
+            normalized_json_object = _normalize_json_ordered(
+                data=data,
+                separator=separator,
+                max_level=max_level,
+                encoder=encoder,
+            )
+        elif isinstance(data, list):
+            normalized_json_list = [
+                _simple_json_normalize(
+                    row,
+                    separator=separator,
+                    max_level=max_level,
+                    encoder=encoder,
+                )
+                for row in data
+            ]
+            return normalized_json_list
+        return normalized_json_object
+    else:
+        return data
+
+
+def _normalize_json(
+    data: Any,
+    key_string: str,
+    normalized_dict: dict[str, Any],
+    separator: str,
+    max_level: int,
+    encoder: JSONEncoder,
+) -> dict[str, Any]:
+    """
+    Main recursive function.
+
+    Designed for the most basic use case of `pl.json_normalize(data)`,
+    intended as a performance improvement.
+
+    Parameters
+    ----------
+    data : Any
+        Type dependent on types contained within nested Json
+    key_string : str
+        New key (with separator(s) in) for data
+    normalized_dict : dict
+        The new normalized/flattened Json dict
+    separator : str, default '.'
+        Nested records will generate names separated by sep,
+        e.g., for sep='.', { 'foo' : { 'bar' : 0 } } -> foo.bar
+    max_level
+        recursion depth
+    encoder
+        Custom JSON encoder; if not given, `json.dumps` is used.
+    """
+    if isinstance(data, dict):
+        if max_level > 0:
+            key_root = f"{key_string}{separator}" if key_string else ""
+            nested_max_level = max_level - 1
+
+            for key, value in data.items():
+                new_key = f"{key_root}{key}" if key_root else key
+                _normalize_json(
+                    data=value,
+                    key_string=new_key,
+                    normalized_dict=normalized_dict,
+                    separator=separator,
+                    max_level=nested_max_level,
+                    encoder=encoder,
+                )
+        else:
+            normalized_dict[key_string] = encoder(data)
+            return normalized_dict
+    else:
+        normalized_dict[key_string] = data
+    return normalized_dict
+
+
+def _normalize_json_ordered(
+    data: dict[str, Any],
+    separator: str,
+    max_level: int,
+    encoder: JSONEncoder,
+) -> dict[str, Any]:
+    """
+    Order the top level keys and then recursively go to depth.
+
+    Parameters
+    ----------
+    data
+        Deserialized JSON objects (dict or list of dicts)
+    separator
+        Nested records will generate names separated by sep. e.g.,
+        for `separator=".", {"foo": {"bar": 0}}` -> foo.bar.
+    max_level
+        Max number of levels(depth of dict) to normalize.
+    encoder
+        Custom JSON encoder; if not given, `json.dumps` is used.
+
+    Returns
+    -------
+    dict or list of dicts, matching `normalized_json_object`
+    """
+    top_, nested_data = {}, {}
+    for k, v in data.items():
+        if isinstance(v, dict):
+            nested_data[k] = v
+        else:
+            top_[k] = v
+
+    nested_ = _normalize_json(
+        data=nested_data,
+        key_string="",
+        normalized_dict={},
+        separator=separator,
+        max_level=max_level,
+        encoder=encoder,
+    )
+    return {**top_, **nested_}
+
+
+@unstable()
+def json_normalize(
+    data: dict[Any, Any] | Sequence[dict[Any, Any] | Any],
+    *,
+    separator: str = ".",
+    max_level: int | None = None,
+    schema: Schema | None = None,
+    strict: bool = True,
+    infer_schema_length: int | None = N_INFER_DEFAULT,
+    encoder: JSONEncoder | None = None,
+) -> DataFrame:
+    """
+    Normalize semi-structured deserialized JSON data into a flat table.
+
+    Dictionary objects that will not be unnested/normalized are encoded
+    as json string data. Unlike it pandas' counterpart, this function will
+    not encode dictionaries as objects at any level.
+
+    .. warning::
+        This functionality is considered **unstable**. It may be changed
+        at any point without it being considered a breaking change.
+
+    Parameters
+    ----------
+    data
+        Deserialized JSON objects.
+    separator
+        Nested records will generate names separated by sep. e.g.,
+        for `separator=".", {"foo": {"bar": 0}}` -> foo.bar.
+    max_level
+        Max number of levels(depth of dict) to normalize.
+        If None, normalizes all levels.
+    schema
+        Overwrite the `Schema` when the normalized data is passed to
+        the `DataFrame` constructor.
+    strict
+        Whether Polars should be strict when constructing the DataFrame.
+    infer_schema_length
+        Number of rows to take into consideration to determine the schema.
+    encoder
+        Custom JSON encoder function; if not given, `json.dumps` is used.
+
+    Examples
+    --------
+    >>> data = [
+    ...     {
+    ...         "id": 1,
+    ...         "name": "Cole Volk",
+    ...         "fitness": {"height": 180, "weight": 85},
+    ...     },
+    ...     {
+    ...         "id": 2,
+    ...         "name": "Faye Raker",
+    ...         "fitness": {"height": 155, "weight": 58},
+    ...     },
+    ...     {
+    ...         "name": "Mark Reg",
+    ...         "fitness": {"height": 170, "weight": 78},
+    ...     },
+    ... ]
+    >>> pl.json_normalize(data, max_level=1)
+    shape: (3, 4)
+    ┌──────┬────────────┬────────────────┬────────────────┐
+    │ id   ┆ name       ┆ fitness.height ┆ fitness.weight │
+    │ ---  ┆ ---        ┆ ---            ┆ ---            │
+    │ i64  ┆ str        ┆ i64            ┆ i64            │
+    ╞══════╪════════════╪════════════════╪════════════════╡
+    │ 1    ┆ Cole Volk  ┆ 180            ┆ 85             │
+    │ 2    ┆ Faye Raker ┆ 155            ┆ 58             │
+    │ null ┆ Mark Reg   ┆ 170            ┆ 78             │
+    └──────┴────────────┴────────────────┴────────────────┘
+
+    Normalize to a specific depth, using a custom JSON encoder
+    (note that `orson.dumps` encodes to bytes, not str).
+
+    >>> import orjson
+    >>> pl.json_normalize(data, max_level=0, encoder=orjson.dumps)
+    shape: (3, 3)
+    ┌──────┬────────────┬───────────────────────────────┐
+    │ id   ┆ name       ┆ fitness                       │
+    │ ---  ┆ ---        ┆ ---                           │
+    │ i64  ┆ str        ┆ binary                        │
+    ╞══════╪════════════╪═══════════════════════════════╡
+    │ 1    ┆ Cole Volk  ┆ b"{"height":180,"weight":85}" │
+    │ 2    ┆ Faye Raker ┆ b"{"height":155,"weight":58}" │
+    │ null ┆ Mark Reg   ┆ b"{"height":170,"weight":78}" │
+    └──────┴────────────┴───────────────────────────────┘
+    """
+    if max_level is None:
+        max_level = 1 << 32  # eg: u32
+    max_level += 1
+
+    if isinstance(data, Sequence) and len(data) == 0:
+        return DataFrame(schema=schema)
+    elif isinstance(data, Mapping):
+        data = [data]
+    elif isinstance(data, Iterable) and not isinstance(data, str):  # type: ignore[redundant-expr]
+        data = list(data)
+    else:
+        msg = "expected list or dict of objects"
+        raise ValueError(msg)
+
+    if encoder is None:
+        encoder = json.dumps
+
+    return DataFrame(
+        _simple_json_normalize(
+            data,
+            separator=separator,
+            max_level=max_level,
+            encoder=encoder,
+        ),
+        schema=schema,
+        strict=strict,
+        infer_schema_length=infer_schema_length,
+    )

polars/dataframe/__init__.py

@@ -0,0 +1,5 @@
+from polars.dataframe.frame import DataFrame
+
+__all__ = [
+    "DataFrame",
+]

polars/dataframe/_html.py

@@ -0,0 +1,186 @@
+"""Module for formatting output data in HTML."""
+
+from __future__ import annotations
+
+import os
+import re
+from textwrap import dedent
+from typing import TYPE_CHECKING
+
+from polars._dependencies import html
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+    from types import TracebackType
+
+    from polars import DataFrame
+
+
+def replace_consecutive_spaces(s: str) -> str:
+    """Replace consecutive spaces with HTML non-breaking spaces."""
+    return re.sub(r"( {2,})", lambda match: " " * len(match.group(0)), s)
+
+
+class Tag:
+    """Class for representing an HTML tag."""
+
+    def __init__(
+        self,
+        elements: list[str],
+        tag: str,
+        attributes: dict[str, str] | None = None,
+    ) -> None:
+        self.tag = tag
+        self.elements = elements
+        self.attributes = attributes
+
+    def __enter__(self) -> None:
+        if self.attributes is not None:
+            s = f"<{self.tag} "
+            for k, v in self.attributes.items():
+                s += f'{k}="{v}" '
+            s = f"{s.rstrip()}>"
+            self.elements.append(s)
+        else:
+            self.elements.append(f"<{self.tag}>")
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        self.elements.append(f"</{self.tag}>")
+
+
+class HTMLFormatter:
+    def __init__(
+        self,
+        df: DataFrame,
+        *,
+        max_cols: int = 75,
+        max_rows: int = 40,
+        from_series: bool = False,
+    ) -> None:
+        self.df = df
+        self.elements: list[str] = []
+        self.max_cols = max_cols
+        self.max_rows = max_rows
+        self.from_series = from_series
+        self.row_idx: Iterable[int]
+        self.col_idx: Iterable[int]
+
+        if max_rows < df.height:
+            half, rest = divmod(max_rows, 2)
+            self.row_idx = [
+                *list(range(half + rest)),
+                -1,
+                *list(range(df.height - half, df.height)),
+            ]
+        else:
+            self.row_idx = range(df.height)
+        if max_cols < df.width:
+            self.col_idx = [
+                *list(range(max_cols // 2)),
+                -1,
+                *list(range(df.width - max_cols // 2, df.width)),
+            ]
+        else:
+            self.col_idx = range(df.width)
+
+    def write_header(self) -> None:
+        """Write the header of an HTML table."""
+        with Tag(self.elements, "thead"):
+            if not bool(int(os.environ.get("POLARS_FMT_TABLE_HIDE_COLUMN_NAMES", "0"))):
+                with Tag(self.elements, "tr"):
+                    columns = self.df.columns
+                    for c in self.col_idx:
+                        with Tag(self.elements, "th"):
+                            if c == -1:
+                                self.elements.append("&hellip;")
+                            else:
+                                self.elements.append(html.escape(columns[c]))
+            if not bool(
+                int(os.environ.get("POLARS_FMT_TABLE_HIDE_COLUMN_DATA_TYPES", "0"))
+            ):
+                with Tag(self.elements, "tr"):
+                    dtypes = self.df._df.dtype_strings()
+                    for c in self.col_idx:
+                        with Tag(self.elements, "td"):
+                            if c == -1:
+                                self.elements.append("&hellip;")
+                            else:
+                                self.elements.append(dtypes[c])
+
+    def write_body(self) -> None:
+        """Write the body of an HTML table."""
+        str_len_limit = int(os.environ.get("POLARS_FMT_STR_LEN", default=30))
+        with Tag(self.elements, "tbody"):
+            for r in self.row_idx:
+                with Tag(self.elements, "tr"):
+                    for c in self.col_idx:
+                        with Tag(self.elements, "td"):
+                            if r == -1 or c == -1:
+                                self.elements.append("&hellip;")
+                            else:
+                                series = self.df[:, c]
+                                self.elements.append(
+                                    replace_consecutive_spaces(
+                                        html.escape(series._s.get_fmt(r, str_len_limit))
+                                    )
+                                )
+
+    def write(self, inner: str) -> None:
+        """Append a raw string to the inner HTML."""
+        self.elements.append(inner)
+
+    def render(self) -> list[str]:
+        """Return the lines needed to render a HTML table."""
+        if not bool(
+            int(
+                os.environ.get("POLARS_FMT_TABLE_HIDE_DATAFRAME_SHAPE_INFORMATION", "0")
+            )
+        ):
+            # format frame/series shape with '_' thousand-separators
+            s = self.df.shape
+            shape = f"({s[0]:_},)" if self.from_series else f"({s[0]:_}, {s[1]:_})"
+
+            self.elements.append(f"<small>shape: {shape}</small>")
+
+        with Tag(
+            # be careful changing the CSS class ref here...
+            # ref: https://github.com/pola-rs/polars/issues/7443
+            self.elements,
+            "table",
+            {"border": "1", "class": "dataframe"},
+        ):
+            self.write_header()
+            self.write_body()
+        return self.elements
+
+
+class NotebookFormatter(HTMLFormatter):
+    """
+    Class for formatting output data in HTML for display in Jupyter Notebooks.
+
+    This class is intended for functionality specific to DataFrame._repr_html_().
+    """
+
+    def write_style(self) -> None:
+        style = """\
+            <style>
+            .dataframe > thead > tr,
+            .dataframe > tbody > tr {
+              text-align: right;
+              white-space: pre-wrap;
+            }
+            </style>
+        """
+        self.write(dedent(style))
+
+    def render(self) -> list[str]:
+        """Return the lines needed to render a HTML table."""
+        with Tag(self.elements, "div"):
+            self.write_style()
+            super().render()
+        return self.elements