dr-frames 0.1.0__py3-none-any.whl

dr_frames/schema.py

@@ -0,0 +1,244 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Literal
+
+import pandas as pd
+from pydantic import BaseModel, Field, computed_field, model_validator
+
+__all__ = [
+    "DataField",
+    "ComputedField",
+    "MetricDataField",
+    "DataFormat",
+]
+
+
+class DataField(BaseModel):
+    id_string: str
+    description: str | None = None
+    column_name: str | None = None
+    display_name: str | None = None
+    altair_type: Literal["Q", "N", "O", "T"] | None = None
+    scale_hint: Literal["linear", "log"] | None = None
+    is_config: bool = True
+
+    @model_validator(mode="after")
+    def derive_display_name(self) -> DataField:
+        if self.display_name is None:
+            object.__setattr__(
+                self,
+                "display_name",
+                self.id_string.replace("_", " ").replace(".", " ").title(),
+            )
+        return self
+
+    @computed_field
+    @property
+    def is_resolved(self) -> bool:
+        return self.column_name is not None
+
+    def resolve_column(self, df: pd.DataFrame) -> str:
+        if self.column_name:
+            return self.column_name
+        if self.id_string in df.columns:
+            return self.id_string
+        raise ValueError(
+            f"Cannot resolve column for field with id_string='{self.id_string}'. "
+            f"Neither column_name nor id_string '{self.id_string}' found in DataFrame columns. "
+            f"Available columns: {list(df.columns)}"
+        )
+
+    def infer_altair_type(self, df: pd.DataFrame) -> str:
+        if self.altair_type:
+            return self.altair_type
+        try:
+            col = self.resolve_column(df)
+        except ValueError:
+            return "N"
+        dtype = df[col].dtype
+        if pd.api.types.is_numeric_dtype(dtype):
+            return "Q"
+        if pd.api.types.is_datetime64_any_dtype(dtype):
+            return "T"
+        return "N"
+
+
+class ComputedField(DataField):
+    source_columns: list[str] = Field(default_factory=list)
+    compute: Callable[[pd.DataFrame], pd.Series] = Field(exclude=True)
+
+    model_config = {"arbitrary_types_allowed": True}
+
+    def apply(self, df: pd.DataFrame) -> pd.Series:
+        return self.compute(df)
+
+
+class MetricDataField(DataField):
+    group: str = ""
+    metric_type: str = ""
+
+    @classmethod
+    def from_column_name(cls, col: str) -> MetricDataField:
+        parts = col.split("/")
+        if len(parts) >= 4:
+            group = parts[1]
+            metric_type = parts[-1]
+            display = " ".join(parts[2:]).replace("(", "").replace(")", "")
+        else:
+            group = "unknown"
+            metric_type = col
+            display = col
+        return cls(
+            id_string=col,
+            column_name=col,
+            display_name=display,
+            group=group,
+            metric_type=metric_type,
+            altair_type="Q",
+        )
+
+
+class DataFormat(BaseModel):
+    fields: list[DataField] = Field(default_factory=list)
+    computed_fields: list[ComputedField] = Field(default_factory=list)
+    metrics: list[MetricDataField] = Field(default_factory=list)
+    column_overrides: dict[str, str] = Field(default_factory=dict)
+    metric_prefix: str = "eval/"
+
+    @classmethod
+    def from_dict(
+        cls,
+        field_descriptions: dict[str, str],
+        df: pd.DataFrame,
+        column_overrides: dict[str, str] | None = None,
+    ) -> DataFormat:
+        overrides = column_overrides or {}
+        fields = [
+            DataField(
+                id_string=k,
+                description=v,
+                column_name=overrides.get(k) or (k if k in df.columns else None),
+            )
+            for k, v in field_descriptions.items()
+        ]
+        return cls(fields=fields, column_overrides=overrides)
+
+    @classmethod
+    def from_df(
+        cls,
+        df: pd.DataFrame,
+        field_descriptions: dict[str, str] | None = None,
+        computed_fields: list[ComputedField] | None = None,
+        column_overrides: dict[str, str] | None = None,
+        metric_prefix: str = "eval/",
+    ) -> DataFormat:
+        overrides = column_overrides or {}
+        defaults = cls()
+
+        if field_descriptions is not None:
+            fields = [
+                DataField(
+                    id_string=k,
+                    description=v,
+                    column_name=overrides.get(k) or (k if k in df.columns else None),
+                )
+                for k, v in field_descriptions.items()
+            ]
+        else:
+            fields = [
+                field.model_copy(
+                    update={
+                        "column_name": overrides.get(field.id_string)
+                        or (field.id_string if field.id_string in df.columns else None)
+                    }
+                )
+                if field.column_name is None
+                else field
+                for field in defaults.fields
+            ]
+
+        cf_list = (
+            computed_fields if computed_fields is not None else defaults.computed_fields
+        )
+
+        metrics = [
+            MetricDataField.from_column_name(col)
+            for col in df.columns
+            if col.startswith(metric_prefix)
+        ]
+
+        return cls(
+            fields=fields,
+            computed_fields=cf_list,
+            metrics=metrics,
+            column_overrides=overrides,
+            metric_prefix=metric_prefix,
+        )
+
+    @property
+    def unresolved_fields(self) -> list[DataField]:
+        return [f for f in self.fields if not f.is_resolved]
+
+    @computed_field
+    @property
+    def is_fully_resolved(self) -> bool:
+        return all(f.is_resolved for f in self.fields)
+
+    def prepare_for_plotting(
+        self, df: pd.DataFrame, drop_unknown: bool = True
+    ) -> pd.DataFrame:
+        result = df.copy()
+        for cf in self.computed_fields:
+            col_name = cf.column_name or cf.id_string
+            result[col_name] = cf.apply(result)
+
+        if drop_unknown:
+            known_cols = set()
+            for f in self.fields:
+                known_cols.add(f.column_name or f.id_string)
+            for cf in self.computed_fields:
+                known_cols.add(cf.column_name or cf.id_string)
+            for m in self.metrics:
+                if m.column_name:
+                    known_cols.add(m.column_name)
+            keep_cols = [c for c in result.columns if c in known_cols]
+            result = result[keep_cols]
+
+        return result
+
+    def get_metric(self, pattern: str) -> MetricDataField | None:
+        for m in self.metrics:
+            if pattern in (m.column_name or "") or pattern in (m.display_name or ""):
+                return m
+        return None
+
+    def metric_col(self, pattern: str) -> str:
+        metric = self.get_metric(pattern)
+        if metric is None:
+            raise ValueError(f"No metric found matching '{pattern}'")
+        if metric.column_name is None:
+            raise ValueError(
+                f"Metric matching '{pattern}' exists but has no column_name"
+            )
+        return metric.column_name
+
+    def get_metrics(self, df: pd.DataFrame) -> list[str]:
+        return [col for col in df.columns if col.startswith(self.metric_prefix)]
+
+    def get_config_columns(self, use_computed: bool = True) -> list[str]:
+        if use_computed:
+            config_cols = [cf.id_string for cf in self.computed_fields if cf.is_config]
+
+            computed_sources = set()
+            for cf in self.computed_fields:
+                if cf.is_config:
+                    computed_sources.update(cf.source_columns)
+
+            for f in self.fields:
+                if f.is_config and f.id_string not in computed_sources:
+                    config_cols.append(f.id_string)
+
+            return config_cols
+        else:
+            return [f.id_string for f in self.fields if f.is_config]

dr_frames/types.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from typing import cast
+
+import numpy as np
+import pandas as pd
+
+__all__ = [
+    "coerce_numeric_cols",
+    "coerce_string_cols",
+    "is_string_series",
+]
+
+
+def is_string_series(series: pd.Series) -> bool:
+    non_null = series.dropna()
+    if len(non_null) == 0:
+        return False
+    return bool(non_null.map(lambda x: isinstance(x, str)).all())
+
+
+def coerce_numeric_cols(
+    df: pd.DataFrame,
+    columns: Sequence[str] | Iterable[str],
+    dtype: type[float] | type[int] = float,
+) -> pd.DataFrame:
+    columns_list = list(columns)
+    working = df.copy()
+    if not columns_list:
+        return working
+    for c in columns_list:
+        if c not in working.columns:
+            continue
+        coerced = cast(pd.Series, pd.to_numeric(working[c], errors="coerce"))
+        if dtype is int:
+            non_null = coerced.dropna()
+            if not np.isclose(non_null, non_null.astype(int)).all():
+                raise ValueError(
+                    f"Column '{c}' contains non-integer values after coercion."
+                )
+            target_dtype: object = "Int64" if coerced.isna().any() else int
+        else:
+            target_dtype = dtype
+        working[c] = coerced.astype(target_dtype)
+    return working
+
+
+def coerce_string_cols(
+    df: pd.DataFrame,
+    columns: Sequence[str] | Iterable[str],
+) -> pd.DataFrame:
+    columns_list = list(columns)
+    if not columns_list:
+        return df.copy()
+    working = df.copy()
+    for c in columns_list:
+        if c not in working.columns:
+            continue
+        working[c] = working[c].astype("string")
+    return working

dr_frames-0.1.0.dist-info/METADATA

@@ -0,0 +1,207 @@
+Metadata-Version: 2.4
+Name: dr-frames
+Version: 0.1.0
+Summary: Pandas/DataFrame utilities for data manipulation, filtering, aggregation, and schema management
+Project-URL: Homepage, https://github.com/drothermel/dr_frames
+Project-URL: Repository, https://github.com/drothermel/dr_frames
+Author-email: Danielle Rothermel <danielle.rothermel@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: aggregation,data-manipulation,dataframe,filtering,pandas,schema
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: pydantic>=2.0.0
+Provides-Extra: formatting
+Requires-Dist: pyyaml>=6.0.0; extra == 'formatting'
+Requires-Dist: rich>=13.0.0; extra == 'formatting'
+Requires-Dist: tabulate>=0.9.0; extra == 'formatting'
+Description-Content-Type: text/markdown
+
+# dr_frames
+
+Pandas/DataFrame utilities for data manipulation, filtering, aggregation, and schema management.
+
+## Installation
+
+```bash
+pip install dr-frames
+```
+
+For table formatting features (console, markdown, latex):
+```bash
+pip install dr-frames[formatting]
+```
+
+## Quick Start
+
+```python
+import pandas as pd
+from dr_frames import (
+    coerce_numeric_cols,
+    filter_to_range,
+    move_cols_to_beginning,
+    select_subset,
+)
+
+df = pd.DataFrame({
+    "name": ["alice", "bob", "charlie"],
+    "value": ["1.0", "2.0", "3.0"],
+    "category": ["x", "y", "x"],
+})
+
+result = (
+    df.pipe(coerce_numeric_cols, ["value"])
+    .pipe(select_subset, {"category": "x"})
+    .pipe(filter_to_range, "value", 0.5, 2.5)
+)
+```
+
+## Module Overview
+
+| Module | Purpose | Key Functions |
+|--------|---------|---------------|
+| **columns** | Column selection & reordering | `move_cols_to_beginning`, `get_cols_by_prefix`, `strip_col_prefixes` |
+| **filtering** | Row filtering | `select_subset`, `filter_to_range`, `make_filter_fxn` |
+| **cells** | Cell-level operations | `ensure_column`, `map_column_with_fallback`, `force_set_cell` |
+| **types** | Type coercion | `coerce_numeric_cols`, `coerce_string_cols` |
+| **aggregation** | GroupBy & reduction | `aggregate_over_seeds`, `apply_aggregations`, `unique_non_null` |
+| **parsing** | String list parsing | `parse_first_element`, `sum_list_elements`, `is_homogeneous` |
+| **schema** | Data field metadata | `DataField`, `ComputedField`, `DataFormat` |
+| **profiling** | Column auto-tagging | `DFColInfo`, `ColInfo`, `looks_like_json` |
+| **formatting** | Table output | `format_table`, `format_coverage_table` |
+
+## Documentation
+
+- [Full API Reference](docs/api.md)
+- Module guides: [columns](docs/columns.md) | [filtering](docs/filtering.md) | [cells](docs/cells.md) | [types](docs/types.md) | [aggregation](docs/aggregation.md) | [parsing](docs/parsing.md) | [schema](docs/schema.md) | [profiling](docs/profiling.md) | [formatting](docs/formatting.md)
+- [Recipes & Patterns](docs/recipes.md)
+
+### Auto-generated API Docs
+
+```bash
+# Serve interactive docs locally
+uv run pdoc dr_frames
+
+# Generate static HTML
+uv run pdoc dr_frames -o docs/api_html
+```
+
+## Quick Reference
+
+### Column Operations
+```python
+from dr_frames import (
+    contained_cols,          # cols that exist in df
+    remaining_cols,          # cols NOT in a list
+    get_cols_by_prefix,      # cols starting with prefix
+    get_cols_by_contains,    # cols containing substring
+    move_cols_to_beginning,  # reorder cols
+    move_cols_with_prefix_to_end,
+    strip_col_prefixes,      # rename by removing prefix
+    drop_all_null_cols,      # remove empty columns
+)
+```
+
+### Filtering
+```python
+from dr_frames import (
+    select_subset,           # filter by exact column values
+    apply_filters_to_df,     # filter by value lists
+    filter_to_value,         # single value filter
+    filter_to_values,        # multi-value filter
+    filter_to_range,         # numeric range filter
+    filter_to_best_metric,   # keep best per group
+    make_filter_fxn,         # compose filters
+)
+```
+
+### Cell Operations
+```python
+from dr_frames import (
+    ensure_column,           # add column if missing
+    fill_missing_values,     # fillna with defaults dict
+    rename_columns,          # safe rename (skips missing)
+    map_column_with_fallback,# map values, keep unmapped
+    apply_column_converters, # apply functions to columns
+    maybe_update_cell,       # update if currently null
+    force_set_cell,          # always update
+    masked_getter,           # get value where mask is true
+    masked_setter,           # set value where mask is true
+)
+```
+
+### Type Coercion
+```python
+from dr_frames import (
+    coerce_numeric_cols,     # convert to float/int
+    coerce_string_cols,      # convert to string dtype
+    is_string_series,        # check if series is strings
+)
+```
+
+### Aggregation
+```python
+from dr_frames import (
+    aggregate_over_seeds,    # mean/std/count by config
+    apply_aggregations,      # flexible groupby
+    unique_non_null,         # unique values excluding null
+    unique_by_col,           # unique values in column
+    get_constant_cols,       # cols with single value
+    fillna_with_defaults,    # fill nulls from dict
+    maybe_pipe,              # conditional pipe
+)
+```
+
+### Parsing
+```python
+from dr_frames import (
+    parse_list_string,       # "[1,2,3]" -> [1,2,3]
+    parse_first_element,     # "[1,2,3]" -> 1.0
+    sum_list_elements,       # "[1,2,3]" -> 6.0
+    is_homogeneous,          # "[1,1,1]" -> True
+)
+```
+
+### Schema
+```python
+from dr_frames import (
+    DataField,               # field with metadata
+    ComputedField,           # derived field
+    MetricDataField,         # metric with group info
+    DataFormat,              # container for fields
+)
+```
+
+### Profiling
+```python
+from dr_frames import (
+    DFColInfo,               # catalog of column info
+    ColInfo,                 # single column metadata
+    looks_like_json,         # detect JSON strings
+    looks_like_path,         # detect file paths
+    infer_series_base_tag_type,  # infer dtype tags
+)
+```
+
+### Formatting (requires `[formatting]` extra)
+```python
+from dr_frames import (
+    format_table,            # render table in multiple formats
+    format_coverage_table,   # show column coverage stats
+    FORMATTER_TYPES,         # available formatters
+    OUTPUT_FORMATS,          # available output formats
+)
+```
+
+## License
+
+MIT

dr_frames-0.1.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+dr_frames/__init__.py,sha256=tZ1ECxdSo7oUuX7gWQrsiUFNKf8OvA7-Ly33BzMrzFA,3516
+dr_frames/aggregation.py,sha256=59Hl0Iw6Qgt4yl2mnrPgZ-5k9uaiUbkecTuSTNv4rbo,4151
+dr_frames/cells.py,sha256=OtNSWN5qTknwwDuDXMujFQz8MjFdFpStsRIeIg6_xh8,7086
+dr_frames/columns.py,sha256=mk3O5C4AjI31y3BC_8uPlLQfyxBMlDhNaKIbdXeiE-Y,3324
+dr_frames/filtering.py,sha256=Mi-9R4dWXIMzszRySHq3RRRK7m8l9gpmp-5QiE2S76Y,2756
+dr_frames/formatting.py,sha256=95GvlCu1PusB74ZTubdDqjOjVPqZn3l7-tE85M_2EcI,8309
+dr_frames/parsing.py,sha256=_YteOY0Uvh1ndZesdaU8412H1cTSPhffB2gzR-eMEdY,1720
+dr_frames/profiling.py,sha256=Wbg22et-gZGmTTeSYDo9GIzDHV_KpQP9Ne0QkyAHMVA,7029
+dr_frames/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dr_frames/schema.py,sha256=TDDP1Cv8L07ug8Sgo1me8vayX_fDhwOiR2hRFqDyY1g,7834
+dr_frames/types.py,sha256=Ce1D0Cgf6bE47d9cl9G8OmrQeQMnMEL2g4_h-UhXvyA,1688
+dr_frames-0.1.0.dist-info/METADATA,sha256=vOhfw7pPpVExWG4rrgptrIQEBzwrOkWfLUQPe06d7UM,6665
+dr_frames-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+dr_frames-0.1.0.dist-info/licenses/LICENSE,sha256=6tUm1Q55M1UBMbbawzFlF0-DgCazM1BELo_5-RXA1K4,1075
+dr_frames-0.1.0.dist-info/RECORD,,

dr_frames-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any

dr_frames-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Danielle Rothermel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.