dr-frames 0.1.0__py3-none-any.whl

dr_frames/__init__.py

@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+from .aggregation import (
+    aggregate_over_seeds,
+    apply_aggregations,
+    fillna_with_defaults,
+    get_constant_cols,
+    maybe_pipe,
+    unique_by_col,
+    unique_by_cols,
+    unique_non_null,
+)
+from .cells import (
+    apply_column_converters,
+    apply_if_column,
+    ensure_column,
+    fill_missing_values,
+    force_set_cell,
+    group_col_by_prefix,
+    map_column_with_fallback,
+    masked_getter,
+    masked_setter,
+    maybe_update_cell,
+    rename_columns,
+    require_row_index,
+)
+from .columns import (
+    apply_skip,
+    contained_cols,
+    drop_all_null_cols,
+    get_cols_by_contains,
+    get_cols_by_prefix,
+    move_cols_to_beginning,
+    move_cols_with_prefix_to_end,
+    move_numeric_cols_to_end,
+    remaining_cols,
+    strip_col_prefixes,
+    strip_col_prefixes_batch,
+)
+from .filtering import (
+    apply_filters_to_df,
+    filter_to_best_metric,
+    filter_to_range,
+    filter_to_value,
+    filter_to_values,
+    make_filter_fxn,
+    select_subset,
+)
+from .parsing import (
+    is_homogeneous,
+    parse_first_element,
+    parse_list_string,
+    sum_list_elements,
+)
+from .profiling import (
+    ColInfo,
+    DFColInfo,
+    infer_col_name_contains_tags,
+    infer_col_name_prefix_tags,
+    infer_col_name_suffix_tags,
+    infer_series_base_tag_type,
+    infer_tags_from_series_sample,
+    looks_like_json,
+    looks_like_path,
+)
+from .schema import (
+    ComputedField,
+    DataField,
+    DataFormat,
+    MetricDataField,
+)
+from .types import (
+    coerce_numeric_cols,
+    coerce_string_cols,
+    is_string_series,
+)
+
+try:
+    from .formatting import (  # noqa: F401
+        FORMATTER_TYPES,
+        OUTPUT_FORMATS,
+        format_coverage_table,
+        format_table,
+    )
+
+    _HAS_FORMATTING = True
+except ImportError:
+    _HAS_FORMATTING = False
+
+__all__ = [
+    "aggregate_over_seeds",
+    "apply_aggregations",
+    "apply_column_converters",
+    "apply_filters_to_df",
+    "apply_if_column",
+    "apply_skip",
+    "coerce_numeric_cols",
+    "coerce_string_cols",
+    "ColInfo",
+    "ComputedField",
+    "contained_cols",
+    "DataField",
+    "DataFormat",
+    "DFColInfo",
+    "drop_all_null_cols",
+    "ensure_column",
+    "fill_missing_values",
+    "fillna_with_defaults",
+    "filter_to_best_metric",
+    "filter_to_range",
+    "filter_to_value",
+    "filter_to_values",
+    "force_set_cell",
+    "get_cols_by_contains",
+    "get_cols_by_prefix",
+    "get_constant_cols",
+    "group_col_by_prefix",
+    "infer_col_name_contains_tags",
+    "infer_col_name_prefix_tags",
+    "infer_col_name_suffix_tags",
+    "infer_series_base_tag_type",
+    "infer_tags_from_series_sample",
+    "is_homogeneous",
+    "is_string_series",
+    "looks_like_json",
+    "looks_like_path",
+    "make_filter_fxn",
+    "map_column_with_fallback",
+    "masked_getter",
+    "masked_setter",
+    "maybe_pipe",
+    "maybe_update_cell",
+    "MetricDataField",
+    "move_cols_to_beginning",
+    "move_cols_with_prefix_to_end",
+    "move_numeric_cols_to_end",
+    "parse_first_element",
+    "parse_list_string",
+    "remaining_cols",
+    "rename_columns",
+    "require_row_index",
+    "select_subset",
+    "strip_col_prefixes",
+    "strip_col_prefixes_batch",
+    "sum_list_elements",
+    "unique_by_col",
+    "unique_by_cols",
+    "unique_non_null",
+]
+
+if _HAS_FORMATTING:
+    __all__.extend(
+        [
+            "format_table",
+            "format_coverage_table",
+            "FORMATTER_TYPES",
+            "OUTPUT_FORMATS",
+        ]
+    )
+
+__version__ = "0.1.0"

dr_frames/aggregation.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from typing import Any
+
+import pandas as pd
+
+from .columns import (
+    apply_skip,
+    contained_cols,
+    move_cols_to_beginning,
+    move_cols_with_prefix_to_end,
+)
+
+__all__ = [
+    "apply_aggregations",
+    "aggregate_over_seeds",
+    "unique_non_null",
+    "unique_by_col",
+    "unique_by_cols",
+    "get_constant_cols",
+    "fillna_with_defaults",
+    "maybe_pipe",
+]
+
+
+def unique_non_null(values: pd.Series | Iterable[Any]) -> list[Any]:
+    series = values if isinstance(values, pd.Series) else pd.Series(list(values))
+    return series.dropna().unique().tolist()
+
+
+def unique_by_col(df: pd.DataFrame, col: str) -> list[Any]:
+    return df[col].unique().tolist()
+
+
+def unique_by_cols(df: pd.DataFrame, cols: Sequence[str]) -> dict[str, Any]:
+    contained = contained_cols(df, cols)
+    return {col: unique_by_col(df, col) for col in contained}
+
+
+def get_constant_cols(df: pd.DataFrame, skip: Iterable[str] = ()) -> dict[str, Any]:
+    if df.empty or len(df) <= 1:
+        return {}
+    return {
+        c: df[c].iloc[0]
+        for c in apply_skip(df.columns, skip)
+        if df[c].nunique(dropna=False) <= 1
+    }
+
+
+def fillna_with_defaults(
+    df: pd.DataFrame,
+    defaults: Mapping[str, object] | Iterable[tuple[str, object]],
+) -> pd.DataFrame:
+    defaults_dict = dict(defaults)
+    if not defaults_dict:
+        return df
+    present = {c: value for c, value in defaults_dict.items() if c in df.columns}
+    return df.fillna(value=present) if present else df
+
+
+def maybe_pipe(
+    df: pd.DataFrame,
+    condition: bool | Callable[[pd.DataFrame], bool] | Iterable | Mapping,
+    func: Callable[..., pd.DataFrame],
+    *args: Any,
+    **kwargs: Any,
+) -> pd.DataFrame:
+    should_apply = condition(df) if callable(condition) else bool(condition)
+    return df.pipe(func, *args, **kwargs) if should_apply else df
+
+
+def apply_aggregations(
+    df: pd.DataFrame,
+    group_col: str,
+    agg_over_cols: Sequence[str],
+    drop_cols: Sequence[str] | None = None,
+    start_cols: Sequence[str] | None = None,
+    sort_cols: Sequence[str] | None = None,
+    end_prefix: str = "metrics_",
+) -> pd.DataFrame:
+    if df.empty:
+        return df
+    if group_col not in df.columns:
+        raise ValueError(f"Group column '{group_col}' not found in dataframe.")
+    if group_col in (drop_cols or []):
+        raise ValueError(f"Group column '{group_col}' cannot be in drop_cols.")
+
+    cols_to_drop = {*(drop_cols or []), *agg_over_cols}
+    numeric_cols = set(df.select_dtypes(include=["number"]).columns.tolist())
+    cols_to_use = set(df.columns) - cols_to_drop - {group_col}
+    mean_agg_cols = numeric_cols & cols_to_use
+    first_agg_cols = cols_to_use - mean_agg_cols
+
+    df = df.copy()
+    df = df.drop(columns=list(cols_to_drop))
+    df = (
+        df.groupby(group_col)
+        .agg(
+            {
+                **dict.fromkeys(first_agg_cols, "first"),
+                **dict.fromkeys(mean_agg_cols, "mean"),
+            }
+        )
+        .reset_index()
+    )
+    df = move_cols_with_prefix_to_end(df, prefix=end_prefix)
+    df = move_cols_to_beginning(df, [group_col, *(start_cols or [])])
+    if sort_cols:
+        df = df.sort_values(list(sort_cols))
+    return df
+
+
+def aggregate_over_seeds(
+    df: pd.DataFrame,
+    config_cols: list[str],
+    metric_cols: list[str] | None = None,
+    agg_funcs: list[str] | None = None,
+) -> pd.DataFrame:
+    assert config_cols, "config_cols must be provided"
+
+    if agg_funcs is None:
+        agg_funcs = ["mean", "std", "count"]
+
+    if metric_cols is None:
+        metric_cols = [col for col in df.columns if col.startswith("eval/")]
+
+    valid_config_cols = [c for c in config_cols if c in df.columns]
+
+    agg_dict = {metric: agg_funcs for metric in metric_cols if metric in df.columns}
+
+    if not agg_dict:
+        return df.groupby(valid_config_cols, dropna=False).first().reset_index()
+
+    aggregated = df.groupby(valid_config_cols, dropna=False).agg(agg_dict)
+    aggregated.columns = [f"{col}_{agg}" for col, agg in aggregated.columns]
+
+    return aggregated.reset_index()

dr_frames/cells.py

@@ -0,0 +1,262 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable, Mapping
+from typing import Any, cast
+
+import pandas as pd
+
+__all__ = [
+    "ensure_column",
+    "fill_missing_values",
+    "rename_columns",
+    "map_column_with_fallback",
+    "apply_column_converters",
+    "maybe_update_cell",
+    "force_set_cell",
+    "apply_if_column",
+    "masked_getter",
+    "masked_setter",
+    "require_row_index",
+    "group_col_by_prefix",
+]
+
+MissingMarkers = Iterable[Any]
+
+
+def ensure_column(
+    df: pd.DataFrame,
+    column: str,
+    default: Any,
+    *,
+    inplace: bool = False,
+) -> pd.DataFrame:
+    target = df if inplace else df.copy()
+    if column not in target.columns:
+        target[column] = default
+    elif default is None:
+        pass
+    else:
+        target[column] = target[column].fillna(default)
+    return target
+
+
+def fill_missing_values(
+    df: pd.DataFrame,
+    defaults: Mapping[str, Any],
+    *,
+    inplace: bool = False,
+) -> pd.DataFrame:
+    target = df if inplace else df.copy()
+    for column, default in defaults.items():
+        if column in target.columns:
+            target[column] = target[column].fillna(default)
+    return target
+
+
+def rename_columns(
+    df: pd.DataFrame,
+    mapping: Mapping[str, str],
+    *,
+    inplace: bool = False,
+) -> pd.DataFrame:
+    target = df if inplace else df.copy()
+    existing_map = {old: new for old, new in mapping.items() if old in target.columns}
+    if existing_map:
+        target = target.rename(columns=existing_map)
+    return target
+
+
+def map_column_with_fallback(
+    df: pd.DataFrame,
+    column: str,
+    mapping: Mapping[str, Any],
+    *,
+    inplace: bool = False,
+) -> pd.DataFrame:
+    if column not in df.columns:
+        return df if inplace else df.copy()
+
+    target = df if inplace else df.copy()
+
+    def _mapper(value: Any) -> Any:
+        if pd.isna(value):
+            return value
+        return mapping.get(value, value)
+
+    target[column] = target[column].map(_mapper)
+    return target
+
+
+def apply_column_converters(
+    df: pd.DataFrame,
+    converters: Mapping[str, Callable[[Any], Any]],
+    *,
+    inplace: bool = False,
+) -> pd.DataFrame:
+    target = df if inplace else df.copy()
+    for column, converter in converters.items():
+        if column in target.columns:
+            target[column] = target[column].apply(converter)
+    return target
+
+
+def _matches_missing_marker(current: Any, marker: Any) -> bool:
+    """Safely check if current value matches a missing marker.
+
+    Handles NaN values via pd.isna comparisons and catches TypeError
+    for unhashable types to ensure comparisons never raise.
+    """
+    try:
+        # Handle NaN comparisons first
+        if pd.isna(current) and pd.isna(marker):
+            return True
+        if pd.isna(current) or pd.isna(marker):
+            return False
+        # Safe equality check for hashable and unhashable types
+        return current == marker
+    except TypeError:
+        # Unhashable types (e.g., lists, dicts) can't be compared with ==
+        # in some contexts, so return False to be safe
+        return False
+
+
+def maybe_update_cell(
+    df: pd.DataFrame,
+    row_index: int,
+    column: str,
+    value: Any,
+    *,
+    missing_markers: MissingMarkers = (None, "N/A"),
+    inplace: bool = False,
+) -> pd.DataFrame:
+    target = df if inplace else df.copy()
+    if column not in target.columns or row_index not in target.index:
+        return target
+
+    current = target.loc[row_index, column]
+    is_missing = pd.isna(current) or any(
+        _matches_missing_marker(current, m) for m in missing_markers
+    )
+    if is_missing:
+        target.loc[row_index, column] = value
+    return target
+
+
+def force_set_cell(
+    df: pd.DataFrame,
+    row_index: int,
+    column: str,
+    value: Any,
+    *,
+    default: Any = None,
+    inplace: bool = False,
+) -> pd.DataFrame:
+    target = df if inplace else df.copy()
+    target = ensure_column(target, column, default, inplace=True)
+    target.loc[row_index, column] = value
+    return target
+
+
+def apply_if_column(
+    df: pd.DataFrame,
+    column: str,
+    func: Callable[[pd.Series], pd.Series],
+    *,
+    inplace: bool = False,
+) -> pd.DataFrame:
+    if column not in df.columns:
+        return df if inplace else df.copy()
+
+    target = df if inplace else df.copy()
+    target[column] = func(target[column])
+    return target
+
+
+def require_row_index(
+    df: pd.DataFrame,
+    column: str,
+    value: Any,
+) -> int:
+    matches = df.index[df[column] == value]
+    if len(matches) == 0:
+        raise ValueError(f"No rows found where {column} == {value!r}")
+    if len(matches) > 1:
+        raise ValueError(f"Multiple rows found where {column} == {value!r}")
+    return int(matches[0])
+
+
+def masked_getter(df: pd.DataFrame, mask: pd.Series, column: str) -> Any:
+    if column not in df.columns:
+        return None
+
+    selection = df.loc[mask, column]
+    if selection.empty:
+        return None
+    return selection.iloc[0]
+
+
+def masked_setter(
+    df: pd.DataFrame,
+    mask: pd.Series,
+    column: str,
+    value: Any,
+    *,
+    inplace: bool = False,
+) -> pd.DataFrame:
+    target = df if inplace else df.copy()
+    target.loc[mask, column] = value
+    return target
+
+
+def _normalize_prefix_items(
+    prefix_map: Mapping[str, str] | Iterable[tuple[str, str]] | None,
+) -> list[tuple[str, str]]:
+    if prefix_map is None:
+        return []
+    items = list(prefix_map.items() if isinstance(prefix_map, Mapping) else prefix_map)
+    assert all(isinstance(item, tuple) and len(item) == 2 for item in items), (
+        f"Prefix map must be a mapping or iterable of tuples, received {type(prefix_map)!r}."
+    )
+
+    normalized: list[tuple[str, str]] = []
+    for prefix, group in items:
+        assert isinstance(prefix, str), (
+            f"Prefix keys must be strings, received {type(prefix)!r}."
+        )
+        assert isinstance(group, str), (
+            f"Group names must be strings, received {type(group)!r}."
+        )
+        lowered = prefix.strip().lower()
+        assert lowered, "Prefix keys must be non-empty after stripping whitespace."
+        normalized.append((lowered, group))
+    normalized.sort(key=lambda item: (-len(item[0]), item[0]))
+    return normalized
+
+
+def group_col_by_prefix(
+    df: pd.DataFrame,
+    column: str,
+    prefix_map: Mapping[str, str] | Iterable[tuple[str, str]] | None,
+    *,
+    output_col: str,
+) -> pd.Series:
+    assert column in df.columns, f"Column '{column}' not present in DataFrame."
+
+    series = cast(pd.Series, df[column])
+    normalized_prefixes = _normalize_prefix_items(prefix_map)
+
+    if not normalized_prefixes:
+        return series.copy().rename(output_col)
+
+    def resolve(value: object) -> object:
+        if bool(pd.isna(value)):
+            return value
+        if isinstance(value, str):
+            lowered = value.lower()
+            for prefix, group in normalized_prefixes:
+                if lowered.startswith(prefix):
+                    return group
+            return value
+        return value
+
+    return series.map(resolve).rename(output_col)

dr_frames/columns.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping, Sequence
+
+import pandas as pd
+
+from dr_frames.types import is_string_series
+
+__all__ = [
+    "apply_skip",
+    "contained_cols",
+    "remaining_cols",
+    "get_cols_by_prefix",
+    "get_cols_by_contains",
+    "strip_col_prefixes",
+    "strip_col_prefixes_batch",
+    "move_cols_to_beginning",
+    "move_numeric_cols_to_end",
+    "move_cols_with_prefix_to_end",
+    "drop_all_null_cols",
+]
+
+
+def _strip_prefix(text: str, prefix: str) -> str:
+    return text[len(prefix) :] if text.startswith(prefix) else text
+
+
+def apply_skip(
+    columns: Sequence[str] | pd.Index, skip: Iterable[str] = ()
+) -> list[str]:
+    skip_set = set(skip)
+    return [column for column in columns if column not in skip_set]
+
+
+def contained_cols(df: pd.DataFrame, columns: Sequence[str]) -> list[str]:
+    return [column for column in columns if column in df.columns]
+
+
+def remaining_cols(df: pd.DataFrame, cols: Iterable[str]) -> list[str]:
+    skip_set = set(cols)
+    return [column for column in df.columns if column not in skip_set]
+
+
+def get_cols_by_prefix(
+    df: pd.DataFrame, prefix: str, skip: Iterable[str] = ()
+) -> list[str]:
+    return [c for c in apply_skip(df.columns, skip) if c.startswith(prefix)]
+
+
+def get_cols_by_contains(
+    df: pd.DataFrame, substr: str, skip: Iterable[str] = ()
+) -> list[str]:
+    return [c for c in apply_skip(df.columns, skip) if substr in c]
+
+
+def strip_col_prefixes(
+    df: pd.DataFrame, prefix: str, skip: Iterable[str] = ()
+) -> pd.DataFrame:
+    return df.rename(
+        columns={
+            c: _strip_prefix(c, prefix) for c in get_cols_by_prefix(df, prefix, skip)
+        }
+    )
+
+
+def strip_col_prefixes_batch(
+    df: pd.DataFrame,
+    prefix_map: Mapping[str, Iterable[str]] | None = None,
+) -> pd.DataFrame:
+    if not prefix_map:
+        return df
+
+    sorted_items = sorted(prefix_map.items(), key=lambda x: len(x[0]), reverse=True)
+
+    working = df
+    for prefix, skip in sorted_items:
+        working = strip_col_prefixes(working, prefix, skip)
+    return working
+
+
+def move_cols_to_beginning(df: pd.DataFrame, cols: list[str]) -> pd.DataFrame:
+    return df.loc[:, [*contained_cols(df, cols), *remaining_cols(df, cols)]]
+
+
+def move_numeric_cols_to_end(df: pd.DataFrame) -> pd.DataFrame:
+    numeric_columns = df.select_dtypes(include=["number"]).columns.tolist()
+    return df.loc[:, [*remaining_cols(df, numeric_columns), *numeric_columns]]
+
+
+def move_cols_with_prefix_to_end(
+    df: pd.DataFrame, prefix: str, skip: Iterable[str] = ()
+) -> pd.DataFrame:
+    target_columns = get_cols_by_prefix(df, prefix, skip)
+    return df.loc[:, [*remaining_cols(df, target_columns), *target_columns]]
+
+
+def drop_all_null_cols(df: pd.DataFrame) -> pd.DataFrame:
+    working = df.copy()
+    object_cols = working.select_dtypes(include=["object", "string"])
+    blank_mask = pd.DataFrame(False, index=working.index, columns=working.columns)
+    if not object_cols.empty:
+        string_cols = [c for c, col in object_cols.items() if is_string_series(col)]
+        if string_cols:
+            blank_mask[string_cols] = object_cols[string_cols].apply(
+                lambda col: col.str.strip() == ""
+            )
+    working = working.mask(blank_mask, other=pd.NA)
+    return working.dropna(axis=1, how="all")