bs-python-utils 0.0.1__py3-none-any.whl

bs_python_utils/examples_altair.py

@@ -0,0 +1,195 @@
+"""examples using my Altair functions"""
+
+from pathlib import Path
+from typing import Callable
+
+import numpy as np
+import pandas as pd
+from vega_datasets import data
+
+from bs_python_utils.bsutils import mkdir_if_needed
+from bs_python_utils.bs_altair import (
+    alt_superposed_lineplot,
+    alt_superposed_faceted_lineplot,
+    alt_histogram_continuous,
+    alt_histogram_continuous,
+    alt_histogram_by,
+    alt_stacked_area,
+    alt_stacked_area_facets,
+    alt_scatterplot,
+    alt_linked_scatterplots,
+    alt_scatterplot_with_histo,
+    alt_density,
+    alt_faceted_densities,
+    alt_tick_plots,
+    alt_plot_fun,
+    plot_parameterized_estimates,
+    plot_true_sim2_facets,
+    plot_true_sim_facets,
+)
+
+
+FIG_DIRECTORY = "altair_figs"
+save_path = mkdir_if_needed(Path.cwd() / FIG_DIRECTORY)
+
+cars = data.cars()
+
+ch = alt_superposed_lineplot(
+    cars,
+    "Horsepower",
+    "Weight_in_lbs",
+    "Origin",
+    save=save_path / "cars_superposed_lineplot",
+)
+
+ch = alt_superposed_faceted_lineplot(
+    cars,
+    "Horsepower",
+    "Weight_in_lbs",
+    "Origin",
+    "Year",
+    save=save_path / "cars_superposed_faceted_lineplot",
+)
+
+ch = alt_histogram_continuous(cars, "Horsepower", save=save_path / "cars_histo_cont")
+
+ch = alt_histogram_by(
+    cars, "Origin", "Horsepower", str_agg="median", save=save_path / "cars_histo_by"
+)
+
+elec = data.iowa_electricity()
+
+ch = alt_stacked_area(
+    elec,
+    "year",
+    "net_generation",
+    "source",
+    time_series=True,
+    title="Generators",
+    save=save_path / "elec_stacked_areas",
+)
+
+ch = alt_stacked_area_facets(
+    cars,
+    "Year",
+    "Displacement",
+    "Name",
+    "Origin",
+    time_series=True,
+    save=save_path / "cars_stacked_areas_facets",
+)
+
+ch = alt_scatterplot(
+    cars,
+    "Year",
+    "Displacement",
+    time_series=True,
+    title="Average car displacement",
+    aggreg="average",
+    save=save_path / "cars_scatter",
+)
+
+ch = alt_scatterplot(
+    cars,
+    "Year",
+    "Displacement",
+    time_series=True,
+    title="Average car displacement",
+    aggreg="average",
+    save=save_path / "cars_scatter_labx",
+    xlabel="Model year",
+)
+
+ch = alt_scatterplot(
+    cars,
+    "Horsepower",
+    "Displacement",
+    title="Car displacement",
+    color="Origin",
+    selection=True,
+    save=save_path / "cars_scatter_color",
+    xlabel="Horsepower",
+)
+
+ch = alt_linked_scatterplots(
+    cars,
+    "Horsepower",
+    "Displacement",
+    "Miles_per_Gallon",
+    "Origin",
+    save=save_path / "cars_linked_scatters",
+)
+
+ch = alt_scatterplot_with_histo(
+    cars,
+    "Horsepower",
+    "Displacement",
+    "Origin",
+    save=save_path / "cars_linked_scatter_histo",
+)
+
+ch = alt_density(cars, "Horsepower", save=save_path / "horsepower_density")
+
+ch = alt_faceted_densities(
+    cars, "Horsepower", "Origin", save=save_path / "horsepower_distribs"
+)
+
+
+def fnp(x):
+    return x * x - 4.0
+
+
+ch = alt_plot_fun(fnp, -2.0, 3.0, save=save_path / "plot_function")
+
+# test plot_parameterized_estimates
+nvals = 50
+vals_p = np.arange(nvals) / (nvals - 1.0)
+true_vals = np.column_stack((vals_p, np.ones(nvals)))
+estimates_a = np.random.normal(size=((nvals, 2)), scale=0.2) + vals_p.reshape((-1, 1))
+estimates_b = np.random.normal(size=((nvals, 2)), scale=0.2) + np.ones((nvals, 2))
+estimates = np.zeros((nvals, 2, 2))
+estimates[..., 0] = estimates_a
+estimates[..., 1] = estimates_b
+
+ch = plot_parameterized_estimates(
+    "Value of p",
+    vals_p,
+    ["a", "b"],
+    true_vals,
+    ["MLE", "MM"],
+    estimates,
+    colors=["black", "green", "blue"],
+    save=save_path / "ppe.html",
+)
+
+stats = np.reshape(estimates, (nvals, 4))
+true_vals = stats + np.random.normal(loc=-0.1, scale=0.2, size=stats.shape)
+ch = plot_true_sim_facets(
+    "Value of p",
+    vals_p,
+    ["a", "b", "c", "d"],
+    true_vals,
+    stats,
+    colors=["black", "red"],
+    ncols=2,
+    save=save_path / "ptsf.html",
+)
+
+stats2 = stats + np.random.normal(loc=0.1, scale=0.2, size=stats.shape)
+ch = plot_true_sim2_facets(
+    "Value of p",
+    vals_p,
+    ["a", "b", "c", "d"],
+    true_vals,
+    stats,
+    stats2,
+    colors=["black", "red", "green"],
+    ncols=2,
+    save=save_path / "pts2f.html",
+)
+
+ch = alt_tick_plots(cars, "Weight_in_lbs", save=save_path / "weight_ticks")
+
+ch = alt_tick_plots(
+    cars, ["Horsepower", "Weight_in_lbs"], save=save_path / "horse_weight_ticks"
+)

bs_python_utils/examples_distance_covariances.py

@@ -0,0 +1,32 @@
+"""examples using distance_covariances"""
+
+import numpy as np
+
+from bs_python_utils.distance_covariances import pvalue_pdcov, dcov_dcor, pdcov_pdcor
+
+# example page 2396 of Szekely and Rizzo 2014
+n = 2000
+do_bootstrap = False
+Z1 = np.random.normal(size=n)
+Z2 = np.random.normal(size=n)
+Z3 = np.random.normal(size=n)
+X = Z1 + Z3
+Y = Z2 + Z3
+Z = Z3
+print("\n\n     Test of page 2396")
+dcov_XY = dcov_dcor(X, Y)
+dcov_XZ = dcov_dcor(X, Z)
+dcov_YZ = dcov_dcor(Y, Z)
+pdcov_XYZ = pdcov_pdcor(X, Y, Z)
+print(f"         dCor(X, Y)={dcov_XY.dcor}, should be 0.2062 in large samples")
+print(f"         dCor(X, Z)={dcov_XZ.dcor}, should be 0.4319 in large samples")
+print(f"         dCor(Y, Z)={dcov_YZ.dcor}, should be 0.4319 in large samples")
+print(f"         pdCor(X, Y; Z)={pdcov_XYZ.pdcor}, should be 0.0242 in large samples")
+
+if do_bootstrap:
+    ndraws = 499
+    pval = pvalue_pdcov(pdcov_XYZ)
+    print(
+        f"\n\n test stat={pdcov_XYZ.pdcov_stat: >.2f} has p-value {pval} for"
+        f" {ndraws} draws"
+    )

bs_python_utils/examples_mem.py

@@ -0,0 +1,25 @@
+""" example of using bs_mem"""
+import numpy as np
+import tracemalloc
+
+from bs_python_utils.bs_mem import (
+    memory_display_top,
+    memory_display_top_diffs,
+    memory_usage,
+)
+
+tracemalloc.start()
+list_ex = list(range(10000))
+
+
+v1 = np.ones(54546)
+snapshot1 = tracemalloc.take_snapshot()
+memory_display_top(snapshot1)
+d = {i: li for (i, li) in enumerate(list_ex)}
+m = np.random.normal(size=(3498, 12))
+memory_usage(5)
+snapshot2 = tracemalloc.take_snapshot()
+memory_display_top_diffs(snapshot1, snapshot2)
+tracemalloc.stop()
+del d
+memory_usage(5)

bs_python_utils/examples_seaborn.py

@@ -0,0 +1,37 @@
+"""examples using my Seaborn functions"""
+
+import seaborn as sns
+
+from bs_python_utils.bs_seaborn import (
+    bs_sns_bar_x_byf,
+    bs_sns_bar_x_byfg,
+    bs_sns_get_legend,
+)
+
+cars = sns.load_dataset("mpg")
+
+g1 = bs_sns_bar_x_byf(
+    cars,
+    "horsepower",
+    "cylinders",
+    label_x="Horsepower",
+    label_f="Number of cylinders",
+    title="Mean HP by number of cylinders",
+)
+
+g2 = bs_sns_bar_x_byfg(
+    cars,
+    "horsepower",
+    "cylinders",
+    "origin",
+    label_x="Horsepower",
+    label_f="Number of cylinders",
+    label_g="Origin",
+    title="Mean HP by number of cylinders and origin",
+)
+
+# change labels in legend
+l2 = bs_sns_get_legend(g2)
+labels2 = ["USA", "Japan", "Europe"]
+for t, lab in zip(l2.texts, labels2, strict=True):
+    t.set_text(lab)

bs_python_utils/examples_sklearn.py

@@ -0,0 +1,33 @@
+"""examples using my sklearn code"""
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.preprocessing import PolynomialFeatures, StandardScaler
+
+from bs_python_utils.sklearn_utils import skl_npreg_lasso
+
+
+n_obs = 10000
+X1 = -2.0 + 3.0 * np.random.uniform(size=n_obs)
+X2 = np.random.normal(loc=1.0, scale=2.0, size=n_obs)
+y = X1 * X2 * X2 / 100.0 - (X1 / 5.0 - X2 / 3.0) ** 3 + np.random.normal(size=n_obs)
+
+X = np.column_stack((X1, X2))
+
+plt.style.use("seaborn")
+
+degree = 10
+stdsc = StandardScaler()
+sfit = stdsc.fit(X)
+X_scaled = sfit.transform(X)
+pf = PolynomialFeatures(degree)
+# Create the features and fit
+X_poly = pf.fit_transform(X_scaled)
+
+y_pred = skl_npreg_lasso(y, X, alpha=0.001)
+
+plt.clf()
+
+ax = plt.axes()
+ax.scatter(y, y_pred)
+ax.plot(y, y, "-r")
+plt.show()

bs_python_utils/pandas_utils.py

@@ -0,0 +1,239 @@
+"""Utility functions for pandas """
+
+from itertools import product
+
+import numpy as np
+import pandas as pd
+
+from bs_python_utils.bsnputils import bs_error_abort, check_vector_or_matrix
+from bs_python_utils.bsutils import print_stars
+
+
+def bspd_print(
+    df: pd.DataFrame,
+    s: str | None = "",
+    max_rows: int | None = None,
+    max_cols: int | None = None,
+    precision: int | None = None,
+) -> None:
+    """Pretty-prints a data frame
+
+    Args:
+        df: any data frame
+        s: an optional title string
+        max_rows: maximum number of rows to print (all by default)
+        max_cols: maximum number of columns to print (all by default)
+        precision: of numbers. 3 digits by default.
+
+    Returns:
+        nothing.
+    """
+
+    print_stars(s)
+    with pd.option_context(
+        "display.max_rows",
+        max_rows,
+        "display.max_columns",
+        max_cols,
+        "display.precision",
+        precision,
+    ):
+        print(df)
+
+
+def bspd_cross_products(
+    df: pd.DataFrame,
+    l1: list[str],
+    l2: list[str] | None = None,
+    with_squares: bool | None = True,
+) -> pd.DataFrame:
+    """Returns a DataFrame with cross-products of the variables of `df`
+    whose names are in `l1` and `l2`.
+
+    Args:
+        df: any data frame
+        l1: a list of names of variables that belong to `df`
+        l2: ibidem; `l1` by default
+        with_squares: if `False`, we drop the squares. `True` by default.
+
+    Returns:
+        the data frame of cross-products with concatenated names.
+    """
+
+    lp2 = l1 if l2 is None else l2
+    l12 = list(product(l1, lp2))
+    cross_pairs = [[x[0], x[1]] for x in l12 if x[0] != x[1]]
+    unique_pairs = []
+    for _i, c in enumerate(cross_pairs):
+        print(c)
+        c_ordered = c if c[0] < c[1] else list(reversed(c))
+        print(c_ordered)
+        if c_ordered not in unique_pairs:
+            unique_pairs.append(c_ordered)
+        print(unique_pairs)
+
+    col_names = sorted([(x[0], x[1], f"{x[0]}*{x[1]}") for x in unique_pairs])
+
+    if with_squares:
+        col_names_squares = sorted(
+            [(x[0], x[1], f"{x[0]}**2") for x in l12 if x[0] == x[1]]
+        )
+        col_names += col_names_squares
+
+    df_cprods = pd.DataFrame(
+        {col_name: df[x0] * df[x1] for (x0, x1, col_name) in col_names}
+    )
+
+    return df_cprods
+
+
+def _list_str(names: str | list[str], suffix: str = None) -> list[str]:
+    """make a list of strings with possibly the added suffix
+
+    Args:
+        names: a string or a list of strings
+        suffix: a string, if any
+
+    Returns:
+        a list of the strings in names, with the suffix if specified
+    """
+    if isinstance(names, str):
+        if suffix is not None:
+            return [names + suffix]
+        else:
+            return [names]
+    elif isinstance(names, list):
+        if suffix is not None:
+            return [name + suffix for name in names]
+        else:
+            return names
+    else:
+        bs_error_abort("names should be a string or a list of strings.")
+
+
+def _check_colnames(col_names: str | list[str] | list[str | list[str]], n_T: int):
+    if not isinstance(col_names, list):
+        bs_error_abort("If T is a list, then col_names should be a list too.")
+    elif len(col_names) != n_T:
+        bs_error_abort(f"T has {n_T} elements but col_names has {len(col_names)}.")
+
+
+def bspd_statsdf(
+    T: np.ndarray | list[np.ndarray],
+    col_names: str | list[str] | list[str | list[str]],
+) -> pd.DataFrame:
+    """
+    make a dataframe with columns from the array(s) in T and names from col_names
+
+    Args:
+        T: a list of n_T matrices or vectors with N rows, or a matrix or a vector with N rows
+        col_names: a list of n_T name objects; a name object must be a string or a list of strings,
+            with the names for the column(s) of the corresponding T matrix
+
+    Returns:
+        a dataframe with the named columns
+    """
+    if isinstance(T, list):
+        n_T = len(T)
+        _check_colnames(col_names, n_T)
+        shape_T = []
+        for i in range(n_T):
+            shape_T.append(T[i].shape)
+        set_nrows = {shape_i[0] for shape_i in shape_T}
+        if len(set_nrows) > 1:
+            bs_error_abort("All T arrays should have the same number of rows.")
+        big_T = T[0]
+        big_names = _list_str(col_names[0], suffix="_1")
+        for i in range(1, n_T):
+            big_T = np.column_stack((big_T, T[i]))
+            big_names.extend(_list_str(col_names[i], suffix=f"_{i+1}"))
+
+        df = pd.DataFrame(big_T, columns=big_names, copy=True)
+
+    else:  # only one element in T
+        ndims_T = check_vector_or_matrix(T)
+        if ndims_T == 1:
+            if not isinstance(col_names, str):
+                bs_error_abort(f"T is a vector but col_names is {col_names}")
+            df = pd.DataFrame(T, columns=[col_names], copy=True)
+        elif ndims_T == 2:
+            N, K = T.shape
+            K2 = len(col_names)
+            if K2 != K:
+                bs_error_abort(f"T is {T.shape} but col_names has {K2} elements")
+            df = pd.DataFrame(T, columns=col_names, copy=True)
+
+    return df
+
+
+def _check_names_n(col_names: list[str]) -> list[int]:
+    """
+    Tests that if a name in col_names ends with `_n`, with `n` an integer, then all names do
+
+    Args:
+        col_names: a list of names
+
+    Returns:
+        the list of the values of `n`  if all names in the list end in `_n`; `[]` if none of them does.
+        Aborts otherwise.
+    """
+    underscore_n = []
+    ending_integers = []
+    for name in col_names:
+        split_name = name.split("_")
+        len_split = len(split_name)
+        if len_split > 1:  # found at least one '_'
+            last_bit = split_name[-1]
+            try:
+                ending_int = int(last_bit)
+                underscore_n.append(True)  # ends with '_n'
+                ending_integers.append(ending_int)
+            except ValueError:
+                underscore_n.append(False)
+        else:
+            underscore_n.append(False)
+
+    values_integers = set(ending_integers)
+    n_values_integers = len(values_integers)
+
+    if n_values_integers == 0 or all(
+        underscore_n
+    ):  # none ends in '_n' or all end in '_n'
+        return list(values_integers)
+    else:
+        bs_error_abort(
+            "If a column name ends with '_n' where n is an integer,  then all should."
+        )
+        return []  # for mypy
+
+
+def bspd_prepareplot(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Args:
+        df: any dataframe whose column names either all end in '_n' for n an integer, or none does
+
+    Returns:
+        a properly melted dataframe for plotting, with columns 'Sample', 'Statistic', 'Value',
+        and 'Group' if there are several integers
+    """
+    # check the names of the columns
+    values_integers = _check_names_n(df.columns)
+    n_values_integers = len(values_integers)
+
+    df2 = df.copy()
+    df2["Sample"] = np.arange(df.shape[0])
+    dfm = pd.melt(
+        df2,
+        id_vars="Sample",
+        value_vars=list(df.columns),
+        var_name="Statistic",
+        value_name="Value",
+    )
+    if n_values_integers in [0, 1]:
+        return dfm
+    else:  # at least two different groups of statistics
+        stat_group = dfm["Statistic"].str.split("_", n=1, expand=True)
+        dfm.drop(columns=["Statistic"], inplace=True)
+        dfm["Statistic"] = stat_group[0]
+        dfm["Group"] = stat_group[1]
+        return dfm

bs_python_utils/sklearn_utils.py

@@ -0,0 +1,74 @@
+"""
+Cntains various `scikit-learn` utility programs.
+"""
+
+from itertools import cycle
+from typing import cast
+
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.linear_model import Lasso, lasso_path
+from sklearn.preprocessing import PolynomialFeatures, StandardScaler
+
+
+def skl_npreg_lasso(
+    y: np.ndarray, X: np.ndarray, alpha: float, degree: int = 4
+) -> np.ndarray:
+    """
+    Lasso nonparametric regression of `y` over polynomials of `X`
+
+    Args:
+        y:  shape `(nobs)`
+        X: shape  `(nobs, nfeatures)`
+        alpha:  Lasso penalty parameter
+        degree: highest total degree
+
+    Returns:
+        the `(nobs)` array `E(y\\vert X)` over the sample
+    """
+
+    # first scale the X variables
+    stdsc = StandardScaler()
+    sfit = stdsc.fit(X)
+    X_scaled = sfit.transform(X)
+    pf = PolynomialFeatures(degree)
+    # Create the features and fit
+    X_poly = pf.fit_transform(X_scaled)
+    # now run Lasso
+    reg = Lasso(alpha=alpha).fit(X_poly, y)
+    expy_X = reg.predict(X_poly)
+    return cast(np.ndarray, expy_X)
+
+
+def plot_lasso_path(y: np.ndarray, X: np.ndarray, eps: float = 1e-3) -> None:
+    """
+    plot Lasso coefficient paths
+
+    Args:
+        y:  shape `(nobs)`
+        X: shape  `(nobs, nfeatures)`
+        eps: length of path
+
+    Returns:
+        plots the paths
+    """
+    # Compute paths
+    print("Computing regularization path using the lasso...")
+    alphas_lasso, coefs_lasso, _ = lasso_path(X, y, eps)
+
+    plt.clf()
+    # Display results
+    plt.figure(1)
+    colors = cycle(["b", "r", "g", "c", "k"])
+    neg_log_alphas_lasso = -np.log10(alphas_lasso)
+    for coef_l, c in zip(coefs_lasso, colors, strict=True):
+        plt.plot(neg_log_alphas_lasso, coef_l, c=c)
+
+    plt.xlabel("-Log(alpha)")
+    plt.ylabel("coefficients")
+    plt.title("Lasso Paths")
+    plt.axis("tight")
+
+    plt.show()
+
+    return

bs_python_utils-0.0.1.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023, Bernard Salanie
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.