process-improve 0.9.76__tar.gz → 0.9.78__tar.gz

{process_improve-0.9.76/process_improve.egg-info → process_improve-0.9.78}/PKG-INFO

@@ -1,15 +1,14 @@
 Metadata-Version: 2.2
 Name: process-improve
-Version: 0.9.76
+Version: 0.9.78
 Summary: Designed Experiments; Latent Variables (PCA, PLS, multivariate methods with missing data); Process Monitoring; Batch data analysis.
-Home-page: https://github.com/kgdunn/process_improve
-Author: Kevin Dunn
 Author-email: Kevin Dunn <kgdunn@gmail.com>
+License: MIT license
 Project-URL: Homepage, https://github.com/kgdunn/process_improve
 Project-URL: Repository, https://github.com/kgdunn/process_improve
 Project-URL: Issues, https://github.com/kgdunn/process_improve/issues
 Keywords: Designed Experiments,Latent Variables,PCA,PLS,Multivariate Data Analysis,Batch data analysis
-Requires-Python: >=3.8
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: bokeh>=3.6.3
@@ -26,9 +25,6 @@ Requires-Dist: scikit-learn>=1.6.1
 Requires-Dist: seaborn>=0.13.2
 Requires-Dist: statsmodels>=0.14.4
 Requires-Dist: tqdm>=4.67.1
-Dynamic: author
-Dynamic: home-page
-Dynamic: requires-python
 
 # Process Improvement (using Data)
 

{process_improve-0.9.76 → process_improve-0.9.78}/process_improve/__init__.py

@@ -1,3 +1 @@
 """(c) Kevin Dunn, 2010-2025. MIT License."""
-
-__version__ = "0.9.76"

{process_improve-0.9.76 → process_improve-0.9.78}/process_improve/experiments/datasets.py

@@ -1,4 +1,4 @@
-# (c) Kevin Dunn, 2010-2024. MIT License. Based on own private work over the years.
+# (c) Kevin Dunn, 2010-2025. MIT License. Based on own private work over the years.
 
 import pandas as pd
 

{process_improve-0.9.76 → process_improve-0.9.78}/process_improve/experiments/designs_factorial.py

@@ -1,4 +1,4 @@
-# (c) Kevin Dunn, 2010-2024. MIT License.
+# (c) Kevin Dunn, 2010-2025. MIT License.
 
 """Various factorial designs"""
 

{process_improve-0.9.76 → process_improve-0.9.78}/process_improve/experiments/models.py

@@ -1,4 +1,4 @@
-# (c) Kevin Dunn, 2010-2024. MIT License. Based on own private work over the years.
+# (c) Kevin Dunn, 2010-2025. MIT License. Based on own private work over the years.
 
 import warnings
 from collections import defaultdict

{process_improve-0.9.76 → process_improve-0.9.78}/process_improve/experiments/plotting.py

@@ -1,4 +1,4 @@
-# (c) Kevin Dunn, 2010-2024. MIT License.
+# (c) Kevin Dunn, 2010-2025. MIT License.
 import webbrowser
 
 import numpy as np

{process_improve-0.9.76 → process_improve-0.9.78}/process_improve/experiments/simulations.py

@@ -1,4 +1,4 @@
-# (c) Kevin Dunn, 2010-2024. MIT License. Based on own private work over the years.
+# (c) Kevin Dunn, 2010-2025. MIT License. Based on own private work over the years.
 
 import pandas as pd
 from numpy.random import normal
@@ -30,7 +30,7 @@ def popcorn(t=120, T=None):
 
     Source
     ------
-    Kevin Dunn, Process Improvement using Data, Chapter 5, 2010 to 2024,
+    Kevin Dunn, Process Improvement using Data, Chapter 5, 2010 to 2025,
     https://learnche.org/pid
 
 
@@ -70,7 +70,7 @@ def grocery(p=3.46, h=150, P=None, H=None):
 
     Source
     ------
-    Kevin Dunn, Process Improvement using Data, Chapter 5, 2010 to 2024,
+    Kevin Dunn, Process Improvement using Data, Chapter 5, 2010 to 2025,
     https://learnche.org/pid
 
     """

{process_improve-0.9.76 → process_improve-0.9.78}/process_improve/experiments/structures.py

@@ -1,4 +1,4 @@
-# (c) Kevin Dunn, 2010-2024. MIT License. Based on own private work over the years.
+# (c) Kevin Dunn, 2010-2025. MIT License. Based on own private work over the years.
 
 import itertools
 from collections import defaultdict

{process_improve-0.9.76 → process_improve-0.9.78}/process_improve/multivariate/methods.py

@@ -1,4 +1,4 @@
-# (c) Kevin Dunn, 2010-2024. MIT License. Based on own private work over the years.
+# (c) Kevin Dunn, 2010-2025. MIT License. Based on own private work over the years.
 from __future__ import annotations
 
 import time

{process_improve-0.9.76 → process_improve-0.9.78}/process_improve/multivariate/plots.py

@@ -1,4 +1,4 @@
-# (c) Kevin Dunn, 2010-2024. MIT License. Based on own private work over the years.
+# (c) Kevin Dunn, 2010-2025. MIT License. Based on own private work over the years.
 
 # Built-in libraries
 from __future__ import annotations

process_improve-0.9.78/process_improve/notebooks_examples/batch/batch_llm.py

@@ -0,0 +1,245 @@
+# ---
+# jupyter:
+#   jupytext:
+#     cell_metadata_filter: -all
+#     custom_cell_magics: kql
+#     text_representation:
+#       extension: .py
+#       format_name: percent
+#       format_version: '1.3'
+#       jupytext_version: 1.11.2
+#   kernelspec:
+#     display_name: process-improve
+#     language: python
+#     name: python3
+# ---
+
+# %%
+# !pip install git+https://github.com/amazon-science/chronos-forecasting.git
+
+
+# %%
+from __future__ import annotations
+
+import os
+import pathlib
+import sys
+
+import numpy as np
+import pandas as pd
+import plotly.graph_objects as go
+import torch
+from chronos import ChronosPipeline
+
+# Some basic checks, and logging set up.
+while not str(pathlib.Path.cwd().name).lower().startswith("process-improve"):
+    os.chdir(pathlib.Path.cwd().parents[0])
+basecwd = pathlib.Path.cwd()
+sys.path.insert(0, str(basecwd))
+assert basecwd.exists()
+
+import process_improve.datasets.batch as batch_ds
+from process_improve.batch.data_input import melted_to_dict
+from process_improve.batch.plotting import plot_all_batches_per_tag
+
+pd.options.plotting.backend = "plotly"
+pd.options.display.max_columns = 20
+pd.options.display.width = 200
+
+get_to_final_result = True
+device_map = "cpu"  # use "cpu" for CPU inference and "mps" for Apple Silicon; or "cuda"
+
+# Ideally, use more than 1 tag to align on. These columns must exist in all data frames for all batches.
+columns_to_import = ["AgitatorPower", "AgitatorTorque", "JacketTemperature", "DryerTemp"]
+tag_to_plot = columns_to_import[3]
+
+# %%
+pipeline = ChronosPipeline.from_pretrained(
+    "amazon/chronos-t5-base",
+    device_map=device_map,
+    torch_dtype=torch.bfloat16,
+)
+
+# %%
+# Demo code from the repo itself, to validate that it works as expted
+df = pd.read_csv(
+    "https://raw.githubusercontent.com/AileenNielsen/TimeSeriesAnalysisWithPython/master/data/AirPassengers.csv"
+)
+df["#Passengers"].plot()
+context = torch.tensor(df["#Passengers"])
+embeddings, tokenizer_state = pipeline.embed(context)
+
+
+# context must be either a 1D tensor, a list of 1D tensors,
+# or a left-padded 2D tensor with batch as the first dimension
+prediction_length = 12
+forecast = pipeline.predict(
+    context,
+    prediction_length=prediction_length,
+    num_samples=20,
+    temperature=1.0,
+    top_k=50,
+    top_p=1.0,
+)  # forecast shape: [num_series, num_samples, prediction_length]
+
+
+# %%
+# visualize the forecast
+forecast_index = range(len(df), len(df) + prediction_length)
+low, median, high = np.quantile(forecast[0].numpy(), [0.1, 0.5, 0.9], axis=0)
+
+
+# %%
+# Now our example starts. Settings for the batch case study
+
+prediction_length = 35
+num_samples = 20  # number or repeated predictions to make; we will average out over these
+first_n_samples = 65
+temperature = 1.0
+top_k = 50
+top_p = 1.0
+
+# Import the data: a dictionary of dataframes
+dryer_raw = (
+    pd.read_csv(pathlib.Path(batch_ds.__path__._recalculate()[0]) / "dryer.csv")
+    .rename({"ClockTime": "Pulse"}, axis=1)
+    .astype({"Pulse": "int"})
+)
+if not get_to_final_result:
+    dryer_raw[tag_to_plot].plot()
+
+# %%
+# Plot some data, to get an idea of what is present
+plt = plot_all_batches_per_tag(
+    df_dict=melted_to_dict(dryer_raw, batch_id_col="batch_id"),
+    tag=tag_to_plot,
+    time_column="ClockTime",
+    x_axis_label="Time [hours]",
+)
+if not get_to_final_result:
+    plt.show()
+
+# %%
+training_batches = list(range(1, 11))
+batch_dict = melted_to_dict(dryer_raw, batch_id_col="batch_id")
+sequences = [batch_dict[k][tag_to_plot].to_list() for k in training_batches if tag_to_plot in batch_dict[k]]
+training_sequence = pd.DataFrame(sequences).T
+plt = training_sequence.plot(title="Training data")
+if not get_to_final_result:
+    plt.show()
+
+# %%
+# Unravel all columnes, and make a single vector sequence
+all_sequences = []
+for k in training_batches:
+    all_sequences.extend(batch_dict[k][tag_to_plot].to_list())
+training_sequence = pd.Series(all_sequences)
+plt = training_sequence.plot(title="Unravelled training data")
+if not get_to_final_result:
+    plt.show()
+
+# %%
+# Train the model with the tag from the training batches
+context = torch.tensor(training_sequence)
+
+forecast = pipeline.predict(
+    context,
+    prediction_length,
+    num_samples=num_samples,
+    temperature=temperature,
+    top_k=top_k,
+    top_p=top_p,
+)  # forecast shape: [num_series, num_samples, prediction_length]
+
+# %%
+# Testing data: the first 50 samples of the next 10 batches
+
+testing_batches = [15]
+batch_dict = melted_to_dict(dryer_raw, batch_id_col="batch_id")
+sequences = []
+for k in testing_batches:
+    if tag_to_plot in batch_dict[k]:
+        values = batch_dict[k][tag_to_plot].to_list()[0 : first_n_samples + prediction_length]
+        values[first_n_samples:] = [None] * prediction_length
+        sequences.append(values)
+
+
+testing_sequence = pd.DataFrame(sequences).T
+if not get_to_final_result:
+    testing_sequence.plot(title="Testing data").show()
+
+# %%
+# Testing data:
+for batch in testing_batches:
+    context = torch.tensor(
+        pd.concat([training_sequence, batch_dict[batch].iloc[0:first_n_samples][tag_to_plot]]).values
+    )
+    cast = pipeline.predict(
+        context,
+        prediction_length,
+        num_samples=num_samples,
+        temperature=temperature,
+        top_k=top_k,
+        top_p=top_p,
+    )  # forecast shape: [num_series, num_samples, prediction_length]
+    lower, median, upper = np.quantile(cast[0].numpy(), [0.1, 0.5, 0.9], axis=0)
+    actual = batch_dict[batch][tag_to_plot]
+    forecast_index = actual.index[first_n_samples : (first_n_samples + prediction_length)]
+    fig = actual[0 : (first_n_samples + prediction_length)].plot(title="Actual and Forecast", height=1000)
+    fig.add_scatter(
+        x=forecast_index, y=actual[forecast_index], mode="lines", name="Actual", line=dict(width=3, color="blue")
+    )
+    fig.add_scatter(x=forecast_index, y=median, mode="lines", name="Forecast", line=dict(width=3, color="red"))
+    light_grey = "rgba(0, 0, 0, 0.1)"
+    for i in range(num_samples):
+        fig.add_scatter(
+            x=forecast_index,
+            y=cast[0].numpy()[i],
+            mode="lines",
+            legendgroup="i^th forecast",
+            name=f"Forecast {i+1}",
+            line=dict(color=light_grey),
+        )
+
+    fill_colour = "rgba(1, 0, 0, 0.2)"
+    fig.add_traces(
+        go.Scatter(
+            x=forecast_index,
+            y=upper,
+            line=dict(color=fill_colour),
+            mode="lines",
+            legendgroup="Bounds",
+            showlegend=False,
+            name="Bounds",
+        )
+    )
+    fig.add_traces(
+        go.Scatter(
+            x=forecast_index,
+            y=lower,
+            line=dict(color=fill_colour),
+            fill="tonexty",  # fill in the range
+            fillcolor=fill_colour,
+            mode="lines",
+            legendgroup="Bounds",
+            name="Bounds",
+        )
+    )
+
+    fig.show()
+
+
+# %%
+
+# raw_sequences = [batch_dict[k][tag_to_plot] for k in training_batches if tag_to_plot in batch_dict[k]]
+# embeddings = []
+# for sequence in raw_sequences:
+#     a=2
+#     embeddings, tokenizer_state = pipeline.embed(torch.tensor(pd.DataFrame(sequence)))
+#     q=embeddings.numpy()
+
+
+# sequences =
+# training_sequence = pd.DataFrame(sequences).T
+# plt = training_sequence.plot(title="Training data")
+# if not get_to_final_resul

process_improve-0.9.78/process_improve/notebooks_examples/batch/batch_llm_multivariate.py

@@ -0,0 +1,202 @@
+# ---
+# jupyter:
+#   jupytext:
+#     cell_metadata_filter: -all
+#     custom_cell_magics: kql
+#     text_representation:
+#       extension: .py
+#       format_name: percent
+#       format_version: '1.3'
+#       jupytext_version: 1.11.2
+#   kernelspec:
+#     display_name: process-improve
+#     language: python
+#     name: python3
+# ---
+
+# %%
+# !pip install git+https://github.com/SalesforceAIResearch/uni2ts
+
+
+from __future__ import annotations
+
+import os
+import pathlib
+import sys
+
+import numpy as np
+import pandas as pd
+import torch
+from datasets import load_dataset
+from gluonts.dataset.pandas import PandasDataset
+from huggingface_hub import hf_hub_download
+from uni2ts.model.moirai import MoiraiForecast
+
+# Some basic checks, and logging set up.
+while not str(pathlib.Path.cwd().name).lower().startswith("process-improve"):
+    os.chdir(pathlib.Path.cwd().parents[0])
+basecwd = pathlib.Path.cwd()
+sys.path.insert(0, str(basecwd))
+assert basecwd.exists()
+
+import process_improve.datasets.batch as batch_ds
+from process_improve.batch.data_input import melted_to_dict
+from process_improve.batch.plotting import plot_all_batches_per_tag
+
+pd.options.plotting.backend = "plotly"
+pd.options.display.max_columns = 20
+pd.options.display.width = 200
+
+device_map = "cpu"  # use "cpu" for CPU inference and "mps" for Apple Silicon; or "cuda"
+
+# Ideally, use more than 1 tag to align on. These columns must exist in all data frames for all batches.
+columns_to_import = ["AgitatorPower", "AgitatorTorque", "JacketTemperature", "DryerTemp"]
+tag_to_plot = columns_to_import[3]
+
+
+TIME_COL = "Date"
+TARGET = "visits"
+DYNAMIC_COV = ["CPI", "Inflation_Rate", "GDP"]
+SEAS_COV = [
+    "month_1",
+    "month_2",
+    "month_3",
+    "month_4",
+    "month_5",
+    "month_6",
+    "month_7",
+    "month_8",
+    "month_9",
+    "month_10",
+    "month_11",
+    "month_12",
+]
+FORECAST_HORIZON = 8  # months
+FREQ = "M"
+
+
+# %%
+def preprocess_dataset(dataframe: pd.DataFrame, dynamic_cov: list, time_col: str, target: str) -> pd.DataFrame:
+    """
+    Receives the raw dataframe and create.
+
+        - unique id column
+        - make dynamic start dates for each series based on the first date where visits is different than 0
+
+    Args:
+        dataframe (pd.DataFrame): raw data
+        dynamic_cov (list): column names with dynamic cov
+        time_col (str): time column name
+        target (str): target name
+    Returns:
+        pd.DataFrame: cleaned data
+    """
+
+    # save dynamic cov for later
+    dynamic_cov_df = dataframe[dynamic_cov].reset_index().drop_duplicates()
+
+    # create target and unique id columns
+    dataframe = (
+        dataframe.loc[:, ~dataframe.columns.isin(dynamic_cov)]
+        .melt(ignore_index=False)
+        .reset_index()
+        .rename(columns={"variable": "unique_id", "value": target})
+    )
+
+    # crete dynamic start dates for each series
+    cleaned_df = []
+    for i in dataframe["unique_id"].unique():
+        temp = dataframe[dataframe["unique_id"] == i]
+        cleaned_df.append(temp[temp[time_col] >= min(temp[temp[target] > 0][time_col])])
+    cleaned_df = pd.concat(cleaned_df)
+
+    # join dynamic cov
+    cleaned_df = pd.merge(cleaned_df, dynamic_cov_df, on=[time_col], how="left")
+
+    return cleaned_df
+
+
+def moirai_forecast_to_pandas(forecast, test_df: pd.DataFrame, forecast_horizon: int, time_col: str) -> pd.DataFrame:
+    """
+    Convert MOIRAI forecast into pandas dataframe.
+
+    Args:
+        forecast: MOIRAI's forecast
+        test_df: dataframe with actuals
+        forecast_horizon: forecast horizon
+        time_col: date column
+    Returns:
+        pd.DataFrame: forecast in pandas format
+    """
+
+    d = {"unique_id": [], time_col: [], "forecast": [], "forecast_lower": [], "forecast_upper": []}
+
+    for ts in forecast:
+        for j in range(forecast_horizon):
+            d["unique_id"].append(ts.item_id)
+            d[time_col].append(test_df[test_df["unique_id"] == ts.item_id][time_col].tolist()[j])
+
+            temp = [sample[j] for sample in ts.samples]
+
+            d["forecast"].append(np.median(temp))
+            d["forecast_lower"].append(np.percentile(temp, 10))
+            d["forecast_upper"].append(np.percentile(temp, 90))
+
+    return pd.DataFrame(d)
+
+
+# load data and exogenous features
+df = pd.DataFrame(load_dataset("zaai-ai/time_series_datasets", data_files={"train": "data.csv"})["train"]).drop(
+    columns=["Unnamed: 0"]
+)
+df[TIME_COL] = pd.to_datetime(df[TIME_COL])
+
+# one hot encode month
+df["month"] = df[TIME_COL].dt.month
+df = pd.get_dummies(df, columns=["month"], dtype=int)
+
+print(f"Distinct number of time series: {len(df['unique_id'].unique())}")
+df.head()
+
+# 8 months to test
+train = df[df[TIME_COL] <= (max(df[TIME_COL]) - pd.DateOffset(months=FORECAST_HORIZON))]
+test = df[df[TIME_COL] > (max(df[TIME_COL]) - pd.DateOffset(months=FORECAST_HORIZON))]
+
+print(
+    f"Months for training: {len(train[TIME_COL].unique())} from {min(train[TIME_COL]).date()} to {max(train[TIME_COL]).date()}"
+)
+print(
+    f"Months for testing: {len(test[TIME_COL].unique())} from {min(test[TIME_COL]).date()} to {max(test[TIME_COL]).date()}"
+)
+
+
+# create GluonTS dataset from pandas
+ds = PandasDataset.from_long_dataframe(
+    pd.concat([train, test[["unique_id", TIME_COL] + DYNAMIC_COV + SEAS_COV]]).set_index(
+        TIME_COL
+    ),  # concatenaation with test dynamic covaraiates
+    item_id="unique_id",
+    feat_dynamic_real=DYNAMIC_COV + SEAS_COV,
+    target=TARGET,
+    freq=FREQ,
+)
+
+
+# Prepare pre-trained model by downloading model weights from huggingface hub
+model = MoiraiForecast.load_from_checkpoint(
+    checkpoint_path=hf_hub_download(repo_id="Salesforce/moirai-R-large", filename="model.ckpt"),
+    prediction_length=FORECAST_HORIZON,
+    context_length=24,
+    patch_size="auto",
+    num_samples=100,
+    target_dim=1,
+    feat_dynamic_real_dim=ds.num_feat_dynamic_real,
+    past_feat_dynamic_real_dim=ds.num_past_feat_dynamic_real,
+    map_location="cuda:0" if torch.cuda.is_available() else "cpu",
+)
+
+predictor = model.create_predictor(batch_size=32)
+forecasts = predictor.predict(ds)
+
+# convert forecast into pandas
+forecast_df = moirai_forecast_to_pandas(forecasts, test, FORECAST_HORIZON, TIME_COL)