jrpybestpracccc 0.1.1__py3-none-any.whl

jrpybestpracccc/extra_scripts/example_projects/good_project/setup.cfg

@@ -0,0 +1,7 @@
+[flake8]
+max-line-length = 88
+per-file-ignores = [
+    utils/data/__init__.py:F401,
+    utils/modelling/__init__.py:F401,
+    utils/deployment/__init__.py:F401,
+]

jrpybestpracccc/extra_scripts/example_projects/good_project/src/config/filenames.py

@@ -0,0 +1,3 @@
+"""Commonly used filenames and filepaths"""
+
+DATA_PATH = "data"

jrpybestpracccc/extra_scripts/example_projects/good_project/src/config/theme.py

@@ -0,0 +1,147 @@
+import matplotlib.pyplot as plt
+import plotly.graph_objects as go
+import plotly.io as pio
+from cycler import cycler
+
+Colors = {
+    "vibrant purple": "#7041FF",
+    "orange": "#FFAC00",
+    "aubergine": "#280049",
+    "lavender": "#8C57CC",
+    "purple 4": "#AB6B99",
+    "purple 5": "#CA7880",
+    "red": "#FF2200",
+    "yellow": "#FFFF4B",
+    "green": "#A2D800",
+    "forest green": "#1A5F31",
+    "sea green": "#369992",
+    "sky blue": "#AEC5EB",
+}
+
+# set default colours to use
+plt.rcParams["axes.prop_cycle"] = cycler(color=list(Colors.values()))
+
+# style plots to match CCC branding
+plt.rcParams["figure.figsize"] = [6, 3]
+plt.rcParams["figure.constrained_layout.use"] = True
+plt.rcParams["figure.dpi"] = 120
+plt.rcParams["axes.grid"] = True
+plt.rcParams["axes.grid.axis"] = "y"
+plt.rcParams["axes.labelweight"] = "bold"
+plt.rcParams["axes.titlecolor"] = Colors["vibrant purple"]
+plt.rcParams["axes.titleweight"] = "bold"
+plt.rcParams["axes.labelcolor"] = Colors["vibrant purple"]
+plt.rcParams["axes.edgecolor"] = Colors["vibrant purple"]
+plt.rcParams["axes.spines.left"] = False
+plt.rcParams["axes.spines.right"] = False
+plt.rcParams["axes.spines.top"] = False
+plt.rcParams["grid.linewidth"] = 0.4
+plt.rcParams["grid.color"] = "silver"
+plt.rcParams["xtick.color"] = Colors["vibrant purple"]
+plt.rcParams["ytick.color"] = Colors["vibrant purple"]
+plt.rcParams["ytick.left"] = False
+plt.rcParams["legend.frameon"] = False
+plt.rcParams["legend.labelcolor"] = Colors["vibrant purple"]
+plt.rcParams["font.family"] = "century gothic"
+
+SCENARIO_COLORS = {
+    "Baseline": Colors["orange"],
+    "Pathway": Colors["vibrant purple"],
+    "Historical": Colors["aubergine"],
+}
+
+# Plotly template
+ccc_template = go.layout.Template(
+    layout=go.Layout(
+        width=720,
+        height=472,
+        
+        font=dict(
+            family="Century Gothic, sans-serif",
+            size=12,
+            color=Colors["vibrant purple"]
+        ),
+        
+        title=dict(
+            font=dict(
+                family="Century Gothic, sans-serif",
+                size=14,
+                color=Colors["vibrant purple"]
+            ),
+            x=0.5,
+            xanchor="center"
+        ),
+        
+        plot_bgcolor="white",
+        paper_bgcolor="white",
+        
+        xaxis=dict(
+            showgrid=False,
+            showline=True,
+            linewidth=1,
+            linecolor=Colors["vibrant purple"],
+            tickcolor=Colors["vibrant purple"],
+            title_font=dict(
+                family="Century Gothic, sans-serif",
+                size=14,
+                color=Colors["vibrant purple"]
+            ),
+            tickfont=dict(
+                family="Century Gothic, sans-serif",
+                size=12,
+                color=Colors["vibrant purple"]
+            ),
+            mirror=False,
+            zeroline=False
+        ),
+        
+        yaxis=dict(
+            showgrid=True,
+            gridwidth=0.4,
+            gridcolor="silver",
+            showline=False,
+            zeroline=False,
+            tickcolor=Colors["vibrant purple"],
+            ticks="",
+            title_font=dict(
+                family="Century Gothic, sans-serif",
+                size=14,
+                color=Colors["vibrant purple"]
+            ),
+            tickfont=dict(
+                family="Century Gothic, sans-serif",
+                size=12,
+                color=Colors["vibrant purple"]
+            )
+        ),
+        
+        legend=dict(
+            bgcolor="rgba(0,0,0,0)",
+            bordercolor="rgba(0,0,0,0)",
+            font=dict(
+                family="Century Gothic, sans-serif",
+                color=Colors["vibrant purple"]
+            ),
+            orientation="v",
+            yanchor="top",
+            y=1,
+            xanchor="left",
+            x=0.01
+        ),
+        
+        colorway=list(Colors.values()),
+        
+        hoverlabel=dict(
+            bgcolor="white",
+            font_size=12,
+            font_family="Century Gothic, sans-serif",
+            bordercolor=Colors["vibrant purple"]
+        ),
+        
+        margin=dict(l=60, r=20, t=60, b=50)
+    )
+)
+
+# Register and set as default
+pio.templates["ccc"] = ccc_template
+pio.templates.default = "ccc"

jrpybestpracccc/extra_scripts/example_projects/good_project/src/data/__init__.py

@@ -0,0 +1,2 @@
+from .storage import load_data, save_data
+from .preprocessing import calibrate_transformer, preprocess, transform_data

jrpybestpracccc/extra_scripts/example_projects/good_project/src/data/preprocessing.py

@@ -0,0 +1,341 @@
+"""Functions for dealing with data preprocessing, including cleaning, scaling and
+one-hot encoding
+"""
+
+import pandas as pd
+from sklearn.compose import ColumnTransformer
+from sklearn.preprocessing import OneHotEncoder, StandardScaler
+
+from ..config.filenames import DATA_PATH
+
+
+def clean_data(data):
+    """Remove missing values and clean up variable names
+
+    :parameters:
+    data: pd.DataFrame
+      Pandas DataFrame with the data to be cleaned
+
+    :returns:
+    pd.DataFrame object with the tidy data
+    """
+    data.dropna(inplace=True)
+    data = data.clean_names()
+    return data
+
+
+def join_on_archetypes(
+    df: pd.DataFrame, archetype_df: pd.DataFrame
+) -> pd.DataFrame:
+
+    """
+    This function handles the joining of the two capex datasets with the
+    archetypes dataset.
+
+    Args
+    ------
+    df: the capex DataFrame to join with.
+    archetype_df: the archetype DataFrame.
+
+    Returns
+    ------
+
+    df: the new DataFrame with the archetype data joined on.
+
+    """
+
+    df = df.merge(archetype_df,
+                  left_on="Heating system",
+                  right_on="Heating system",
+                  how="inner")
+
+    return df
+
+
+def get_size_scaled_capex(capex_by_power_df: pd.DataFrame) -> pd.DataFrame:
+    """
+    This function handles the scaling of the capex amounts by
+    heating size for the capex by power data.
+
+    Args
+    -----
+    capex_by_power_df: the capex by power DataFrame.
+
+    Returns
+    ------
+    capex_by_power_df: the scaled capex DataFrame.
+    """
+
+    year_cols = [
+        col for col in capex_by_power_df.columns if col.startswith("20")
+    ]
+
+    capex_by_power_df[year_cols] = (
+        capex_by_power_df[year_cols]
+        .multiply(capex_by_power_df["Heating system size"], axis=0)
+    )
+
+    return capex_by_power_df
+
+
+def concatenate_dfs(
+    capex_by_power_df: pd.DataFrame,
+    fixed_capex_df: pd.DataFrame,
+) -> pd.DataFrame:
+
+    """
+    Concatenates the capex by power DataFrame and the fixed capex DataFrame
+    into a single DataFrame.
+
+    Args
+    -----
+    capex_by_power_df: DataFrame containing capex data scaled by power.
+    fixed_capex_df: DataFrame containing fixed capex data.
+
+    Returns
+    ------
+    all_capex_df: The concatenated DataFrame containing all capex data.
+    """
+
+    all_capex_df = pd.concat([capex_by_power_df, fixed_capex_df])
+
+    return all_capex_df
+
+
+def adjust_prices(all_capex_df: pd.DataFrame) -> pd.DataFrame:
+
+    """
+    Adjusts the capex values in the DataFrame for each year by multiplying
+    them with the corresponding price index.
+
+    Args
+    -----
+    all_capex_df: DataFrame containing capex data with year columns and a
+    'Price index' column.
+
+    Returns
+    ------
+    all_capex_df: DataFrame with year columns adjusted by the price index.
+    """
+
+    year_cols = [col for col in all_capex_df.columns if col.startswith("20")]
+    all_capex_df[year_cols] = (
+        all_capex_df[year_cols]
+        .multiply(all_capex_df["Price index"], axis=0)
+    )
+
+    return all_capex_df
+
+
+def get_grouped_total_capex(all_capex_df: pd.DataFrame) -> pd.DataFrame:
+
+    """
+    Groups the capex DataFrame by 'Constrained archetype number' and sums the
+    capex values for each year.
+
+    Args
+    -----
+    all_capex_df: DataFrame containing capex data with year columns and a
+    'Constrained archetype number' column.
+
+    Returns
+    ------
+    all_capex_df: DataFrame with summed capex values for each 'Constrained
+    archetype number' and year.
+    """
+
+    year_cols = [col for col in all_capex_df.columns if col.startswith("20")]
+    agg_dict = {
+        col: 'first' for col in all_capex_df.columns if col not in year_cols
+    }
+    agg_dict.update({col: 'sum' for col in year_cols})
+
+    summed_capex_df = (
+        all_capex_df
+        .groupby(by="Constrained archetype number", as_index=False)
+        .agg(agg_dict)
+    )
+
+    return summed_capex_df
+
+
+def drop_and_add_columns(
+    all_capex_df: pd.DataFrame,
+    columns_to_drop=["Cost type variant", "Assumptions", "Heating system"],
+) -> pd.DataFrame:
+
+    """
+    Drops specified columns from the DataFrame, removes duplicate rows, and
+    adds two new columns.
+
+    -----
+    all_capex_df: DataFrame from which columns will be dropped.
+    columns_to_drop: List of column names to be dropped from the DataFrame
+    (default: ["Cost type variant", "Assumptions", "Heating system"]).
+
+    Returns
+    ------
+    all_capex_df: DataFrame with specified columns dropped, new columns added,
+    and duplicates removed.
+    """
+
+    all_capex_df = all_capex_df.drop(columns=columns_to_drop)
+
+    all_capex_df["Variable unit"] = "£"
+    all_capex_df["Data name"] = "Baseline capex per home by renewal year"
+
+    return all_capex_df
+
+
+def create_output(
+    all_capex_df: pd.DataFrame,
+    output_filename="Baseline heating system capex.csv",
+    output_path=DATA_PATH,
+):
+
+    """
+    Writes the provided DataFrame to a CSV file at the specified output path.
+
+    Args
+    -----
+    all_capex_df: DataFrame to be written to CSV.
+    output_filename: Name of the output CSV file (default: "Baseline heating
+    system capex.csv").
+    output_path: Path where the CSV file will be saved (default: DATA_PATH).
+
+    Returns
+    ------
+    None
+    """
+
+    all_capex_df.to_csv(output_path/output_filename, index=False)
+
+    return
+
+
+def create_long_format(df: pd.DataFrame, index: int = -31):
+    """
+    Converts an input DataFrame into long format
+
+    parameters
+    ----------
+    df: pd.DataFrame
+      The data should initially have capex columns for the years
+      "2020" to "2050". These typically span index -31 to the end of
+      the DataFrame.
+    index: int (default -31)
+      Index of the "2020" capex column
+
+    returns
+    -------
+    pd.DataFrame
+      Long-format version of the data with a "Year" column and
+      "Capex" column
+    """
+    columns = list(df.columns)
+    return df.melt(
+        id_vars=columns[:index],
+        value_vars=columns[index:],
+        var_name="Year",
+        value_name="Capex",
+    )
+
+
+def preprocess(
+    archetype_df: pd.DataFrame,
+    fixed_capex_df: pd.DataFrame,
+    capex_by_power_df: pd.DataFrame,
+    long_format: bool = False,
+    index: int = -31,
+) -> pd.DataFrame:
+    """
+    Run all of the preprocessing steps on the input data
+
+    Args
+    -----
+    archetype_df: The archetype DataFrame.
+    fixed_capex_df: The fixed capex DataFrame.
+    capex_by_power_df: The capex by power.
+    long_format: If True, convert the output data into long
+      format with "Year" and "Capex" columns (False by default)
+    index: The index of the "2020" column in the preprocessed
+      data (-31 by default)
+
+    Returns
+    ------
+    pd.DataFrame
+      Preprocessed DataFrame
+    """
+
+    fixed_capex_df = join_on_archetypes(
+        df=fixed_capex_df, archetype_df=archetype_df
+    )
+    capex_by_power_df = join_on_archetypes(
+        df=capex_by_power_df, archetype_df=archetype_df
+    )
+
+    capex_by_power_df = get_size_scaled_capex(
+        capex_by_power_df=capex_by_power_df
+    )
+
+    all_capex_df = concatenate_dfs(
+        capex_by_power_df=capex_by_power_df,
+        fixed_capex_df=fixed_capex_df,
+    )
+    all_capex_df = get_grouped_total_capex(all_capex_df=all_capex_df)
+    all_capex_df = drop_and_add_columns(all_capex_df=all_capex_df)
+
+    if long_format:
+        return create_long_format(
+            df=all_capex_df,
+            index=index,
+        )
+
+    return all_capex_df
+
+
+def calibrate_transformer(
+    data: pd.DataFrame,
+    numerical_features: list,
+    categorical_features: list,
+):
+    """Calibrate a data transformer using the input data
+
+    :parameters:
+    data: pd.DataFrame
+      Input data used for calibrating the transformer
+    numerical_features: list
+      List of the numerical variables to be transformed
+    categorical_features: list
+      List of the categorical variables to be transformed
+
+    :returns:
+    sklearn ColumnTransformer object for applying consistent scaling
+    and encoding to the data
+    """
+    transformer = ColumnTransformer(
+        [
+            ("num", StandardScaler(), numerical_features),
+            ("cat", OneHotEncoder(drop="first"), categorical_features),
+        ]
+    )
+    transformer.fit(data)
+    return transformer
+
+
+def transform_data(
+    data: pd.DataFrame,
+    transformer: ColumnTransformer,
+):
+    """ Scale and encode data using a pre-calibrated transformer
+
+    :parameters:
+    data: pd.DataFrame
+      Data to be preprocessed
+    transformer: sklearn ColumnTransformer
+      Pre-calibrated transformer for scaling and encoding the data
+
+    :returns:
+    pd.DataFrame object with transformed data
+    """
+    return transformer.transform(data)

jrpybestpracccc/extra_scripts/example_projects/good_project/src/data/storage.py

@@ -0,0 +1,99 @@
+"""Functions for managing direct interactions with the raw data"""
+
+import os
+import pandas as pd
+
+from ..config.filenames import DATA_PATH
+
+
+def construct_data_filepath(filename="raw_data"):
+    """
+    Construct filepath to data CSV file
+
+    Args
+    ----
+
+        filename: str (default: "raw_data")
+          Filename of the data (without the .csv extension)
+
+    Returns
+    -------
+        string with the full filepath for loading and writing the data
+    """
+    file = f"{filename}.csv"
+    return os.path.join(DATA_PATH, file)
+
+
+def load_csv(filename: str, header_height: int = 4):
+    """
+    Loads a CSV file from the project data folder
+
+    parameters
+    ----------
+    filename: str
+      Name of the CSV file to load
+
+    header_height: int
+      Number of rows to skip at the top of the CSV
+      (4 by default)
+
+    returns
+    -------
+    pd.DataFrame
+    """
+    filepath = construct_data_filepath(filename)
+    return pd.read_csv(filepath, skiprows=header_height)
+
+
+def load_data() -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
+
+    """
+    Reads in the three CSV files used in the workflow.
+
+    returns
+    -------
+
+        archetype_df: DataFrame containing the archetypes data.
+        fixed_capex_df: DataFrame containing the fixed capex data.
+        capex_by_power_df: DataFrame containing the capex by power data.
+
+    """
+
+    archetype_df = load_csv(
+        filename="Baseline archetype properties",
+        header_height=0,
+    )
+
+    # Skip the first 4 rows as they contain metadata or headers not relevant
+    # to the DataFrame
+    fixed_capex_df = load_csv(
+        filename="Input Baseline capex fixed",
+        header_height=4,
+    )
+    # Skip the first 4 rows as they contain metadata or headers not relevant
+    # to the DataFrame
+    capex_by_power_df = load_csv(
+        filename="Input Baseline capex by power",
+        header_height=4,
+    )
+
+    return archetype_df, fixed_capex_df, capex_by_power_df
+
+
+def save_data(data, output_filename="raw_data.csv"):
+    """
+    Save data to a CSV file in the project data folder
+
+    parameters
+    ----------
+    data: pd.DataFrame
+      Pandas DataFrame containing the data to be saved
+    output: str
+      Output filename (without extension) for the CSV data
+
+    returns
+    -------
+    None
+    """
+    filepath = construct_data_filepath(output_filename)
+    data.to_csv(filepath, index=False)

jrpybestpracccc/extra_scripts/example_projects/good_project/src/modelling/__init__.py

@@ -0,0 +1,4 @@
+from .ingredients import extract_target_variable, setup_model, setup_pipeline
+from .predictions import model_predict
+from .scoring import score_model
+from .training import create_test_data, train_model

jrpybestpracccc/extra_scripts/example_projects/good_project/src/modelling/ingredients.py

@@ -0,0 +1,58 @@
+"""
+Functions associated with setting up the model object and the data input
+variables
+"""
+
+from sklearn.linear_model import LinearRegression
+from sklearn.pipeline import Pipeline
+
+
+def extract_target_variable(data, target, covariates):
+    """Extract target variable to be predicted by trained model
+
+    :parameters:
+    data: pd.DataFrame
+      Pandas DataFrame with the raw data from which to select the target variable
+    target: str
+      String with the name of the column to set as the target variable y
+    covariates: list
+      List of column names to set as the covariates x
+
+    :returns:
+    Two outputs as tuple (y, X):
+    - Target variable column
+    - Predictor variable columns
+    """
+    y = data[target]
+    x = data[covariates]
+    return y, x
+
+
+def setup_model():
+    """Constucts a model object (currently LinearRegression) to use for training
+
+    :returns:
+    Sklearn model class
+    """
+    return LinearRegression()
+
+
+def setup_pipeline(transformer, model):
+    """Constructs a pipeline for data transformation and modelling
+
+    :parameters:
+    transformer: Sklearn ColumnTransformer object
+      Calibrated transformer for applying scaling and one-hot encoding to data
+    model: Sklean model object
+      Model to be trained on data
+
+    :returns:
+    Sklearn Pipeline class including a calibrated data transformer and a model that is
+    ready for training
+    """
+    return Pipeline(
+        [
+            ("transform", transformer),
+            ("model", model),
+        ]
+    )

jrpybestpracccc/extra_scripts/example_projects/good_project/src/modelling/predictions.py

@@ -0,0 +1,18 @@
+"""
+Functions involved in generating model predictions using the sklearn model
+"""
+
+
+def model_predict(model, data):
+    """Get predictions from model
+
+    :parameters:
+    model: sklearn model or pipeline object
+      Trained sklearn model or pipeline ready for predictions
+    data: 2-D data structure
+      Untransformed input data for predictions
+
+    :returns:
+    1-D data structure with the model predictions
+    """
+    return model.predict(data)

jrpybestpracccc/extra_scripts/example_projects/good_project/src/modelling/scoring.py

@@ -0,0 +1,18 @@
+"""Scoring functions for model testing"""
+
+from sklearn.metrics import mean_absolute_error
+
+
+def score_model(y_true, y_pred):
+    """Scoring function for the mean absolute error
+
+    :parameters:
+    y_true: 1-D data structure
+      True values of the target variable
+    y_pred: 1-D data structure
+      Predicted values of the target variable
+
+    :returns:
+    Maximum absolute error
+    """
+    return mean_absolute_error(y_true, y_pred)