baticlust 0.1.0__py3-none-any.whl

baticlust/__init__.py

@@ -0,0 +1,18 @@
+from .clustering import stratify_data, cluster_stratum, generate_archetypes, select_medoids
+from .preprocessing import scale_features, one_hot_encode
+from .validation import compute_archetype_performance, validate_typology, plot_medoid_validation, validate_typology_with_medoids, plot_distribution_comparison
+from pathlib import Path
+
+root_path = Path(__file__).parent
+
+# paths for the various data folders
+data_path = {
+    "base": root_path / "data",
+    "gis": root_path / "data" / "gis",
+    "climate": root_path / "data" / "climate",
+    "census": root_path / "data" / "census",
+    "diagnosis": root_path / "data" / "diagnosis",
+    "building_statistics": root_path / "data" / "building_statistics",
+    "energy_use": root_path / "data" / "energy_use",
+    "energy_open_data": root_path / "data" / "energy_open_data",
+}

baticlust/clustering.py

@@ -0,0 +1,130 @@
+import polars as pl
+import numpy as np
+import faiss
+import fastcluster
+from scipy.cluster.hierarchy import fcluster
+from scipy.spatial.distance import cdist
+from joblib import Parallel, delayed
+from tqdm import tqdm
+
+def stratify_data(df: pl.DataFrame, strata_cols: list[str]) -> list[tuple[tuple, pl.DataFrame]]:
+    """
+    Splits the DataFrame into strata based on unique combinations of strata_cols.
+    Returns a list of (stratum_values, sub_df).
+    """
+    partitions = df.partition_by(strata_cols, as_dict=True)
+    return list(partitions.items())
+
+def cluster_stratum(sub_df: pl.DataFrame, features: list[str], method: str, n_clusters: int, **kwargs) -> pl.DataFrame:
+    """
+    Clusters a single sub-sample (stratum).
+    Returns the sub_df with a new 'cluster_id' column.
+    """
+    if sub_df.height == 0:
+        return sub_df.with_columns(pl.lit(-1, dtype=pl.Int64).alias("cluster_id"))
+    
+    # Prepare data for clustering
+    # Ensure features are float32 for FAISS
+    X = sub_df.select(features).to_numpy().astype(np.float32)
+    
+    # Handle case where n_clusters >= n_samples
+    if n_clusters >= sub_df.height:
+         return sub_df.with_columns(
+            pl.int_range(0, sub_df.height, dtype=pl.Int64).alias("cluster_id")
+        )
+
+    if method == 'kmeans':
+        # FAISS K-Means
+        d = X.shape[1]
+        # faiss.Kmeans expects n_clusters (k) and dimension (d)
+        kmeans = faiss.Kmeans(d, n_clusters, niter=kwargs.get('niter', 20), verbose=kwargs.get('verbose', False))
+        kmeans.train(X)
+        _, labels = kmeans.index.search(X, 1)
+        labels = labels.flatten()
+        
+    elif method == 'hierarchical':
+        # fastcluster
+        linkage_method = kwargs.get('linkage_method', 'ward')
+        metric = kwargs.get('metric', 'euclidean')
+        
+        Z = fastcluster.linkage(X, method=linkage_method, metric=metric)
+        labels = fcluster(Z, n_clusters, criterion='maxclust')
+        labels = labels - 1 # 0-based
+        
+    else:
+        raise ValueError(f"Unknown clustering method: {method}")
+
+    return sub_df.with_columns(pl.Series("cluster_id", labels).cast(pl.Int64))
+
+def select_medoids(df: pl.DataFrame, cluster_col: str, features: list[str]) -> pl.DataFrame:
+    """
+    Identifies the medoid for each cluster.
+    The medoid is the point in the cluster with the minimum sum of distances to all other points in the cluster.
+    Returns a DataFrame containing only the medoid rows.
+    """
+    # We need global indices to select rows later
+    df_with_idx = df.with_row_index("row_idx")
+    
+    # Group by cluster
+    partitions = df_with_idx.partition_by(cluster_col, as_dict=True)
+    
+    medoid_indices = []
+    
+    for cluster_id, group in partitions.items():
+        if group.height == 0:
+            continue
+            
+        X_group = group.select(features).to_numpy()
+        
+        # Calculate pairwise distances
+        # cdist returns a matrix of distances
+        # If group is large, this can be memory intensive (N^2).
+        # For large groups, we might want to approximate or sample.
+        # Assuming reasonable cluster sizes for archetypes.
+        dists = cdist(X_group, X_group, metric='euclidean')
+        
+        # Sum of distances for each point
+        sum_dists = dists.sum(axis=1)
+        
+        # Find index of minimum sum
+        local_medoid_idx = np.argmin(sum_dists)
+        
+        # Get the global row index
+        global_idx = group["row_idx"][int(local_medoid_idx)]
+        medoid_indices.append(global_idx)
+
+    # Filter the original dataframe to keep only medoids
+    medoids_df = df_with_idx.filter(pl.col("row_idx").is_in(medoid_indices)).drop("row_idx")
+    
+    return medoids_df
+
+def _process_stratum(sub_df, total_samples, total_n_clusters, features, method, **kwargs):
+    n_samples = sub_df.height
+    # Proportional allocation
+    if total_samples > 0:
+        n_clusters = max(1, int(round(total_n_clusters * (n_samples / total_samples))))
+    else:
+        n_clusters = 1
+        
+    n_clusters = min(n_clusters, n_samples)
+    
+    return cluster_stratum(sub_df, features, method, n_clusters, **kwargs)
+
+def generate_archetypes(df: pl.DataFrame, strata_cols: list[str], features: list[str], method: str, total_n_clusters: int, **kwargs) -> pl.DataFrame:
+    """
+    Orchestrates stratification and clustering.
+    Distributes total_n_clusters proportionally to strata size.
+    """
+    strata = stratify_data(df, strata_cols)
+    total_samples = df.height
+    
+    results = Parallel(n_jobs=-1)(
+        delayed(_process_stratum)(
+            sub_df, total_samples, total_n_clusters, features, method, **kwargs
+        ) for _, sub_df in tqdm(strata, desc="Clustering strata")
+    )
+        
+    if not results:
+        return df.with_columns(pl.lit(None).alias("cluster_id"))
+
+    return pl.concat(results)

baticlust/preprocessing.py

@@ -0,0 +1,24 @@
+import polars as pl
+
+def scale_features(df: pl.DataFrame, features: list[str], method: str = 'standard') -> pl.DataFrame:
+    """
+    Scales numerical features using Standard or MinMax scaling.
+    """
+    if method == 'standard':
+        return df.with_columns([
+            ((pl.col(f) - pl.col(f).mean()) / pl.col(f).std()).fill_nan(0).alias(f)
+            for f in features
+        ])
+    elif method == 'minmax':
+        return df.with_columns([
+            ((pl.col(f) - pl.col(f).min()) / (pl.col(f).max() - pl.col(f).min())).fill_nan(0).alias(f)
+            for f in features
+        ])
+    else:
+        raise ValueError("Method must be 'standard' or 'minmax'")
+
+def one_hot_encode(df: pl.DataFrame, columns: list[str]) -> pl.DataFrame:
+    """
+    One-hot encodes categorical columns.
+    """
+    return df.to_dummies(columns)

baticlust/validation.py

@@ -0,0 +1,164 @@
+import polars as pl
+import numpy as np
+from scipy.stats import wasserstein_distance, ks_2samp, gaussian_kde
+import matplotlib.pyplot as plt
+from sklearn.metrics import r2_score
+
+def compute_archetype_performance(df: pl.DataFrame, cluster_col: str, target_col: str) -> pl.DataFrame:
+    """
+    Calculate the representative energy performance for each cluster.
+    """
+    return df.group_by(cluster_col).agg([
+        pl.col(target_col).mean().alias(f"{target_col}_mean"),
+        pl.col(target_col).median().alias(f"{target_col}_median"),
+        pl.col(target_col).std().alias(f"{target_col}_std"),
+        pl.col(target_col).count().alias("count")
+    ])
+
+def validate_typology_with_medoids(df: pl.DataFrame, medoids_df: pl.DataFrame, cluster_col: str, target_col: str) -> dict[str, float]:
+    """
+    Compare the aggregated cluster-based distribution of the target_col (energy performance)
+    against the actual global distribution in the original data, using medoids as representatives.
+    """
+    # Original distribution
+    original_dist = df.select(target_col).to_numpy().flatten()
+    
+    # Reconstructed distribution using medoids
+    # We need to map each row in the original df to its medoid's target_col value.
+    
+    # Medoid values for each cluster
+    # We need to make sure we select only the cluster_col and target_col from medoids_df
+    medoid_values = medoids_df.select([cluster_col, target_col]).rename({target_col: "medoid_val"})
+    
+    # Join back to the full clustered dataframe
+    reconstructed_df = df.join(medoid_values, on=cluster_col, how="left")
+    reconstructed_dist = reconstructed_df.select("medoid_val").to_numpy().flatten()
+    
+    wd = wasserstein_distance(original_dist, reconstructed_dist)
+    ks_stat, _ = ks_2samp(original_dist, reconstructed_dist)
+    
+    return {
+        "wasserstein_distance": wd,
+        "ks_statistic": ks_stat
+    }
+
+def validate_typology(df: pl.DataFrame, cluster_col: str, target_col: str) -> dict[str, float]:
+    """
+    Compare the aggregated cluster-based distribution of the target_col (energy performance)
+    against the actual global distribution in the original data.
+    """
+    # Original distribution
+    original_dist = df.select(target_col).to_numpy().flatten()
+    
+    # Cluster-based distribution (reconstructed)
+    # We assume the cluster representative value (e.g., mean) represents all members of that cluster
+    # We need to map each row to its cluster's mean value.
+    
+    # Calculate cluster means
+    cluster_means = df.group_by(cluster_col).agg(pl.col(target_col).mean().alias("mean_val"))
+    
+    # Join back to get the 'predicted' value for each row
+    reconstructed_df = df.join(cluster_means, on=cluster_col, how="left")
+    reconstructed_dist = reconstructed_df.select("mean_val").to_numpy().flatten()
+    
+    wd = wasserstein_distance(original_dist, reconstructed_dist)
+    ks_stat, _ = ks_2samp(original_dist, reconstructed_dist)
+    
+    return {
+        "wasserstein_distance": wd,
+        "ks_statistic": ks_stat
+    }
+
+def plot_medoid_validation(df: pl.DataFrame, medoids_df: pl.DataFrame,
+                           cluster_col: str, target_col: str,
+                           output_path: str = None,
+                           max_value: float = None,
+                           ):
+    """
+    Plots a 2D Histogram of actual target_col vs predicted target_col from medoids,
+    with the R2 of the regression on the plot.
+    """
+    # Original values
+    y_true = df.select(target_col).to_numpy().flatten()
+    
+    # Predicted values (medoid values)
+    medoid_values = medoids_df.select([cluster_col, target_col]).rename({target_col: "medoid_val"})
+    reconstructed_df = df.join(medoid_values, on=cluster_col, how="left")
+    y_pred = reconstructed_df.select("medoid_val").to_numpy().flatten()
+    
+    # Calculate R2
+    r2 = r2_score(y_true, y_pred)
+    
+    # Prepare data
+    x = y_true
+    y = y_pred
+    if max_value is not None:
+        mask = (x <= max_value) & (y <= max_value)
+        x = x[mask]
+        y = y[mask]
+    
+    # Plot
+    plt.figure(figsize=(10, 8))
+    
+    nbins = 50
+    plt.hist2d(x, y, bins=nbins, cmap=plt.cm.BuGn)
+    
+    # Add diagonal line
+    min_val = min(x.min(), y.min())
+    max_val = max(x.max(), y.max())
+    plt.plot([min_val, max_val], [min_val, max_val], 'r--', lw=2, label='Identity')
+    
+    plt.colorbar(label='Count')
+    plt.xlabel(f"Actual {target_col}")
+    plt.ylabel(f"Predicted {target_col} (Medoid)")
+    plt.title(f"Actual vs Predicted {target_col} 2D Histogram (R2 = {r2:.4f})")
+    plt.legend()
+    
+    if output_path:
+        plt.savefig(output_path)
+        plt.close()
+    else:
+        plt.show()
+
+def plot_distribution_comparison(df: pl.DataFrame, medoids_df: pl.DataFrame,
+                                 cluster_col: str, target_col: str,
+                                 output_path: str = None, max_value: float = None):
+    """
+    Plots histograms of the distribution of the target_col for the original data 
+    and the medoids of the cluster, taking into account the size of the cluster as weights.
+    """
+    # Original distribution
+    original_data = df.select(target_col).to_numpy().flatten()
+    
+    # Medoids distribution with weights
+    # Calculate cluster sizes
+    cluster_counts = df.group_by(cluster_col).count()
+    
+    # Join counts to medoids
+    medoids_with_counts = medoids_df.join(cluster_counts, on=cluster_col, how="left")
+
+    if max_value is not None:
+        medoids_with_counts = medoids_with_counts.filter(pl.col(target_col) <= max_value)
+        original_data = original_data[original_data <= max_value]
+
+    medoid_values = medoids_with_counts.select(target_col).to_numpy().flatten()
+    weights = medoids_with_counts.select("count").to_numpy().flatten()
+
+    plt.figure(figsize=(10, 6))
+    
+    # Plot original data
+    plt.hist(original_data, bins=50, alpha=0.5, label='Original Data', density=True)
+    
+    # Plot medoids weighted
+    plt.hist(medoid_values, bins=50, weights=weights, alpha=0.5, label='Medoids (Weighted)', density=True)
+    
+    plt.xlabel(target_col)
+    plt.ylabel('Density')
+    plt.title(f'Distribution Comparison: {target_col}')
+    plt.legend()
+    
+    if output_path:
+        plt.savefig(output_path)
+        plt.close()
+    else:
+        plt.show()

baticlust-0.1.0.dist-info/METADATA

@@ -0,0 +1,138 @@
+Metadata-Version: 2.4
+Name: baticlust
+Version: 0.1.0
+Summary: Building stock archetype generation using stratified clustering
+Author-email: Yassine Abdelouadoud <yassine.abdelouadoud@gmail.com>
+License-Expression: MIT
+Project-URL: Repository, https://gitlab.com/energytransition/baticlust
+Keywords: buildings,archetypes,clustering,energy,building stock
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: polars
+Requires-Dist: numpy
+Requires-Dist: faiss-cpu
+Requires-Dist: fastcluster
+Requires-Dist: scipy
+Requires-Dist: matplotlib
+Requires-Dist: scikit-learn
+Requires-Dist: joblib
+Requires-Dist: tqdm
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Dynamic: license-file
+
+# **BatiClust 🏘️**
+
+**BatiClust** is a Python framework designed to generate and validate building stock archetypes. It leverages unsupervised machine learning to segment residential dwellings into representative groups and provides a rigorous external validation suite using Energy Performance Certificate (EPC) data.  
+Unlike static typologies, **BatiClust** allows researchers to define archetypes at any level of granularity, quantifying the trade-off between model simplicity and its accuracy in representing national energy signatures.
+
+It is built on top of **Polars** for high-performance data manipulation and uses **Faiss** and **Fastcluster** for efficient clustering.
+
+## **Key Features**
+
+* **Stratified Subsampling:** Automatically split large building databases into homogeneous subsamples based on categorical attributes (e.g., construction era, heating system) before clustering.  
+* **Scalable Clustering:** Implements flexible clustering algorithms (K-Means via Faiss, Hierarchical via Fastcluster) to identify natural archetypes within building stocks.  
+* **External Validation:** A dedicated module to compare cluster-derived distributions against real-world energy performance data.  
+* **Medoid Selection:** Identifies representative building archetypes (medoids) for each cluster.
+
+## **Installation**
+
+```bash
+pip install baticlust
+```
+
+Note: This package requires `polars`, `numpy`, `faiss-cpu`, `fastcluster`, `scipy`, `matplotlib`, and `scikit-learn`.
+
+## **Quick Start**
+
+### **1. Data Preparation**
+
+Prepare your dataset with descriptive features and target performance metrics. BatiClust uses **Polars** DataFrames.
+
+```python
+import polars as pl
+from baticlust import scale_features
+
+# Load your data
+df = pl.read_parquet("housing_stock.parquet")
+
+# Define features
+strata_cols = ['construction_year_class', 'heating_energy_type']
+features = ['living_area', 'wall_insulation_thickness']
+
+# Scale numerical features
+df_scaled = scale_features(df, features, method="standard")
+```
+
+### **2. Stratified Clustering**
+
+BatiClust splits your database into strategic subsamples before applying clustering.
+
+```python
+from baticlust import generate_archetypes
+
+# Generate archetypes
+clustered_df = generate_archetypes(
+    df_scaled,
+    strata_cols=strata_cols,
+    features=features,
+    method="kmeans",
+    total_n_clusters=1000,
+    niter=20,
+    verbose=True
+)
+
+# The result is the original DataFrame with a new 'cluster_id' column
+print(clustered_df.head())
+```
+
+### **3. Medoid Selection & Validation**
+
+Identify representative archetypes and validate them against actual performance data.
+
+```python
+from baticlust import select_medoids, validate_typology_with_medoids
+
+# Create a unique cluster identifier if needed (combining strata and cluster_id)
+clustered_df = clustered_df.with_columns(
+    pl.concat_str([pl.col(c) for c in strata_cols] + [pl.col("cluster_id").cast(pl.Utf8)], separator="_")
+    .alias("unique_cluster_id")
+)
+
+# Select medoids (representative buildings)
+medoids_df = select_medoids(clustered_df, "unique_cluster_id", features)
+
+# Validate using a target column (e.g., energy consumption)
+metrics = validate_typology_with_medoids(
+    clustered_df,
+    medoids_df,
+    cluster_col="unique_cluster_id",
+    target_col="energy_consumption"
+)
+
+print(f"Wasserstein Distance: {metrics['wasserstein_distance']}")
+print(f"KS Statistic: {metrics['ks_statistic']}")
+```
+
+## **Methodology**
+
+The BatiClust workflow follows the methodology established in recent building stock research:
+
+1. **Pre-processing:** Cleaning and scaling of building characteristics using Polars.
+2. **Stratification:** Subdividing the stock into logical categories to ensure rare but important building types are not lost.
+3. **Clustering:** Applying algorithms like K-Means (Faiss) or Hierarchical Clustering (Fastcluster) to find representative centroids within each stratum.
+4. **Validation:** Using Kolmogorov-Smirnov tests and Wasserstein distance to verify if the clusters accurately mirror the energy performance distribution of the original stock.
+
+## **Citing BatiClust**
+
+If you use this package in your research, please cite the following paper:  
+*A Scalable Framework for Generating and Validating Dwelling Archetypes: A Clustering Approach for Energy Performance Analysis* (2026). \[Link to Paper\]
+
+## **License**
+
+This project is licensed under the MIT License - see the LICENSE file for details.

baticlust-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+baticlust/__init__.py,sha256=4f9mCJZrr3lj-w-g25zfAZrGWPHxQZMieUv08U0diG0,826
+baticlust/clustering.py,sha256=f9tIS_Fsues6BK9qLyJi1YJSN4Y8lHRLWQqOljQzR-E,4853
+baticlust/preprocessing.py,sha256=PeDbj6r9xMvfA64QRT1AGp_wWorob05BXhqUMO66G3E,837
+baticlust/validation.py,sha256=9VLKHIO1ERExt_PnvhukJugsfyuYtNTbWzEKYHNIKPo,6316
+baticlust-0.1.0.dist-info/licenses/LICENSE,sha256=wt5buagmQdj-zdsTpwY-IEOhYHda7UVdlD-fnvT3x1Q,1077
+baticlust-0.1.0.dist-info/METADATA,sha256=rjq2qODON6IUhTo29N7fOYF4IEXAAEhHfepr1M9VSZ4,5359
+baticlust-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+baticlust-0.1.0.dist-info/top_level.txt,sha256=4augevclK7P2_VvCRxiG9nAYWN6HfgelKpEyXQVrqtM,10
+baticlust-0.1.0.dist-info/RECORD,,

baticlust-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

baticlust-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Yassine Abdelouadoud
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

baticlust-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+baticlust