itertoolkit 1.5.0__py3-none-any.whl

bm_preprocessing/__init__.py

@@ -0,0 +1,14 @@
+"""bm_preprocessing package."""
+
+# Keep legacy module paths working, e.g. ``import bm_preprocessing.DM``.
+import sys
+
+from .importer import DM, IR, PY, Finals
+
+sys.modules[__name__ + ".DM"] = DM
+sys.modules[__name__ + ".PY"] = PY
+sys.modules[__name__ + ".IR"] = IR
+sys.modules[__name__ + ".Finals"] = Finals
+
+
+__all__ = ["DM", "PY", "IR"]

bm_preprocessing/importer/DM/__init__.py

@@ -0,0 +1,7 @@
+from .agg import agg
+from .dbscan import dbscan
+from .finals import finals
+from .gsp import gsp
+from .test import test
+
+__all__ = ["finals", "test", "agg", "dbscan", "gsp"]

bm_preprocessing/importer/DM/agg.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+from .._module_printer import SourceCodeModule
+
+_source_file = Path(__file__).parents[2] / "src" / "DM" / "agg.py"
+agg = SourceCodeModule("bm_preprocessing.DM.agg", _source_file)

bm_preprocessing/importer/DM/dbscan.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+from .._module_printer import SourceCodeModule
+
+_source_file = Path(__file__).parents[2] / "src" / "DM" / "dbscan.py"
+dbscan = SourceCodeModule("bm_preprocessing.DM.dbscan", _source_file)

bm_preprocessing/importer/DM/finals.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+from .._module_printer import SourceCodeModule
+
+_source_file = Path(__file__).parents[2] / "src" / "DM" / "finals.py"
+finals = SourceCodeModule("bm_preprocessing.DM.finals", _source_file)

bm_preprocessing/importer/DM/gsp.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+from .._module_printer import SourceCodeModule
+
+_source_file = Path(__file__).parents[2] / "src" / "DM" / "gsp.py"
+gsp = SourceCodeModule("bm_preprocessing.DM.gsp", _source_file)

bm_preprocessing/importer/DM/test.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+from .._module_printer import SourceCodeModule
+
+_source_file = Path(__file__).parents[2] / "src" / "DM" / "test.py"
+test = SourceCodeModule("bm_preprocessing.DM.test", _source_file)

bm_preprocessing/importer/Finals/__init__.py

@@ -0,0 +1,7 @@
+"""Finals source snippets."""
+from .kaadhal import kaadhal
+from .raaka import raaka
+from .seedan import seedan
+from .vikram import vikram
+
+__all__ = ["kaadhal", "raaka", "seedan", "vikram"]

bm_preprocessing/importer/Finals/kaadhal.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+from .._module_printer import SourceCodeModule
+
+_source_file = Path(__file__).parents[2] / "src" / "Finals" / "kaadhal.py"
+kaadhal = SourceCodeModule("bm_preprocessing.Finals.kaadhal", _source_file)

bm_preprocessing/importer/Finals/raaka.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+from .._module_printer import SourceCodeModule
+
+_source_file = Path(__file__).parents[2] / "src" / "Finals" / "raaka.py"
+raaka = SourceCodeModule("bm_preprocessing.Finals.raaka", _source_file)

bm_preprocessing/importer/Finals/seedan.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+from .._module_printer import SourceCodeModule
+
+_source_file = Path(__file__).parents[2] / "src" / "Finals" / "seedan.py"
+seedan = SourceCodeModule("bm_preprocessing.Finals.seedan", _source_file)

bm_preprocessing/importer/Finals/vikram.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+from .._module_printer import SourceCodeModule
+
+_source_file = Path(__file__).parents[2] / "src" / "Finals" / "vikram.py"
+vikram = SourceCodeModule("bm_preprocessing.Finals.vikram", _source_file)

bm_preprocessing/importer/IR/__init__.py

@@ -0,0 +1,6 @@
+from .finals import finals
+from .pagerank import pagerank
+from .recommenders_pca import recommenders_pca
+from .test import test
+
+__all__ = ["finals", "test", "pagerank", "recommenders_pca"]

bm_preprocessing/importer/IR/finals.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+from .._module_printer import SourceCodeModule
+
+_source_file = Path(__file__).parents[2] / "src" / "IR" / "finals.py"
+finals = SourceCodeModule("bm_preprocessing.IR.finals", _source_file)

bm_preprocessing/importer/IR/pagerank.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+from .._module_printer import SourceCodeModule
+
+_source_file = Path(__file__).parents[2] / "src" / "IR" / "pagerank.py"
+pagerank = SourceCodeModule("bm_preprocessing.IR.pagerank", _source_file)

bm_preprocessing/importer/IR/recommenders_pca.py

@@ -0,0 +1,8 @@
+from pathlib import Path
+
+from .._module_printer import SourceCodeModule
+
+_source_file = Path(__file__).parents[2] / "src" / "IR" / "recommenders_pca.py"
+recommenders_pca = SourceCodeModule(
+    "bm_preprocessing.IR.recommenders_pca", _source_file
+)

bm_preprocessing/importer/IR/test.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+from .._module_printer import SourceCodeModule
+
+_source_file = Path(__file__).parents[2] / "src" / "IR" / "test.py"
+test = SourceCodeModule("bm_preprocessing.IR.test", _source_file)

bm_preprocessing/importer/PY/__init__.py

@@ -0,0 +1,4 @@
+from .lib_doc import lib_doc
+from .python_doc import python_doc
+
+__all__ = ["lib_doc", "python_doc"]

bm_preprocessing/importer/PY/lib_doc.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+from .._module_printer import SourceCodeModule
+
+_source_file = Path(__file__).parents[2] / "src" / "PY" / "lib_doc.py"
+lib_doc = SourceCodeModule("bm_preprocessing.PY.lib_doc", _source_file)

bm_preprocessing/importer/PY/python_doc.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+from .._module_printer import SourceCodeModule
+
+_source_file = Path(__file__).parents[2] / "src" / "PY" / "python_doc.py"
+python_doc = SourceCodeModule("bm_preprocessing.PY.python_doc", _source_file)

bm_preprocessing/importer/__init__.py

@@ -0,0 +1,8 @@
+"""Importer layer that exposes printable source wrappers."""
+
+from .DM import *
+from .IR import *
+from .PY import *
+from .Finals import *
+
+__all__ = ["DM", "PY", "IR", "Finals"]

bm_preprocessing/importer/_module_printer.py

@@ -0,0 +1,23 @@
+from pathlib import Path
+
+
+class SourceCodeModule:
+    """A class that displays source code when printed."""
+
+    def __init__(self, name: str, source_path: Path):
+        self.name = name
+        self._source_path = source_path
+        self._source_code = None
+
+    @property
+    def source_code(self) -> str:
+        """Lazily load source code."""
+        if self._source_code is None:
+            self._source_code = self._source_path.read_text(encoding="utf-8")
+        return self._source_code
+
+    def __repr__(self) -> str:
+        return self.source_code
+
+    def __str__(self) -> str:
+        return self.source_code

bm_preprocessing/src/DM/__init__.py

@@ -0,0 +1 @@
+"""Data-mining source snippets."""

bm_preprocessing/src/DM/agg.py

@@ -0,0 +1,267 @@
+# -*- coding: utf-8 -*-
+"""PS13.ipynb
+
+Automatically generated by Colab.
+
+Original file is located at
+    https://colab.research.google.com/drive/141X72rA2VxgIbA93KoMkwHzHSc1fsbty
+
+# Agglomerative hierarchical clustering
+
+### Setup
+"""
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import plotly.figure_factory as ff
+import plotly.graph_objects as go
+import seaborn as sns
+from scipy.cluster.hierarchy import dendrogram, linkage
+from sklearn.cluster import AgglomerativeClustering
+from sklearn.preprocessing import LabelEncoder
+
+csv_path = "/content/drive/MyDrive/Academic_Resources/Semester 08/20XW87/PS13/mall_customers_dataset.csv"
+
+df = pd.read_csv(csv_path)
+display(df)
+
+"""### Remove CustomerID column"""
+
+df = df.drop(columns=["CustomerID"])
+display(df)
+
+"""### Check for missing values"""
+
+missing_counts = df.isna().sum()
+print(missing_counts)
+
+missing_counts = df.isna().sum()
+missing_percent = (missing_counts / len(df)) * 100
+print(missing_percent)
+
+print("Any missing values?", df.isna().values.any())
+
+for col in df.columns:
+    if df[col].isna().sum() == 0:
+        print(f"Column '{col}' has no missing values.")
+    else:
+        print(f"Column '{col}' has {df[col].isna().sum()} missing values.")
+
+"""### Convert categorical variable value of gender to numerical (Male-1, Female-0)"""
+
+le = LabelEncoder()
+df["gender_encoded"] = le.fit_transform(df["Gender"])
+
+print("Classes:", le.classes_)
+
+display(df)
+
+"""### Display male, female ratio as pie chart"""
+
+gender_counts = df["gender_encoded"].value_counts()
+labels = ["Female", "Male"]
+
+plt.figure(figsize=(5, 5))
+plt.pie(
+    gender_counts,
+    labels=labels,
+    autopct="%1.1f%%",
+    startangle=90,
+    colors=["#66b3ff", "#ff9999"],
+)
+plt.title("Gender Distribution")
+plt.show()
+
+"""### Display age, annual income as bar graph"""
+
+values = [df["Age"].mean(), df["Annual Income (k$)"].mean()]
+labels = ["Age", "Annual Income (k$)"]
+
+plt.bar(labels, values, color=["skyblue", "orange"])
+plt.title("Age and Annual Income")
+plt.ylabel("Value")
+plt.show()
+
+df["Age_group"] = pd.cut(df["Age"], bins=[18, 25, 35, 45, 55, 65], right=False)
+
+plt.figure(figsize=(8, 5))
+sns.barplot(x="Age_group", y="Annual Income (k$)", data=df)
+
+plt.title("Average Income by Age Group")
+plt.xticks(rotation=45)
+plt.show()
+
+sns.barplot(x="Age", y="Annual Income (k$)", data=df, estimator="mean")
+
+import plotly.express as px
+
+fig = px.bar(
+    df.groupby("Age")["Annual Income (k$)"].mean().reset_index(),
+    x="Age",
+    y="Annual Income (k$)",
+    title="Mean Income by Age",
+)
+fig.show()
+
+"""### Perform agglomerative clustering with ward algorithm as linkage."""
+
+agg_clust = AgglomerativeClustering(n_clusters=2, linkage="ward")
+df["Cluster"] = agg_clust.fit_predict(df[["Age", "Annual Income (k$)"]])
+
+print(df)
+
+linked = linkage(df[["Age", "Annual Income (k$)"]], method="ward")
+
+plt.figure(figsize=(20, 5))
+dendrogram(
+    linked, labels=df.index.tolist(), distance_sort="descending", show_leaf_counts=True
+)
+plt.title("Dendrogram")
+plt.xlabel("Sample index")
+plt.ylabel("Distance")
+plt.show()
+
+fig = ff.create_dendrogram(
+    df[["Age", "Annual Income (k$)"]].values,
+    labels=df.index.tolist(),
+    linkagefun=lambda x: linkage(x, "ward"),
+)
+fig.update_layout(width=1200, height=600)
+fig.show()
+
+"""## Custom
+
+### Impl
+"""
+
+
+def euclidean(a, b):
+    return np.sqrt(np.sum((a - b) ** 2))
+
+
+def ward_distance(cluster_i, cluster_j, data):
+    """
+    Ward's formula:
+    d(i,j) = sqrt( (2 * ni * nj) / (ni + nj) ) * ||mean_i - mean_j||
+    """
+    ni, nj = len(cluster_i), len(cluster_j)
+    mean_i = np.mean(data[cluster_i], axis=0)
+    mean_j = np.mean(data[cluster_j], axis=0)
+    return np.sqrt((2 * ni * nj) / (ni + nj)) * euclidean(mean_i, mean_j)
+
+
+def custom_agglomerative(data, n_clusters=2):
+    n = len(data)
+    clusters = {i: [i] for i in range(n)}
+    linkage_matrix = []
+    next_id = n
+
+    while len(clusters) > 1:
+        keys = list(clusters.keys())
+        min_dist = float("inf")
+        merge_a, merge_b = None, None
+
+        for i in range(len(keys)):
+            for j in range(i + 1, len(keys)):
+                d = ward_distance(clusters[keys[i]], clusters[keys[j]], data)
+                if d < min_dist:
+                    min_dist = d
+                    merge_a, merge_b = keys[i], keys[j]
+
+        new_members = clusters[merge_a] + clusters[merge_b]
+
+        id_a = float(min(merge_a, merge_b))
+        id_b = float(max(merge_a, merge_b))
+        prev_dist = linkage_matrix[-1][2] if linkage_matrix else 0.0
+        safe_dist = float(max(min_dist, prev_dist))
+
+        linkage_matrix.append([id_a, id_b, safe_dist, float(len(new_members))])
+
+        clusters[next_id] = new_members
+        del clusters[merge_a]
+        del clusters[merge_b]
+        next_id += 1
+
+    Z = np.array(linkage_matrix, dtype=float)
+
+    from scipy.cluster.hierarchy import fcluster
+
+    labels = fcluster(Z, t=n_clusters, criterion="maxclust") - 1
+
+    return labels, Z
+
+
+features = df[["Age", "Annual Income (k$)"]].values
+
+labels, Z = custom_agglomerative(features, n_clusters=2)
+df["Cluster"] = labels
+print(df)
+
+"""### Visualizations"""
+
+plt.figure(figsize=(8, 5))
+for c in np.unique(labels):
+    mask = labels == c
+    plt.scatter(features[mask, 0], features[mask, 1], label=f"Cluster {c}", s=60)
+plt.xlabel("Age")
+plt.ylabel("Annual Income (k$)")
+plt.title("Custom Agglomerative Clustering (Ward)")
+plt.legend()
+plt.tight_layout()
+plt.show()
+
+plt.figure(figsize=(12, 5))
+dendrogram(linked, truncate_mode="lastp", p=20, leaf_rotation=45, leaf_font_size=10)
+plt.title("Dendrogram (Ward Linkage)")
+plt.xlabel("Cluster index / size")
+plt.ylabel("Ward distance")
+plt.tight_layout()
+plt.show()
+
+ddata = dendrogram(Z, truncate_mode="lastp", p=20, no_plot=True)
+
+fig = go.Figure()
+for x_seg, y_seg in zip(ddata["icoord"], ddata["dcoord"]):
+    fig.add_trace(
+        go.Scatter(
+            x=x_seg,
+            y=y_seg,
+            mode="lines",
+            line=dict(color="royalblue", width=1.5),
+            showlegend=False,
+            hoverinfo="skip",
+        )
+    )
+
+fig.update_layout(
+    title="Dendrogram (Custom Ward Linkage)",
+    xaxis=dict(
+        title="Cluster index / size",
+        tickangle=45,
+        tickfont=dict(size=10),
+        showgrid=False,
+    ),
+    yaxis=dict(title="Ward Distance", showgrid=True, gridcolor="lightgrey"),
+    width=1200,
+    height=500,
+    plot_bgcolor="white",
+    paper_bgcolor="white",
+    showlegend=False,
+)
+fig.show()
+
+"""## Tests"""
+
+test_data = np.array([[1, 2], [1, 4], [5, 8], [5, 9], [8, 1], [8, 2]], dtype=float)
+
+# Custom
+_, Z_custom = custom_agglomerative(test_data, n_clusters=2)
+
+# Scipy
+from scipy.cluster.hierarchy import linkage
+
+Z_scipy = linkage(test_data, method="ward")
+
+print("Custom:\n", Z_custom.round(3))
+print("\nScipy:\n", Z_scipy.round(3))

bm_preprocessing/src/DM/dbscan.py

@@ -0,0 +1,218 @@
+import os
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from google.colab import drive
+from sklearn.metrics import (
+    adjusted_rand_score,
+    classification_report,
+    normalized_mutual_info_score,
+)
+from sklearn.preprocessing import StandardScaler
+
+drive.mount("/content/drive")
+
+csv_path = "/content/drive/MyDrive/Academic_Resources/Semester 08/20XW87/PS14/wholesale_customers_data.csv"
+df = pd.read_csv(csv_path)
+
+print(df.head())
+
+"""### DF Report"""
+
+print(df.describe())
+
+"""### Drop 'Channel' and 'Region' columns"""
+
+df = df.drop(columns=["Channel", "Region"])
+
+display(df.head())
+
+"""### Consider Groceries and Milk attributes. Normalize these attribute values by scaling it from 0 mean to unit variance. Visualize normalized dataset."""
+
+plt.scatter(df["Grocery"], df["Milk"], alpha=0.6)
+plt.title("Normalized Groceries vs Milk")
+plt.xlabel("Grocery (before normalized)")
+plt.ylabel("Milk (before normalized)")
+plt.show()
+
+# Select Groceries and Milk columns
+data_to_normalize = df[["Grocery", "Milk"]]
+
+# Normalize (StandardScaler standardizes to mean=0, std=1)
+scaler = StandardScaler()
+normalized_data = scaler.fit_transform(data_to_normalize)
+
+# Convert back to DataFrame for convenience
+normalized_df = pd.DataFrame(normalized_data, columns=["Grocery", "Milk"])
+
+print(normalized_df.mean())
+print(normalized_df.std())
+
+mean_grocery = normalized_df["Grocery"].mean()
+mean_milk = normalized_df["Milk"].mean()
+
+plt.figure(figsize=(7, 5))
+
+plt.scatter(normalized_df["Grocery"], normalized_df["Milk"], alpha=0.6)
+
+plt.axhline(mean_milk, linestyle="--", label=f"Milk Mean ({mean_milk:.2e})")
+plt.axvline(mean_grocery, linestyle="--", label=f"Grocery Mean ({mean_grocery:.2e})")
+
+plt.title("Normalized Grocery vs Milk")
+plt.xlabel("Grocery (Standardized)")
+plt.ylabel("Milk (Standardized)")
+
+plt.legend()
+plt.grid()
+plt.show()
+
+"""### Implementation of DBSCAN"""
+
+
+def euclidean_distance(p1, p2):
+    return np.sqrt(np.sum((p1 - p2) ** 2))
+
+
+def region_query(data, point_idx, eps):
+    neighbors = []
+    for i in range(len(data)):
+        if euclidean_distance(data[point_idx], data[i]) <= eps:
+            neighbors.append(i)
+    return neighbors
+
+
+def expand_cluster(data, labels, point_idx, neighbors, cluster_id, eps, min_pts):
+    labels[point_idx] = cluster_id
+    i = 0
+    while i < len(neighbors):
+        neighbor_idx = neighbors[i]
+        if labels[neighbor_idx] == -1:
+            labels[neighbor_idx] = cluster_id
+        elif labels[neighbor_idx] == 0:
+            labels[neighbor_idx] = cluster_id
+            new_neighbors = region_query(data, neighbor_idx, eps)
+            if len(new_neighbors) >= min_pts:
+                neighbors += new_neighbors
+        i += 1
+
+
+def dbscan(data, eps=0.5, min_pts=15):
+    labels = [0] * len(data)  # 0 means unvisited
+    cluster_id = 0
+
+    for point_idx in range(len(data)):
+        if labels[point_idx] != 0:
+            continue
+        neighbors = region_query(data, point_idx, eps)
+        if len(neighbors) < min_pts:
+            labels[point_idx] = -1  # Noise
+        else:
+            cluster_id += 1
+            expand_cluster(data, labels, point_idx, neighbors, cluster_id, eps, min_pts)
+
+    return labels
+
+
+# Run on normalized grocery & milk data
+data_np = normalized_df.values
+labels = dbscan(data_np, eps=0.5, min_pts=15)
+
+# Convert labels to numpy array for plotting
+labels = np.array(labels)
+
+plt.figure(figsize=(8, 6))
+unique_labels = set(labels)
+colors = plt.cm.get_cmap("tab10", len(unique_labels))
+
+for k in unique_labels:
+    class_member_mask = labels == k
+    xy = data_np[class_member_mask]
+    if k == -1:
+        # Noise in black
+        plt.scatter(xy[:, 0], xy[:, 1], c="k", marker="x", label="Noise")
+    else:
+        plt.scatter(xy[:, 0], xy[:, 1], c=[colors(k - 1)], label=f"Cluster {k}")
+plt.title("DBSCAN Clustering Results (Custom Implementation)")
+plt.xlabel("Grocery (normalized)")
+plt.ylabel("Milk (normalized)")
+plt.legend()
+plt.show()
+
+from sklearn.cluster import DBSCAN
+from sklearn.datasets import make_moons
+
+# Generate moon-shaped data
+X, _ = make_moons(n_samples=2000, noise=0.05)
+
+# Run DBSCAN
+dbscan_builtin = DBSCAN(eps=0.3, min_samples=15)
+labels_builtin = dbscan_builtin.fit_predict(X)
+
+# Plot
+plt.figure(figsize=(8, 6))
+unique_labels = set(labels_builtin)
+colors = plt.cm.get_cmap("tab10", len(unique_labels))
+
+for k in unique_labels:
+    class_member_mask = labels_builtin == k
+    xy = X[class_member_mask]
+    if k == -1:
+        plt.scatter(xy[:, 0], xy[:, 1], c="k", marker="x", label="Noise")
+    else:
+        plt.scatter(xy[:, 0], xy[:, 1], c=[colors(k)], label=f"Cluster {k}")
+plt.title("DBSCAN on make_moons")
+plt.legend()
+plt.show()
+
+# Generate synthetic moon-shaped data
+X, true_labels = make_moons(n_samples=2000, noise=0.05)
+
+# Custom DBSCAN on X
+custom_labels = dbscan(X, eps=0.3, min_pts=15)
+custom_labels = np.array(custom_labels)
+
+# Sklearn DBSCAN on X
+dbscan_builtin = DBSCAN(eps=0.3, min_samples=15)
+sklearn_labels = dbscan_builtin.fit_predict(X)
+
+# Clustering metrics comparing the two
+ari = adjusted_rand_score(custom_labels, sklearn_labels)
+nmi = normalized_mutual_info_score(custom_labels, sklearn_labels)
+
+print(f"Adjusted Rand Index (ARI) between custom and sklearn: {ari:.4f}")
+print(f"Normalized Mutual Information (NMI) between custom and sklearn: {nmi:.4f}")
+
+# Classification report (treating custom as true, sklearn as predicted)
+print("\nClassification report (custom as true, sklearn as predicted):")
+print(classification_report(custom_labels, sklearn_labels))
+
+"""### Add some noise data and again visualize the results."""
+
+# Add random noise points
+noise = np.random.uniform(low=-2, high=3, size=(200, 2))
+
+# Combine with original data
+X_noisy = np.vstack([X, noise])
+
+# Run DBSCAN again
+dbscan_noisy = DBSCAN(eps=0.3, min_samples=15)
+labels_noisy = dbscan_noisy.fit_predict(X_noisy)
+
+# Plot
+plt.figure(figsize=(8, 6))
+unique_labels = set(labels_noisy)
+colors = plt.cm.get_cmap("tab10", len(unique_labels))
+
+for k in unique_labels:
+    class_member_mask = labels_noisy == k
+    xy = X_noisy[class_member_mask]
+
+    if k == -1:
+        plt.scatter(xy[:, 0], xy[:, 1], c="k", marker="x", label="Noise")
+    else:
+        plt.scatter(xy[:, 0], xy[:, 1], c=[colors(k)], label=f"Cluster {k}")
+
+plt.title("DBSCAN with Added Noise")
+plt.legend()
+plt.show()