itertoolkit 1.5.0__py3-none-any.whl

bm_preprocessing/src/Finals/vikram.py

@@ -0,0 +1,520 @@
+# -*- coding: utf-8 -*-
+"""ir dm final code sem.ipynb
+
+Automatically generated by Colab.
+
+Original file is located at
+    https://colab.research.google.com/drive/1pShgFSMDWpmjt0iCqFTyic4hYGP5vgFc
+"""
+
+#-------------------------------------------------------------------------------------------------------#
+#IR CODE#
+#-------------------------------------------------------------------------------------------------------#
+
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+from sklearn.metrics.pairwise import cosine_similarity
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.decomposition import PCA
+from sklearn.preprocessing import StandardScaler
+import networkx as nx
+import os
+
+sns.set()
+
+figures = []
+
+print("Current Directory:", os.getcwd())
+print("Files in folder:", os.listdir())
+
+# ============================================================
+
+# 1. CONTENT-BASED RECOMMENDER SYSTEM
+
+# ============================================================
+
+try:
+    print("\nCONTENT-BASED RECOMMENDER SYSTEM")
+
+
+    movies = pd.read_csv("movies.csv")
+    movies.columns = movies.columns.str.lower().str.strip()
+
+    title_col = [c for c in movies.columns if 'title' in c][0]
+    genre_col = [c for c in movies.columns if 'genre' in c][0]
+
+    tfidf = TfidfVectorizer()
+    tfidf_matrix = tfidf.fit_transform(movies[genre_col])
+    cosine_sim = cosine_similarity(tfidf_matrix)
+
+    movie_idx = 0
+    scores = sorted(list(enumerate(cosine_sim[movie_idx])), key=lambda x: x[1], reverse=True)[1:4]
+
+    print("Recommendations for:", movies[title_col].iloc[movie_idx])
+    for i in scores:
+        print(movies[title_col].iloc[i[0]])
+
+    fig1 = plt.figure()
+    sns.heatmap(cosine_sim, cmap="Blues")
+    plt.title("Content Similarity")
+    figures.append(fig1)
+
+
+except Exception as e:
+    print("Error in Content-Based:", e)
+
+# ============================================================
+
+# 2. COLLABORATIVE FILTERING
+
+# ============================================================
+
+try:
+    print("\nCOLLABORATIVE FILTERING")
+
+
+    ratings = pd.read_csv("data.csv")
+    ratings.columns = ratings.columns.str.lower().str.strip()
+
+    user_col = [c for c in ratings.columns if 'user' in c][0]
+    item_col = [c for c in ratings.columns if 'movie' in c or 'item' in c][0]
+    rating_col = [c for c in ratings.columns if 'rating' in c][0]
+
+    user_item = ratings.pivot(index=user_col, columns=item_col, values=rating_col).fillna(0)
+    user_sim = cosine_similarity(user_item)
+
+    print("User Similarity Matrix:\n", user_sim)
+
+    fig2 = plt.figure()
+    sns.heatmap(user_item, cmap="coolwarm")
+    plt.title("User-Item Matrix")
+    figures.append(fig2)
+
+
+except Exception as e:
+    print("Error in Collaborative:", e)
+
+# ============================================================
+
+# 3. PCA
+
+# ============================================================
+
+try:
+    print("\nPCA")
+
+    data = pd.read_csv("data.csv")
+    data.columns = data.columns.str.lower().str.strip()
+
+    scaler = StandardScaler()
+    scaled = scaler.fit_transform(data)
+
+    pca = PCA(n_components=2)
+    result = pca.fit_transform(scaled)
+
+    print("Reduced Data:\n", result)
+
+    fig3 = plt.figure()
+    plt.scatter(result[:, 0], result[:, 1])
+    plt.title("PCA Projection")
+    figures.append(fig3)
+
+
+except Exception as e:
+    print("Error in PCA:", e)
+
+# ============================================================
+
+# 4. PAGERANK
+
+# ============================================================
+
+try:
+    print("\nPAGERANK")
+
+    graph = pd.read_csv("graph.csv")
+    graph.columns = graph.columns.str.lower().str.strip()
+
+    source_col = graph.columns[0]
+    target_col = graph.columns[1]
+
+    G = nx.from_pandas_edgelist(graph, source_col, target_col, create_using=nx.DiGraph())
+    pr = nx.pagerank(G)
+
+    print("PageRank Scores:")
+    for k, v in pr.items():
+        print(k, ":", round(v, 4))
+
+    fig4 = plt.figure()
+    nx.draw(G, with_labels=True, node_color='lightblue')
+    plt.title("Web Graph")
+    figures.append(fig4)
+
+except Exception as e:
+    print("Error in PageRank:", e)
+
+# ============================================================
+
+# 5. INVERTED INDEX
+
+# ============================================================
+
+try:
+    print("\nINVERTED INDEX")
+
+    docs = pd.read_csv("documents.csv")
+    docs.columns = docs.columns.str.lower().str.strip()
+
+    id_col = [c for c in docs.columns if 'id' in c][0]
+    text_col = [c for c in docs.columns if 'content' in c or 'text' in c][0]
+
+    index = {}
+
+    for _, row in docs.iterrows():
+        words = str(row[text_col]).lower().split()
+        for w in words:
+            if w not in index:
+                index[w] = []
+            index[w].append(row[id_col])
+
+    query = "data"
+    print("Search:", query)
+
+    if query in index:
+        for doc_id in index[query]:
+            print(docs[docs[id_col] == doc_id][text_col].values[0])
+    else:
+        print("No results")
+
+
+except Exception as e:
+    print("Error in IR:", e)
+
+# ============================================================
+
+# SHOW ALL PLOTS
+
+# ============================================================
+print("\nShowing all plots...")
+
+# Activate all figures created
+for num in plt.get_fignums():
+    plt.figure(num)
+
+# Display all together
+plt.show()
+
+#-------------------------------------------------------------------------------------------------------#
+#DM CODE#
+#-------------------------------------------------------------------------------------------------------#
+# ============================================================
+# PROBLEM SHEET 13 – AGGLOMERATIVE CLUSTERING (WARD)
+# ============================================================
+
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.cluster.hierarchy import dendrogram, linkage
+
+print("\n===== AGGLOMERATIVE CLUSTERING =====")
+
+# Load dataset
+df = pd.read_csv("Mall_Customers.csv")
+
+# Remove CustomerID
+df = df.drop("CustomerID", axis=1)
+
+# Missing values
+print("Missing values:\n", df.isnull().sum())
+
+# Convert Gender
+df['Gender'] = df['Gender'].map({'Male': 1, 'Female': 0})
+
+# Pie chart
+gender_counts = df['Gender'].value_counts()
+plt.figure()
+plt.pie(gender_counts, labels=['Male', 'Female'], autopct='%1.1f%%')
+plt.title("Male vs Female Ratio")
+plt.show()
+
+# Bar graphs
+plt.figure()
+plt.bar(range(len(df)), df['Age'])
+plt.title("Age Distribution")
+plt.show()
+
+plt.figure()
+plt.bar(range(len(df)), df['Annual Income (k$)'])
+plt.title("Income Distribution")
+plt.show()
+
+# Ward linkage clustering
+data = df[['Age', 'Annual Income (k$)', 'Spending Score (1-100)']].values
+Z = linkage(data, method='ward')
+
+plt.figure(figsize=(10,5))
+dendrogram(Z)
+plt.title("Dendrogram (Ward Linkage)")
+plt.xlabel("Data Points")
+plt.ylabel("Distance")
+plt.show()
+
+
+# ============================================================
+# PROBLEM SHEET 14 – DBSCAN
+# ============================================================
+
+from sklearn.preprocessing import StandardScaler
+from sklearn.datasets import make_moons
+from sklearn.cluster import DBSCAN
+
+print("\n===== DBSCAN CLUSTERING =====")
+
+# Load dataset
+df = pd.read_csv("Wholesale customers data.csv")
+df = df.drop(['Channel', 'Region'], axis=1)
+
+print("First few records:\n", df.head())
+
+# Normalize
+data = df[['Grocery', 'Milk']].values
+scaler = StandardScaler()
+data_scaled = scaler.fit_transform(data)
+
+plt.scatter(data_scaled[:,0], data_scaled[:,1])
+plt.title("Normalized Data")
+plt.xlabel("Grocery")
+plt.ylabel("Milk")
+plt.show()
+
+# Manual DBSCAN
+EPS = 0.5
+MINPTS = 15
+
+UNVISITED = 0
+NOISE = -1
+
+labels = [UNVISITED] * len(data_scaled)
+cluster_id = 0
+
+def euclidean(p1, p2):
+    return np.sqrt(np.sum((p1 - p2)**2))
+
+def region_query(idx):
+    neighbors = []
+    for i in range(len(data_scaled)):
+        if euclidean(data_scaled[idx], data_scaled[i]) <= EPS:
+            neighbors.append(i)
+    return neighbors
+
+def expand_cluster(idx, neighbors, cluster_id):
+    labels[idx] = cluster_id
+
+    i = 0
+    while i < len(neighbors):
+        point = neighbors[i]
+
+        if labels[point] == UNVISITED:
+            labels[point] = cluster_id
+            new_neighbors = region_query(point)
+
+            if len(new_neighbors) >= MINPTS:
+                neighbors += new_neighbors
+
+        elif labels[point] == NOISE:
+            labels[point] = cluster_id
+
+        i += 1
+
+# Main loop
+for i in range(len(data_scaled)):
+    if labels[i] != UNVISITED:
+        continue
+
+    neighbors = region_query(i)
+
+    if len(neighbors) < MINPTS:
+        labels[i] = NOISE
+    else:
+        cluster_id += 1
+        expand_cluster(i, neighbors, cluster_id)
+
+# Plot clusters
+plt.scatter(data_scaled[:,0], data_scaled[:,1], c=labels, cmap='rainbow')
+plt.title("Manual DBSCAN Clustering")
+plt.show()
+
+# Built-in DBSCAN (moon dataset)
+X, _ = make_moons(n_samples=2000, noise=0.05)
+
+model = DBSCAN(eps=0.2, min_samples=5)
+labels_moon = model.fit_predict(X)
+
+plt.scatter(X[:,0], X[:,1], c=labels_moon, cmap='rainbow')
+plt.title("DBSCAN on Moon Dataset")
+plt.show()
+
+# Add noise
+noise = np.random.uniform(low=-1.5, high=2.5, size=(200,2))
+X_noisy = np.vstack([X, noise])
+
+labels_noisy = model.fit_predict(X_noisy)
+
+plt.scatter(X_noisy[:,0], X_noisy[:,1], c=labels_noisy, cmap='rainbow')
+plt.title("DBSCAN with Noise")
+plt.show()
+
+
+# ============================================================
+# PROBLEM SHEET 15 – MS-GSP (FULLY CORRECT)
+# ============================================================
+
+import re
+
+print("\n===== MS-GSP SEQUENTIAL PATTERN MINING =====")
+
+# Read parameters
+MIS = {}
+SDC = 0
+
+def convert(x):
+    try:
+        return int(x)
+    except:
+        return x
+
+with open("para.txt") as f:
+    for line in f:
+        if "MIS" in line:
+            item = re.findall(r'\((.*?)\)', line)[0]
+            item = convert(item)
+            MIS[item] = float(line.split("=")[1])
+        elif "SDC" in line:
+            SDC = float(line.split("=")[1])
+
+# Read data
+def parse_sequence(line):
+    sets = re.findall(r'\{(.*?)\}', line)
+    return [set(convert(x) for x in s.split(',')) for s in sets]
+
+sequences = []
+with open("data (1).txt") as f:
+    for line in f:
+        sequences.append(parse_sequence(line.strip()))
+
+N = len(sequences)
+
+# Support
+def item_support(item):
+    return sum(1 for seq in sequences if any(item in s for s in seq)) / N
+
+support = {i: item_support(i) for i in MIS}
+
+# Subsequence check
+def is_subsequence(pattern, sequence):
+    i = 0
+    for itemset in sequence:
+        if pattern[i].issubset(itemset):
+            i += 1
+            if i == len(pattern):
+                return True
+    return False
+
+def sequence_support(pattern):
+    return sum(1 for seq in sequences if is_subsequence(pattern, seq))
+
+# Init pass
+L = sorted(MIS.keys(), key=lambda x: MIS[x])
+F = []
+
+F1 = []
+for item in L:
+    if support[item] >= MIS[item]:
+        F1.append([{item}])
+
+F.append(F1)
+print("F1:", F1)
+
+# Join
+def join_step(Fk):
+    candidates = []
+    for s1 in Fk:
+        for s2 in Fk:
+            if s1[1:] == s2[:-1]:
+                candidates.append(s1 + [s2[-1]])
+
+            if len(s1) == len(s2) and s1[:-1] == s2[:-1]:
+                new_seq = s1[:-1] + [s1[-1] | s2[-1]]
+                if new_seq not in candidates:
+                    candidates.append(new_seq)
+    return candidates
+
+# Prune
+def prune_step(candidates, Fk):
+    pruned = []
+    Fk_set = [tuple(tuple(sorted(s)) for s in seq) for seq in Fk]
+
+    for c in candidates:
+        valid = True
+        for i in range(len(c)):
+            sub = c[:i] + c[i+1:]
+            sub_tuple = tuple(tuple(sorted(s)) for s in sub)
+
+            if sub_tuple not in Fk_set:
+                valid = False
+                break
+
+        if valid:
+            pruned.append(c)
+
+    return pruned
+
+# Main loop
+k = 1
+while True:
+    if len(F[k-1]) == 0:
+        break
+
+    Ck = prune_step(join_step(F[k-1]), F[k-1])
+    Fk = []
+
+    for c in Ck:
+        count = sequence_support(c)
+
+        items = set()
+        for s in c:
+            items.update(s)
+
+        min_mis = min(MIS[i] for i in items)
+
+        if (count / N) >= min_mis:
+            valid = True
+            item_list = list(items)
+
+            for i in range(len(item_list)):
+                for j in range(i+1, len(item_list)):
+                    if abs(support[item_list[i]] - support[item_list[j]]) > SDC:
+                        valid = False
+                        break
+
+            if valid:
+                Fk.append(c)
+
+    if not Fk:
+        break
+
+    print(f"F{k+1}:", Fk)
+    F.append(Fk)
+    k += 1
+
+# Output
+def format_pattern(p):
+    return "<" + "".join(
+        ["{" + ",".join(map(str, sorted(s))) + "}" for s in p]
+    ) + ">"
+
+print("\nFINAL PATTERNS:")
+for level in F:
+    for p in level:
+        print(f"Pattern :{format_pattern(p)} count: {sequence_support(p)}")

bm_preprocessing/src/IR/__init__.py

@@ -0,0 +1 @@
+"""IR source snippets."""

bm_preprocessing/src/IR/finals.py

@@ -0,0 +1,14 @@
+"IR source snippets."
+
+import pandas as pd
+
+# Dimensionality reduction source snippets
+
+df = pd.DataFrame(
+    {
+        "A": [1, 2, 3],
+        "B": [4, 5, 6],
+        "C": [7, 8, 9],
+    }
+)
+print(df)

bm_preprocessing/src/IR/pagerank.py

@@ -0,0 +1,109 @@
+import matplotlib.pyplot as plt
+import networkx as nx
+import numpy as np
+import pandas as pd
+
+# ===== INPUT =====
+# rows = source page, cols = destination page
+pages = ["A", "B", "C", "D"]
+A = np.array(
+    [
+        [0, 1, 1, 0],  # A -> B,C
+        [0, 0, 1, 0],  # B -> C
+        [1, 0, 0, 0],  # C -> A
+        [0, 0, 1, 0],  # D -> C
+    ],
+    dtype=int,
+)
+
+d = 0.85
+max_iter = 10
+tol = 1e-8
+
+# ===== PREP =====
+n = len(pages)
+base = (1 - d) / n
+adj = pd.DataFrame(A, index=pages, columns=pages)
+
+incoming = {p: [src for src in pages if adj.loc[src, p] == 1] for p in pages}
+outgoing = {p: int(adj.loc[p].sum()) for p in pages}
+pr = {p: 1 / n for p in pages}
+
+print("Step 1: Web Graph")
+print(adj)
+print("\nTotal pages N =", n)
+print("\nInitial PageRank:")
+print(pd.DataFrame({"Page": pages, "Initial PR": [pr[p] for p in pages]}))
+
+print("\nStep 2: Incoming Links")
+print(
+    pd.DataFrame(
+        {
+            "Page": pages,
+            "Incoming Links": [
+                ", ".join(incoming[p]) if incoming[p] else "—" for p in pages
+            ],
+        }
+    )
+)
+
+print("\nOutgoing Links Count")
+print(pd.DataFrame({"Page": pages, "Outgoing Links": [outgoing[p] for p in pages]}))
+
+# ===== ITERATIONS =====
+history = [pr.copy()]
+
+for it in range(1, max_iter + 1):
+    new_pr = {}
+    print(f"\n\n===== Iteration {it} =====")
+    for p in pages:
+        s = 0
+        terms, nums = [], []
+        for src in incoming[p]:
+            c = pr[src] / outgoing[src] if outgoing[src] > 0 else 0
+            s += c
+            terms.append(f"PR({src})/{outgoing[src]}")
+            nums.append(f"{pr[src]:.4f}/{outgoing[src]}={c:.4f}")
+        new_pr[p] = base + d * s
+
+        print(f"\nPage {p}")
+        print("Incoming links:", ", ".join(incoming[p]) if incoming[p] else "—")
+        if terms:
+            print(f"PR({p}) = {base:.4f} + {d} * (" + " + ".join(terms) + ")")
+            print("Substitution:", " + ".join(nums))
+        else:
+            print(f"PR({p}) = {base:.4f}")
+        print(f"PR({p}) = {new_pr[p]:.6f}")
+
+    history.append(new_pr.copy())
+    diff = sum(abs(new_pr[p] - pr[p]) for p in pages)
+    print("\nPageRank after this iteration:")
+    print(
+        pd.DataFrame(
+            {"Page": pages, f"PR Iteration {it}": [round(new_pr[p], 6) for p in pages]}
+        )
+    )
+    pr = new_pr
+    if diff < tol:
+        break
+
+# ===== FINAL RESULT =====
+print("\n\nFinal PageRank:")
+final_df = pd.DataFrame(
+    {"Page": pages, "Final PR": [pr[p] for p in pages]}
+).sort_values("Final PR", ascending=False)
+print(final_df)
+
+# ===== OPTIONAL VISUALIZATION =====
+G = nx.DiGraph()
+for i, src in enumerate(pages):
+    for j, dst in enumerate(pages):
+        if A[i, j] == 1:
+            G.add_edge(src, dst)
+
+plt.figure(figsize=(6, 4))
+pos = nx.spring_layout(G, seed=42)
+sizes = [pr[p] * 5000 + 800 for p in G.nodes()]
+nx.draw(G, pos, with_labels=True, node_size=sizes, arrows=True)
+plt.title("Graph Visualization (node size ~ Final PageRank)")
+plt.show()