bm-preprocessing 0.1.0__tar.gz

bm_preprocessing-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv

bm_preprocessing-0.1.0/PKG-INFO

@@ -0,0 +1,33 @@
+Metadata-Version: 2.4
+Name: bm-preprocessing
+Version: 0.1.0
+Summary: A package to view source code from DM and IR modules
+Requires-Python: >=3.8
+Requires-Dist: build>=1.2.2.post1
+Requires-Dist: twine>=6.1.0
+Description-Content-Type: text/markdown
+
+# bm-preprocessing
+
+A Python package that displays source code from DM and IR modules.
+
+## Usage
+
+```python
+from bm_preprocessing.IR import all
+print(all)  # Prints the entire source code of IR/all.py
+
+from bm_preprocessing.DM import apriori
+print(apriori)  # Prints the entire source code of DM/apriori.py
+```
+
+## Available Modules
+
+### IR
+- `all` - Information Retrieval algorithms
+
+### DM
+- `all` - Data Mining algorithms
+- `apriori` - Apriori algorithm implementation
+- `hash` - Hash-based mining
+- `preprocessing` - Data preprocessing utilities

bm_preprocessing-0.1.0/README.md

@@ -0,0 +1,24 @@
+# bm-preprocessing
+
+A Python package that displays source code from DM and IR modules.
+
+## Usage
+
+```python
+from bm_preprocessing.IR import all
+print(all)  # Prints the entire source code of IR/all.py
+
+from bm_preprocessing.DM import apriori
+print(apriori)  # Prints the entire source code of DM/apriori.py
+```
+
+## Available Modules
+
+### IR
+- `all` - Information Retrieval algorithms
+
+### DM
+- `all` - Data Mining algorithms
+- `apriori` - Apriori algorithm implementation
+- `hash` - Hash-based mining
+- `preprocessing` - Data preprocessing utilities

bm_preprocessing-0.1.0/pyproject.toml

@@ -0,0 +1,17 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "bm-preprocessing"
+version = "0.1.0"
+description = "A package to view source code from DM and IR modules"
+readme = "README.md"
+requires-python = ">=3.8"
+dependencies = [
+    "build>=1.2.2.post1",
+    "twine>=6.1.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/bm_preprocessing"]

bm_preprocessing-0.1.0/src/bm_preprocessing/DM/__init__.py

@@ -0,0 +1,8 @@
+"""DM subpackage - Data Mining source code."""
+
+from .all import all
+from .apriori import apriori
+from .hash import hash
+from .preprocessing import preprocessing
+
+__all__ = ["all", "apriori", "hash", "preprocessing"]

bm_preprocessing-0.1.0/src/bm_preprocessing/DM/all.py

@@ -0,0 +1,30 @@
+"""Source code loader for DM/all.py"""
+
+from pathlib import Path
+
+
+class SourceCodeModule:
+    """A class that displays source code when printed."""
+    
+    def __init__(self, name: str, source_path: Path):
+        self.name = name
+        self._source_path = source_path
+        self._source_code = None
+    
+    @property
+    def source_code(self) -> str:
+        """Lazily load source code."""
+        if self._source_code is None:
+            self._source_code = self._source_path.read_text(encoding='utf-8')
+        return self._source_code
+    
+    def __repr__(self) -> str:
+        return self.source_code
+    
+    def __str__(self) -> str:
+        return self.source_code
+
+
+# Get the path to the source file
+_source_file = Path(__file__).parent / "sources" / "all.py"
+all = SourceCodeModule("DM.all", _source_file)

bm_preprocessing-0.1.0/src/bm_preprocessing/DM/apriori.py

@@ -0,0 +1,30 @@
+"""Source code loader for DM/apriori.py"""
+
+from pathlib import Path
+
+
+class SourceCodeModule:
+    """A class that displays source code when printed."""
+    
+    def __init__(self, name: str, source_path: Path):
+        self.name = name
+        self._source_path = source_path
+        self._source_code = None
+    
+    @property
+    def source_code(self) -> str:
+        """Lazily load source code."""
+        if self._source_code is None:
+            self._source_code = self._source_path.read_text(encoding='utf-8')
+        return self._source_code
+    
+    def __repr__(self) -> str:
+        return self.source_code
+    
+    def __str__(self) -> str:
+        return self.source_code
+
+
+# Get the path to the source file
+_source_file = Path(__file__).parent / "sources" / "apriori.py"
+apriori = SourceCodeModule("DM.apriori", _source_file)

bm_preprocessing-0.1.0/src/bm_preprocessing/DM/hash.py

@@ -0,0 +1,30 @@
+"""Source code loader for DM/hash.py"""
+
+from pathlib import Path
+
+
+class SourceCodeModule:
+    """A class that displays source code when printed."""
+    
+    def __init__(self, name: str, source_path: Path):
+        self.name = name
+        self._source_path = source_path
+        self._source_code = None
+    
+    @property
+    def source_code(self) -> str:
+        """Lazily load source code."""
+        if self._source_code is None:
+            self._source_code = self._source_path.read_text(encoding='utf-8')
+        return self._source_code
+    
+    def __repr__(self) -> str:
+        return self.source_code
+    
+    def __str__(self) -> str:
+        return self.source_code
+
+
+# Get the path to the source file
+_source_file = Path(__file__).parent / "sources" / "hash.py"
+hash = SourceCodeModule("DM.hash", _source_file)

bm_preprocessing-0.1.0/src/bm_preprocessing/DM/preprocessing.py

@@ -0,0 +1,30 @@
+"""Source code loader for DM/preprocessing.py"""
+
+from pathlib import Path
+
+
+class SourceCodeModule:
+    """A class that displays source code when printed."""
+    
+    def __init__(self, name: str, source_path: Path):
+        self.name = name
+        self._source_path = source_path
+        self._source_code = None
+    
+    @property
+    def source_code(self) -> str:
+        """Lazily load source code."""
+        if self._source_code is None:
+            self._source_code = self._source_path.read_text(encoding='utf-8')
+        return self._source_code
+    
+    def __repr__(self) -> str:
+        return self.source_code
+    
+    def __str__(self) -> str:
+        return self.source_code
+
+
+# Get the path to the source file
+_source_file = Path(__file__).parent / "sources" / "preprocessing.py"
+preprocessing = SourceCodeModule("DM.preprocessing", _source_file)

bm_preprocessing-0.1.0/src/bm_preprocessing/DM/sources/all.py

@@ -0,0 +1,157 @@
+from itertools import combinations, chain
+from collections import defaultdict
+
+min_support = 2
+min_conf = 0.7
+
+transactions = {
+    "T1": {"I1","I2","I4","I5","I6"},
+    "T2": {"I2","I4","I6"},
+    "T3": {"I2","I3"},
+    "T4": {"I1","I2","I4"},
+    "T5": {"I1","I2","I3"},
+    "T6": {"I2","I3"},
+    "T7": {"I1","I3"},
+    "T8": {"I1","I2","I3","I5"},
+    "T9": {"I1","I2","I3"},
+    "T10": {"I1","I2","I4","I5"},
+    "T11": {"I5","I6"}
+}
+
+genL = lambda C: {k: v for k, v in C.items() if v >= min_support}
+
+C, L = {}, {}
+
+C[1] = defaultdict(int)
+for t in transactions.values():
+    for i in t:
+        C[1][frozenset([i])] += 1
+L[1] = genL(C[1])
+
+h = lambda p, b=7: sum(sum(ord(c) for c in s) for s in p) % b
+buckets = defaultdict(int)
+for t in transactions.values():
+    for p in combinations(sorted(t), 2):
+        buckets[h(p)] += 1
+freq = {b for b, c in buckets.items() if c >= min_support}
+
+C[2] = defaultdict(int)
+for t in transactions.values():
+    for p in combinations(sorted(t), 2):
+        if h(p) in freq:
+            C[2][frozenset(p)] += 1
+L[2] = genL(C[2])
+
+k = 3
+while L[k - 1]:
+    C[k] = {
+        frozenset(a | b): 0
+        for a, b in combinations(L[k - 1], 2)
+        if sorted(a)[: k - 2] == sorted(b)[: k - 2]
+        and all(frozenset(s) in L[k - 1] for s in combinations(a | b, k - 1))
+    }
+    for t in transactions.values():
+        for c in C[k]:
+            if c.issubset(t):
+                C[k][c] += 1
+    L[k] = genL(C[k])
+    if not L[k]:
+        break
+    k += 1
+
+frequent_itemsets = {k: v for Lk in L.values() for k, v in Lk.items()}
+total = len(transactions)
+rules = []
+for itemset, count in frequent_itemsets.items():
+    if len(itemset) < 2:
+        continue
+    for a in chain.from_iterable(
+        combinations(itemset, r) for r in range(1, len(itemset))
+    ):
+        antecedent = frozenset(a)
+        consequent = itemset - antecedent
+        if not consequent:
+            continue
+        support = count / total
+        confidence = (
+            count / C[1 if len(antecedent) == 1 else len(antecedent)][antecedent]
+        )
+        if confidence >= min_conf:
+            rules.append((antecedent, consequent, support, confidence))
+
+for k, v in C.items():
+    print(
+        f"\nC{k}:\n",
+        "Empty" if not v else "\n".join(f"{set(x)} : {y}" for x, y in v.items()),
+    )
+for k, v in L.items():
+    print(f"\nL{k}:\n", "Empty" if not v else "\n".join(f"{set(x)} : {v[x]}" for x in v))
+
+print(f"\nAssociation Rules (conf >= {min_conf:.0%}):")
+for a, c, s, conf in rules:
+    print(f"{set(a)} => {set(c)} | support: {s:.2f}, confidence: {conf:.2f}")
+
+from itertools import combinations, chain
+
+transactions = {
+    "10": {"A", "C", "D"},
+    "20": {"B", "C", "E"},
+    "30": {"A", "B", "C", "E"},
+    "40": {"B", "E"},
+}
+
+min_support = 2
+min_conf = 0.7
+
+genL = lambda C: {k: v for k, v in C.items() if len(v) >= min_support}
+
+C = {1: {}}
+for tid, items in transactions.items():
+    for i in items:
+        C[1].setdefault(frozenset([i]), set()).add(tid)
+
+L = {1: genL(C[1])}
+
+k = 2
+while L[k - 1]:
+    prev = list(L[k - 1].keys())
+    C[k] = {
+        frozenset(a | b): L[k - 1][a] & L[k - 1][b]
+        for i, a in enumerate(prev)
+        for b in prev[i + 1 :]
+        if sorted(a)[: k - 2] == sorted(b)[: k - 2]
+    }
+    L[k] = genL(C[k])
+    if not L[k]:
+        break
+    k += 1
+
+frequent_itemsets = {k: v for Lk in L.values() for k, v in Lk.items()}
+total = len(transactions)
+rules = []
+for itemset, tids in frequent_itemsets.items():
+    if len(itemset) < 2:
+        continue
+    for a in chain.from_iterable(
+        combinations(itemset, r) for r in range(1, len(itemset))
+    ):
+        antecedent = frozenset(a)
+        consequent = itemset - antecedent
+        if len(consequent) == 0:
+            continue
+        support = len(tids) / total
+        confidence = len(tids) / len(frequent_itemsets[antecedent])
+        if confidence >= min_conf:
+            rules.append((antecedent, consequent, support, confidence))
+
+for k, v in C.items():
+    print(f"\nC{k}:")
+    print("Empty" if not v else "\n".join(f"{set(x)} : {y}" for x, y in v.items()))
+
+for k, v in L.items():
+    print(f"\nL{k}:")
+    print("Empty" if not v else "\n".join(f"{set(x)} : {len(y)}" for x, y in v.items()))
+
+print(f"\nAssociation Rules (conf >= {min_conf:.0%}):")
+for a, c, s, conf in rules:
+    print(f"{set(a)} => {set(c)} | support: {s:.2f}, confidence: {conf:.2f}")

bm_preprocessing-0.1.0/src/bm_preprocessing/DM/sources/apriori.py

@@ -0,0 +1,107 @@
+from collections import defaultdict
+from itertools import combinations
+
+def print_table(data, title):
+    print(f"\n--- {title} ---")
+    for itemset, count in data.items():
+        print(f"{itemset}: {count}")
+
+C = {}
+L = {}
+
+
+def generate_candidates(prev_frequent_itemsets, k):
+
+    candidates = set()
+
+    for itemset1, itemset2 in combinations(prev_frequent_itemsets, 2):
+        union_set = set(itemset1).union(set(itemset2))
+        if len(union_set) == k:
+            candidates.add(tuple(sorted(union_set)))
+            
+    return sorted(list(candidates))
+
+
+def count_candidates(candidates, transactions):
+    
+    candidate_count = defaultdict(int)
+    
+    for candidate in candidates:
+        for transaction in transactions.values():
+            if all(item in transaction for item in candidate):
+                candidate_count[candidate] += 1
+    
+    return candidate_count
+
+
+def prune_candidates(candidate_count, min_support, prev_freq_itemsets=None):
+
+    filtered_candidates = {}
+    
+    for itemset, count in candidate_count.items():
+
+        if count >= min_support:
+            if prev_freq_itemsets is None or len(itemset) == 1:
+                filtered_candidates[itemset] = count
+            else:
+                subsets = combinations(itemset, len(itemset) - 1)
+                if all(tuple(sorted(subset)) in prev_freq_itemsets for subset in subsets):
+                    filtered_candidates[itemset] = count
+
+    return filtered_candidates
+
+
+def apriori(transactions, min_support):
+
+    items = sorted(set(item for transaction in transactions.values() for item in transaction))
+    c1_list = [(item,) for item in items]
+    
+    C[1] = count_candidates(c1_list, transactions)
+    L[1] = prune_candidates(C[1], min_support)
+    
+    print_table(C[1], "Candidate 1-itemsets (C1)")
+    print_table(L[1], "Frequent 1-itemsets (L1)")
+
+    k = 2
+
+    while True:
+
+        candidates = generate_candidates(L[k-1].keys(), k)
+        if not candidates:
+            break
+
+        C[k] = count_candidates(candidates, transactions)
+        L[k] = prune_candidates(C[k], min_support, L[k-1].keys())
+
+        if not L[k]:
+            print_table(C[k], f"Candidate {k}-itemsets (C{k})")
+            print(f"\nNo frequent {k}-itemsets found. Terminating.\n\n")
+            break
+
+        print_table(C[k], f"Candidate {k}-itemsets (C{k})")
+        print_table(L[k], f"Frequent {k}-itemsets (L{k})")
+
+        k += 1
+
+
+def main():
+
+    transactions = {
+        "T100": ["I1", "I2", "I5"],
+        "T200": ["I2", "I4"],
+        "T300": ["I2", "I3"],
+        "T400": ["I1", "I2", "I4"],
+        "T500": ["I1", "I3"],
+        "T600": ["I2", "I3"],
+        "T700": ["I1", "I3"],
+        "T800": ["I1", "I2", "I3", "I5"],
+        "T900": ["I1", "I2", "I3"],
+    }
+
+    min_support = 2
+    
+    apriori(transactions, min_support)
+
+
+if __name__ == "__main__":
+    main()

bm_preprocessing-0.1.0/src/bm_preprocessing/DM/sources/hash.py

@@ -0,0 +1,145 @@
+from collections import defaultdict
+from itertools import combinations
+
+def print_table(data, title):
+    print(f"\n--- {title} ---")
+    for itemset, count in data.items():
+        print(f"{itemset}: {count}")
+
+C = {}
+L = {}
+
+class Bucket:
+    def __init__(self):
+        self.address: int
+        self.count: int = 0
+        self.itemsets: list[tuple] = []
+
+
+def generate_candidates(prev_frequent_itemsets, k):
+
+    candidates = set()
+
+    for itemset1, itemset2 in combinations(prev_frequent_itemsets, 2):
+        union_set = set(itemset1).union(set(itemset2))
+        if len(union_set) == k:
+            candidates.add(tuple(sorted(union_set)))
+            
+    return sorted(list(candidates))
+
+
+def count_candidates(candidates, transactions):
+    
+    candidate_count = defaultdict(int)
+    
+    for candidate in candidates:
+        for transaction in transactions.values():
+            if all(item in transaction for item in candidate):
+                candidate_count[candidate] += 1
+    
+    return candidate_count
+
+
+def prune_candidates(candidate_count, min_support, prev_freq_itemsets=None):
+
+    filtered_candidates = {}
+    
+    for itemset, count in candidate_count.items():
+
+        if count >= min_support:
+            if prev_freq_itemsets is None or len(itemset) == 1:
+                filtered_candidates[itemset] = count
+            else:
+                subsets = combinations(itemset, len(itemset) - 1)
+                if all(tuple(sorted(subset)) in prev_freq_itemsets for subset in subsets):
+                    filtered_candidates[itemset] = count
+
+    return filtered_candidates
+
+
+def apriori(transactions, min_support):
+
+    items = sorted(set(item for transaction in transactions.values() for item in transaction))
+    c1_list = [(item,) for item in items]
+    
+    C[1] = count_candidates(c1_list, transactions)
+    L[1] = prune_candidates(C[1], min_support)
+    
+    print_table(C[1], "Candidate 1-itemsets (C1)")
+    print_table(L[1], "Frequent 1-itemsets (L1)")
+
+    k = 2
+
+    transactions_combinations = {
+        transaction_id: combinations(items, k)
+        for transaction_id, items in transactions.items()
+    }
+
+    buckets = [Bucket(addr) for addr in range(7)]
+
+    items_list = sorted(set(item for transaction in transactions.values() for item in transaction))
+    ranks = {item: idx + 1 for idx, item in enumerate(items_list)}
+
+    hash_fn = lambda item1, item2: (ranks[item1] * 10 + ranks[item2]) % 7
+
+    for itemset in transactions_combinations.values():
+        for item in itemset:
+            item1, item2 = item[0], item[1]
+            address = hash_fn(item1, item2) % 7
+            buckets[address].count += 1
+            buckets[address].itemsets.append(item)
+
+    print("\n--- Hash Table Buckets ---")
+    for bucket in buckets:
+        print(f"Address: {bucket.address}, Count: {bucket.count}, Itemsets: {bucket.itemsets}")
+    
+    # Filter
+    L2 = {itemset: bucket.count for bucket in buckets for itemset in bucket.itemsets if bucket.count >= min_support}
+    print_table(L2, "Frequent 2-itemsets after Hashing (L2)")
+
+    C["2"] = generate_candidates(L[k-1].keys(), k)
+    L["2"] = L2
+
+    k = 3
+
+    while True:
+
+        candidates = generate_candidates(L[k-1].keys(), k)
+        if not candidates:
+            break
+
+        C[k] = count_candidates(candidates, transactions)
+        L[k] = prune_candidates(C[k], min_support, L[k-1].keys())
+
+        if not L[k]:
+            print_table(C[k], f"Candidate {k}-itemsets (C{k})")
+            print(f"\nNo frequent {k}-itemsets found. Terminating.\n\n")
+            break
+
+        print_table(C[k], f"Candidate {k}-itemsets (C{k})")
+        print_table(L[k], f"Frequent {k}-itemsets (L{k})")
+
+        k += 1
+
+
+def main():
+
+    transactions = {
+        "T100": ["I1", "I2", "I5"],
+        "T200": ["I2", "I4"],
+        "T300": ["I2", "I3"],
+        "T400": ["I1", "I2", "I4"],
+        "T500": ["I1", "I3"],
+        "T600": ["I2", "I3"],
+        "T700": ["I1", "I3"],
+        "T800": ["I1", "I2", "I3", "I5"],
+        "T900": ["I1", "I2", "I3"],
+    }
+
+    min_support = 2
+    
+    apriori(transactions, min_support)
+
+
+if __name__ == "__main__":
+    main()

bm_preprocessing-0.1.0/src/bm_preprocessing/DM/sources/preprocessing.py

@@ -0,0 +1,35 @@
+import pandas as pd
+import numpy as np
+
+# 1. Reading from CSV
+def load_csv(file_path):
+    df = pd.read_csv(file_path)
+    return df
+
+# 2. Reading from Excel
+def load_excel(file_path, sheet_name=0):
+    df = pd.read_excel(file_path, sheet_name=sheet_name)
+    return df
+
+# 3. Mean and Median Fill (For Numerical Columns)
+def impute_numerical(df):
+    # Filling with Mean
+    df_mean = df.copy()
+    # Select only numeric columns for mean/median to avoid errors
+    numeric_cols = df_mean.select_dtypes(include=[np.number]).columns
+    df_mean[numeric_cols] = df_mean[numeric_cols].fillna(df_mean[numeric_cols].mean())
+    
+    # Filling with Median
+    df_median = df.copy()
+    df_median[numeric_cols] = df_median[numeric_cols].fillna(df_median[numeric_cols].median())
+    
+    return df_mean, df_median
+
+# 4. General Fill NA (For Categorical/Transaction Data)
+def fill_general_na(df, value="Unknown"):
+    # Often in transaction data, we fill NaNs with a placeholder or empty string
+    return df.fillna(value)
+
+# Example Usage:
+# df = load_csv('transactions.csv')
+# df_filled = fill_general_na(df, value="Missing_Item")

bm_preprocessing-0.1.0/src/bm_preprocessing/IR/__init__.py

@@ -0,0 +1,5 @@
+"""IR subpackage - Information Retrieval source code."""
+
+from .all import all
+
+__all__ = ["all"]