itertoolkit 1.5.0__py3-none-any.whl

bm_preprocessing/src/DM/finals.py

@@ -0,0 +1,19 @@
+"""Data-mining quick reference.
+
+Printing bm_preprocessing.DM.dm_doc displays this source file.
+"""
+
+from pandas import DataFrame, display
+
+df = DataFrame()  # Placeholder for type hinting
+
+df = {
+    "col1": [1, 2, 3],
+    "col2": ["a", "b", "c"],
+}
+
+df["col1"].mean()
+
+print(df["col2"].unique())
+
+display(df.head())

bm_preprocessing/src/DM/gsp.py

@@ -0,0 +1,378 @@
+# -*- coding: utf-8 -*-
+"""PS15.ipynb
+
+Automatically generated by Colab.
+
+Original file is located at
+    https://colab.research.google.com/drive/1L11gBakFRdb1QWPyk4d0e8MxgnAN8WVE
+
+## Own Impl
+"""
+
+import re
+
+# 1. PARSE INPUT FILES
+
+
+def parse_data(filepath):
+    sequences = []
+    with open(filepath) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            itemsets = re.findall(r"\{([^}]+)\}", line)
+            seq = []
+            for itemset in itemsets:
+                items = frozenset(int(x.strip()) for x in itemset.split(","))
+                seq.append(items)
+            if seq:
+                sequences.append(seq)
+    return sequences
+
+
+def parse_params(filepath):
+    mis = {}
+    sdc = None
+    with open(filepath) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            m = re.match(r"MIS\((\d+)\)\s*=\s*([\d.]+)", line)
+            if m:
+                mis[int(m.group(1))] = float(m.group(2))
+                continue
+            m = re.match(r"SDC\s*=\s*([\d.]+)", line)
+            if m:
+                sdc = float(m.group(1))
+    return mis, sdc
+
+
+# 2. SUPPORT COUNTING
+
+
+def is_subsequence(pattern, sequence):
+    """Check if pattern (list of frozensets) occurs in sequence (in order)."""
+    pat_idx = 0
+    for itemset in sequence:
+        if pattern[pat_idx].issubset(itemset):
+            pat_idx += 1
+            if pat_idx == len(pattern):
+                return True
+    return False
+
+
+def count_support(pattern, sequences):
+    return sum(1 for seq in sequences if is_subsequence(pattern, seq))
+
+
+# 3. MS-GSP CHECKS
+
+
+def min_mis(pattern, mis_map):
+    """Minimum MIS among all items in the pattern."""
+    all_items = [item for itemset in pattern for item in itemset]
+    return min(mis_map.get(item, 1.0) for item in all_items)
+
+
+def check_sdc(pattern, mis_map, sdc):
+    """max(MIS of items) - min(MIS of items) <= SDC."""
+    all_items = [item for itemset in pattern for item in itemset]
+    mis_vals = [mis_map.get(item, 1.0) for item in all_items]
+    return (max(mis_vals) - min(mis_vals)) <= sdc
+
+
+def is_frequent(pattern, sequences, mis_map, sdc, n):
+    if not check_sdc(pattern, mis_map, sdc):
+        return False, 0
+    cnt = count_support(pattern, sequences)
+    sup = cnt / n
+    return sup >= min_mis(pattern, mis_map), cnt
+
+
+# 4. CANDIDATE GENERATION
+
+
+def get_all_items(sequences):
+    items = set()
+    for seq in sequences:
+        for itemset in seq:
+            items |= itemset
+    return sorted(items)
+
+
+def generate_size1_candidates(sequences, mis_map, sdc, n):
+    items = sorted(get_all_items(sequences), key=lambda x: mis_map.get(x, 1.0))
+    frequent = []
+    for item in items:
+        pattern = [frozenset([item])]
+        is_freq, cnt = is_frequent(pattern, sequences, mis_map, sdc, n)
+        if is_freq:
+            frequent.append((pattern, cnt))
+    return frequent
+
+
+def join_sequences(s1, s2):
+    """Join two k-sequences to produce (k+1)-sequence candidates."""
+    candidates = []
+    s1_flat = [item for itemset in s1 for item in sorted(itemset)]
+    s2_flat = [item for itemset in s2 for item in sorted(itemset)]
+
+    # Core MS-GSP join condition
+    if s1_flat[1:] != s2_flat[:-1]:
+        return candidates
+
+    last_item = s2_flat[-1]
+
+    # Candidate 1: last item becomes a new itemset
+    candidates.append(s1 + [frozenset([last_item])])
+
+    # Candidate 2: last item merges into last itemset of s1
+    if last_item > max(s1[-1]):
+        new_last = frozenset(s1[-1] | {last_item})
+        candidates.append(s1[:-1] + [new_last])
+
+    return candidates
+
+
+def generate_candidates(freq_prev, mis_map, sdc):
+    candidates = []
+    seen = set()
+    patterns = [p for p, _ in freq_prev]
+
+    for i in range(len(patterns)):
+        for j in range(len(patterns)):
+            for cand in join_sequences(patterns[i], patterns[j]):
+                key = tuple(tuple(sorted(itemset)) for itemset in cand)
+                if key not in seen:
+                    seen.add(key)
+                    if check_sdc(cand, mis_map, sdc):
+                        candidates.append(cand)
+    return candidates
+
+
+# 5. MAIN MS-GSP ALGORITHM
+
+
+def msgsp(data_file, param_file):
+    sequences = parse_data(data_file)
+    mis_map, sdc = parse_params(param_file)
+    n = len(sequences)
+
+    print(f"Loaded {n} sequences | SDC = {sdc}")
+    print(f"MIS values: {mis_map}\n")
+
+    all_frequent = []
+
+    freq1 = generate_size1_candidates(sequences, mis_map, sdc, n)
+    print(f"Frequent 1-sequences: {len(freq1)}")
+    all_frequent.extend(freq1)
+
+    freq_prev = freq1
+    k = 2
+
+    while freq_prev:
+        candidates = generate_candidates(freq_prev, mis_map, sdc)
+        freq_k = []
+        for cand in candidates:
+            is_freq, cnt = is_frequent(cand, sequences, mis_map, sdc, n)
+            if is_freq:
+                freq_k.append((cand, cnt))
+        print(f"Frequent {k}-sequences: {len(freq_k)}")
+        all_frequent.extend(freq_k)
+        freq_prev = freq_k
+        k += 1
+
+    return all_frequent
+
+
+# 6. OUTPUT
+
+
+def format_pattern(pattern):
+    parts = [
+        "{" + ",".join(str(i) for i in sorted(itemset)) + "}" for itemset in pattern
+    ]
+    return "<" + "".join(parts) + ">"
+
+
+def print_results(frequent):
+    print("\n=== Frequent Sequential Patterns ===")
+    if not frequent:
+        print("No frequent patterns found.")
+        return
+    for pattern, count in frequent:
+        print(f"Pattern :{format_pattern(pattern)} count: {count}")
+
+
+# 7. RUN
+
+if __name__ == "__main__":
+    data_txt = (
+        "/content/drive/MyDrive/Academic_Resources/Semester 08/20XW87/PS15/data.txt"
+    )
+    para_txt = (
+        "/content/drive/MyDrive/Academic_Resources/Semester 08/20XW87/PS15/para.txt"
+    )
+    frequent = msgsp(data_txt, para_txt)
+    print_results(frequent)
+
+"""## Lib Impl"""
+
+"""
+MS-GSP using gsppy
+Strategy:
+  1. Run gsppy.GSP with the lowest possible MIS as threshold (catches all candidates)
+  2. Post-filter results using per-item MIS and SDC constraints
+"""
+
+# ── In Colab, run first: !pip install gsppy ──────────────────────────────────
+import re
+
+from gsppy.gsp import GSP
+
+# 1. PARSE INPUT FILES
+
+
+def parse_data(filepath):
+    """Parse data.txt → list of flat item lists for gsppy.
+    Each sequence becomes a flat list of items (gsppy treats each element as a timestep).
+    To preserve itemset structure, we also return the structured form separately.
+    """
+    sequences_flat = []  # for gsppy: [[10, 40, 50, 40, 90], ...]
+    sequences_struct = []  # for MIS/SDC: [[{10,40,50}, {40,90}], ...]
+
+    with open(filepath) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            itemsets = re.findall(r"\{([^}]+)\}", line)
+            struct = []
+            flat = []
+            for itemset in itemsets:
+                items = frozenset(int(x.strip()) for x in itemset.split(","))
+                struct.append(items)
+                flat.extend(sorted(items))  # gsppy needs flat list
+            if struct:
+                sequences_flat.append(flat)
+                sequences_struct.append(struct)
+
+    return sequences_flat, sequences_struct
+
+
+def parse_params(filepath):
+    mis = {}
+    sdc = None
+    with open(filepath) as f:
+        for line in f:
+            line = line.strip()
+            m = re.match(r"MIS\((\d+)\)\s*=\s*([\d.]+)", line)
+            if m:
+                mis[int(m.group(1))] = float(m.group(2))
+                continue
+            m = re.match(r"SDC\s*=\s*([\d.]+)", line)
+            if m:
+                sdc = float(m.group(1))
+    return mis, sdc
+
+
+# 2. MS-GSP FILTERS (applied to gsppy output)
+
+
+def min_mis(pattern_items, mis_map):
+    return min(mis_map.get(item, 1.0) for item in pattern_items)
+
+
+def check_sdc(pattern_items, mis_map, sdc):
+    vals = [mis_map.get(item, 1.0) for item in pattern_items]
+    return (max(vals) - min(vals)) <= sdc
+
+
+def count_support_struct(pattern_flat, sequences_struct):
+    """Count how many structured sequences contain the flat pattern as an ordered subsequence."""
+    count = 0
+    for seq in sequences_struct:
+        flat_seq = [item for itemset in seq for item in sorted(itemset)]
+        # Check if pattern_flat is a subsequence of flat_seq
+        pi = 0
+        for item in flat_seq:
+            if pi < len(pattern_flat) and item == pattern_flat[pi]:
+                pi += 1
+        if pi == len(pattern_flat):
+            count += 1
+    return count
+
+
+# 3. RUN GSPPY + APPLY MS-GSP FILTERING
+
+
+def run_msgsp(data_file, param_file):
+    sequences_flat, sequences_struct = parse_data(data_file)
+    mis_map, sdc = parse_params(param_file)
+    n = len(sequences_flat)
+
+    print(f"Loaded {n} sequences | SDC = {sdc}")
+    print(f"MIS values: {mis_map}\n")
+
+    # Use the global minimum MIS as threshold for gsppy
+    # This lets gsppy find all candidates; we filter with per-item MIS after
+    global_min_mis = min(mis_map.values())
+    print(f"Running gsppy with global min_support = {global_min_mis} (lowest MIS)")
+    print("─" * 50)
+
+    # gsppy returns: list of dicts, one dict per pattern length
+    # e.g. [{(10,): 4, (40,): 3}, {(10,40): 2}, ...]
+    gsp_results = GSP(sequences_flat).search(global_min_mis)
+
+    # ── Apply MS-GSP filters ──────────────────────────────
+    ms_gsp_patterns = []
+
+    for level_dict in gsp_results:
+        for pattern_tuple, gsppy_count in level_dict.items():
+            pattern_items = list(pattern_tuple)
+
+            # Filter 1: SDC check
+            if not check_sdc(pattern_items, mis_map, sdc):
+                continue
+
+            # Filter 2: support must meet min MIS of the pattern
+            threshold = min_mis(pattern_items, mis_map)
+            sup = gsppy_count / n
+            if sup < threshold:
+                continue
+
+            ms_gsp_patterns.append((pattern_tuple, gsppy_count))
+
+    return ms_gsp_patterns
+
+
+# 4. OUTPUT
+
+
+def format_pattern(pattern_tuple):
+    # Each item becomes its own itemset for display: (10, 40) → <{10}{40}>
+    return "<" + "".join("{" + str(item) + "}" for item in pattern_tuple) + ">"
+
+
+def print_results(ms_gsp_patterns):
+    print("\n=== MS-GSP Frequent Sequential Patterns ===")
+    if not ms_gsp_patterns:
+        print("No frequent patterns found.")
+        return
+    for pattern, count in ms_gsp_patterns:
+        print(f"Pattern :{format_pattern(pattern)} count: {count}")
+
+
+# 5. RUN
+
+if __name__ == "__main__":
+    data_txt = (
+        "/content/drive/MyDrive/Academic_Resources/Semester 08/20XW87/PS15/data.txt"
+    )
+    para_txt = (
+        "/content/drive/MyDrive/Academic_Resources/Semester 08/20XW87/PS15/para.txt"
+    )
+    patterns = run_msgsp(data_txt, para_txt)
+    print_results(patterns)

bm_preprocessing/src/DM/test.py

@@ -0,0 +1,19 @@
+"""Data-mining Test quick reference.
+
+Printing bm_preprocessing.DM.dm_doc displays this source file.
+"""
+
+from pandas import DataFrame, display
+
+df = DataFrame()  # Placeholder for type hinting
+
+df = {
+    "col1": [1, 2, 3],
+    "col2": ["a", "b", "c"],
+}
+
+df["col1"].mean()
+
+print(df["col2"].unique())
+
+display(df.head())

bm_preprocessing/src/Finals/__init__.py

@@ -0,0 +1 @@
+"""Finals source snippets."""