bm-preprocessing 1.1.0__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/PKG-INFO +1 -1
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/USAGE.md +7 -1
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/pyproject.toml +1 -1
- bm_preprocessing-1.2.0/src/bm_preprocessing/IR/__init__.py +9 -0
- bm_preprocessing-1.2.0/src/bm_preprocessing/IR/eval_metrics.py +26 -0
- bm_preprocessing-1.2.0/src/bm_preprocessing/IR/ndd.py +26 -0
- bm_preprocessing-1.2.0/src/bm_preprocessing/IR/rel.py +26 -0
- bm_preprocessing-1.2.0/src/bm_preprocessing/IR/sources/eval_metrics.py +224 -0
- bm_preprocessing-1.2.0/src/bm_preprocessing/IR/sources/ndd.py +105 -0
- bm_preprocessing-1.2.0/src/bm_preprocessing/IR/sources/rel.py +116 -0
- bm_preprocessing-1.1.0/src/bm_preprocessing/IR/__init__.py +0 -6
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/.gitignore +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/README.md +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/__init__.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/adaboost.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/all.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/all_vis.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/apriori.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/bagging.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/hash.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/hunts.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/hunts_test.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/id3.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/id3_test.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/lib_doc.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/metrics.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/preprocessing.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/adaboost.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/all.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/all_hunts_tree +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/all_vis.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/apriori.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/bagging.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/data.csv +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/hash.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/hunts.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/hunts_test.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/id3.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/id3_test.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/lib_doc.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/metrics.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/preprocessing.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/tennis.csv +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/IR/all.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/IR/all_vis.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/IR/sources/all.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/IR/sources/all_vis.py +0 -0
- {bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/__init__.py +0 -0
|
@@ -14,7 +14,7 @@ Create a file `example.py`:
|
|
|
14
14
|
|
|
15
15
|
```python
|
|
16
16
|
# Import modules
|
|
17
|
-
from bm_preprocessing.IR import all, all_vis
|
|
17
|
+
from bm_preprocessing.IR import all, all_vis, eval_metrics, ndd, rel
|
|
18
18
|
from bm_preprocessing.DM import adaboost, apriori, bagging, hash, hunts, hunts_test, id3, id3_test, lib_doc, metrics, preprocessing
|
|
19
19
|
from bm_preprocessing.DM import all, all_vis
|
|
20
20
|
|
|
@@ -109,6 +109,9 @@ Then in the Python REPL:
|
|
|
109
109
|
```bash
|
|
110
110
|
python -c "from bm_preprocessing.IR import all; print(all)"
|
|
111
111
|
python -c "from bm_preprocessing.IR import all_vis; print(all_vis)"
|
|
112
|
+
python -c "from bm_preprocessing.IR import eval_metrics; print(eval_metrics)"
|
|
113
|
+
python -c "from bm_preprocessing.IR import ndd; print(ndd)"
|
|
114
|
+
python -c "from bm_preprocessing.IR import rel; print(rel)"
|
|
112
115
|
python -c "from bm_preprocessing.DM import all; print(all)"
|
|
113
116
|
python -c "from bm_preprocessing.DM import all_vis; print(all_vis)"
|
|
114
117
|
python -c "from bm_preprocessing.DM import apriori; print(apriori)"
|
|
@@ -132,6 +135,9 @@ python -c "from bm_preprocessing.DM import preprocessing; print(preprocessing)"
|
|
|
132
135
|
|--------|-------------|
|
|
133
136
|
| `from bm_preprocessing.IR import all` | Information Retrieval (MinHash, LSH, Rocchio, Jaccard, VS) |
|
|
134
137
|
| `from bm_preprocessing.IR import all_vis` | IR algorithms with Matplotlib visualizations |
|
|
138
|
+
| `from bm_preprocessing.IR import eval_metrics` | Jaccard, PRF, Compression Ratio, MAP metrics & plots |
|
|
139
|
+
| `from bm_preprocessing.IR import ndd` | Near Duplicate Documents (MinHash & LSH) |
|
|
140
|
+
| `from bm_preprocessing.IR import rel` | Relevance feedback & query expansion (Rocchio & LCA) |
|
|
135
141
|
| `from bm_preprocessing.DM import all` | All DM algorithms (Hunt's, ID3, Bagging, AdaBoost, metrics) |
|
|
136
142
|
| `from bm_preprocessing.DM import all_vis` | All DM algorithms + graphviz & full visualization |
|
|
137
143
|
| `from bm_preprocessing.DM import apriori` | Apriori algorithm |
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Source code loader for IR/eval_metrics.py"""
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
class SourceCodeModule:
|
|
5
|
+
"""A class that displays source code when printed."""
|
|
6
|
+
def __init__(self, name: str, source_path: Path):
|
|
7
|
+
self.name = name
|
|
8
|
+
self._source_path = source_path
|
|
9
|
+
self._source_code = None
|
|
10
|
+
|
|
11
|
+
@property
|
|
12
|
+
def source_code(self) -> str:
|
|
13
|
+
"""Lazily load source code."""
|
|
14
|
+
if self._source_code is None:
|
|
15
|
+
self._source_code = self._source_path.read_text(encoding="utf-8")
|
|
16
|
+
return self._source_code
|
|
17
|
+
|
|
18
|
+
def __repr__(self) -> str:
|
|
19
|
+
return self.source_code
|
|
20
|
+
def __str__(self) -> str:
|
|
21
|
+
return self.source_code
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# Get the path to the source file
|
|
25
|
+
_source_file = Path(__file__).parent / "sources" / "eval_metrics.py"
|
|
26
|
+
eval_metrics = SourceCodeModule("IR.eval_metrics", _source_file)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Source code loader for IR/ndd.py"""
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
class SourceCodeModule:
|
|
5
|
+
"""A class that displays source code when printed."""
|
|
6
|
+
def __init__(self, name: str, source_path: Path):
|
|
7
|
+
self.name = name
|
|
8
|
+
self._source_path = source_path
|
|
9
|
+
self._source_code = None
|
|
10
|
+
|
|
11
|
+
@property
|
|
12
|
+
def source_code(self) -> str:
|
|
13
|
+
"""Lazily load source code."""
|
|
14
|
+
if self._source_code is None:
|
|
15
|
+
self._source_code = self._source_path.read_text(encoding="utf-8")
|
|
16
|
+
return self._source_code
|
|
17
|
+
|
|
18
|
+
def __repr__(self) -> str:
|
|
19
|
+
return self.source_code
|
|
20
|
+
def __str__(self) -> str:
|
|
21
|
+
return self.source_code
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# Get the path to the source file
|
|
25
|
+
_source_file = Path(__file__).parent / "sources" / "ndd.py"
|
|
26
|
+
ndd = SourceCodeModule("IR.ndd", _source_file)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Source code loader for IR/rel.py"""
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
class SourceCodeModule:
|
|
5
|
+
"""A class that displays source code when printed."""
|
|
6
|
+
def __init__(self, name: str, source_path: Path):
|
|
7
|
+
self.name = name
|
|
8
|
+
self._source_path = source_path
|
|
9
|
+
self._source_code = None
|
|
10
|
+
|
|
11
|
+
@property
|
|
12
|
+
def source_code(self) -> str:
|
|
13
|
+
"""Lazily load source code."""
|
|
14
|
+
if self._source_code is None:
|
|
15
|
+
self._source_code = self._source_path.read_text(encoding="utf-8")
|
|
16
|
+
return self._source_code
|
|
17
|
+
|
|
18
|
+
def __repr__(self) -> str:
|
|
19
|
+
return self.source_code
|
|
20
|
+
def __str__(self) -> str:
|
|
21
|
+
return self.source_code
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# Get the path to the source file
|
|
25
|
+
_source_file = Path(__file__).parent / "sources" / "rel.py"
|
|
26
|
+
rel = SourceCodeModule("IR.rel", _source_file)
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import random
|
|
4
|
+
import hashlib
|
|
5
|
+
import nltk
|
|
6
|
+
from itertools import combinations
|
|
7
|
+
from nltk.corpus import stopwords
|
|
8
|
+
from nltk.tokenize import word_tokenize
|
|
9
|
+
from nltk.stem import PorterStemmer
|
|
10
|
+
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
11
|
+
from sklearn.metrics.pairwise import cosine_similarity
|
|
12
|
+
import matplotlib.pyplot as plt
|
|
13
|
+
|
|
14
|
+
nltk.download('punkt_tab', quiet=True)
|
|
15
|
+
nltk.download('stopwords', quiet=True)
|
|
16
|
+
|
|
17
|
+
random.seed(42)
|
|
18
|
+
np.random.seed(42)
|
|
19
|
+
|
|
20
|
+
docs = [
|
|
21
|
+
"information retrieval is the process of obtaining relevant documents",
|
|
22
|
+
"search engines use ranking algorithms for information retrieval",
|
|
23
|
+
"information retrieval systems index and rank documents",
|
|
24
|
+
"retrieval models help search engines find relevant documents",
|
|
25
|
+
"inverted index is widely used in information retrieval",
|
|
26
|
+
"query expansion improves retrieval effectiveness",
|
|
27
|
+
"query expansion adds related terms to the query",
|
|
28
|
+
"expansion techniques improve search results",
|
|
29
|
+
"duplicate documents appear frequently in search engines",
|
|
30
|
+
"near duplicate detection improves indexing"
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
stop_words = set(stopwords.words('english'))
|
|
34
|
+
stemmer = PorterStemmer()
|
|
35
|
+
|
|
36
|
+
def preprocess(text):
|
|
37
|
+
return [stemmer.stem(w) for w in word_tokenize(text.lower()) if w.isalnum() and w not in stop_words]
|
|
38
|
+
|
|
39
|
+
def sim_df(mat, title):
|
|
40
|
+
df = pd.DataFrame(np.round(np.asarray(mat), 3),
|
|
41
|
+
index=[f"Doc{i}" for i in range(len(docs))],
|
|
42
|
+
columns=[f"Doc{i}" for i in range(len(docs))])
|
|
43
|
+
print(f"\n{title}")
|
|
44
|
+
# print(df)
|
|
45
|
+
return df
|
|
46
|
+
|
|
47
|
+
def prf(tp, fp, fn):
|
|
48
|
+
p = tp / (tp + fp) if tp + fp else 0
|
|
49
|
+
r = tp / (tp + fn) if tp + fn else 0
|
|
50
|
+
f = 2 * p * r / (p + r) if p + r else 0
|
|
51
|
+
return round(p, 3), round(r, 3), round(f, 3)
|
|
52
|
+
|
|
53
|
+
processed_docs = [" ".join(preprocess(doc)) for doc in docs]
|
|
54
|
+
shingles = [set(preprocess(doc)) for doc in docs]
|
|
55
|
+
|
|
56
|
+
num_hash, max_shingle = 50, 1000
|
|
57
|
+
hash_funcs = [(random.randint(1, max_shingle), random.randint(0, max_shingle)) for _ in range(num_hash)]
|
|
58
|
+
vocab = list(set(word for doc in shingles for word in doc))
|
|
59
|
+
shingle_index = {w: i for i, w in enumerate(vocab)}
|
|
60
|
+
|
|
61
|
+
def h(x, a, b): return (a * x + b) % max_shingle
|
|
62
|
+
|
|
63
|
+
signature = np.full((num_hash, len(docs)), np.inf)
|
|
64
|
+
for d, doc in enumerate(shingles):
|
|
65
|
+
for word in doc:
|
|
66
|
+
idx = shingle_index[word]
|
|
67
|
+
for i, (a, b) in enumerate(hash_funcs):
|
|
68
|
+
signature[i, d] = min(signature[i, d], h(idx, a, b))
|
|
69
|
+
signature = signature.astype(int)
|
|
70
|
+
|
|
71
|
+
# =====================================================================
|
|
72
|
+
# 1. JACCARD SIMILARITY
|
|
73
|
+
# =====================================================================
|
|
74
|
+
jaccard = lambda a, b: len(a & b) / len(a | b)
|
|
75
|
+
jaccard_matrix = np.matrix([[jaccard(shingles[i], shingles[j]) for j in range(len(docs))] for i in range(len(docs))])
|
|
76
|
+
|
|
77
|
+
sim_df(jaccard_matrix, "JACCARD SIMILARITY TABLE")
|
|
78
|
+
df_jaccard = pd.DataFrame(np.round(np.asarray(jaccard_matrix), 3))
|
|
79
|
+
print("=" * 60)
|
|
80
|
+
print(df_jaccard)
|
|
81
|
+
|
|
82
|
+
# =====================================================================
|
|
83
|
+
# 2. PRECISION, RECALL, FSCORE WITH DIFFERENT BUCKET SIZES (LSH)
|
|
84
|
+
# =====================================================================
|
|
85
|
+
def get_lsh_candidates(sig, bands):
|
|
86
|
+
rows = sig.shape[0] // bands
|
|
87
|
+
buckets, candidates = {}, set()
|
|
88
|
+
for b in range(bands):
|
|
89
|
+
for d in range(sig.shape[1]):
|
|
90
|
+
band = tuple(sig[b * rows:(b + 1) * rows, d])
|
|
91
|
+
key = hashlib.md5(str(band).encode()).hexdigest()
|
|
92
|
+
buckets.setdefault((b, key), []).append(d)
|
|
93
|
+
for group in buckets.values():
|
|
94
|
+
if len(group) > 1:
|
|
95
|
+
for pair in combinations(group, 2):
|
|
96
|
+
candidates.add(tuple(sorted(pair)))
|
|
97
|
+
return candidates
|
|
98
|
+
|
|
99
|
+
threshold = 0.30
|
|
100
|
+
ground_truth = {(i, j) for i in range(len(docs)) for j in range(i + 1, len(docs)) if float(jaccard_matrix[i, j]) >= threshold}
|
|
101
|
+
|
|
102
|
+
bucket_rows = []
|
|
103
|
+
for b in [5, 10, 25]:
|
|
104
|
+
if num_hash % b == 0:
|
|
105
|
+
cand = get_lsh_candidates(signature, b)
|
|
106
|
+
tp = len(cand & ground_truth)
|
|
107
|
+
fp = len(cand - ground_truth)
|
|
108
|
+
fn = len(ground_truth - cand)
|
|
109
|
+
p, r, f = prf(tp, fp, fn)
|
|
110
|
+
bucket_rows.append([b, len(cand), tp, fp, fn, p, r, f])
|
|
111
|
+
|
|
112
|
+
bucket_df = pd.DataFrame(bucket_rows, columns=["Bucket Size", "Candidate Pairs", "TP", "FP", "FN", "Precision", "Recall", "Fscore"])
|
|
113
|
+
print("\nPRECISION, RECALL, FSCORE WITH DIFFERENT BUCKET SIZES")
|
|
114
|
+
print("=" * 60)
|
|
115
|
+
print(bucket_df)
|
|
116
|
+
|
|
117
|
+
# =====================================================================
|
|
118
|
+
# 3. SIGNATURE SIZE / COMPRESSION RATIO AND ACCURACY
|
|
119
|
+
# =====================================================================
|
|
120
|
+
original_size = len(vocab) * len(docs)
|
|
121
|
+
comp_rows = []
|
|
122
|
+
|
|
123
|
+
for rows_used in [10, 20, 30, 40, 50]:
|
|
124
|
+
sub_sig = signature[:rows_used, :]
|
|
125
|
+
correct, total = 0, 0
|
|
126
|
+
for i in range(len(docs)):
|
|
127
|
+
for j in range(i + 1, len(docs)):
|
|
128
|
+
approx = np.mean(sub_sig[:, i] == sub_sig[:, j]) >= threshold
|
|
129
|
+
actual = float(jaccard_matrix[i, j]) >= threshold
|
|
130
|
+
correct += int(approx == actual)
|
|
131
|
+
total += 1
|
|
132
|
+
comp_rows.append([
|
|
133
|
+
rows_used,
|
|
134
|
+
sub_sig.size,
|
|
135
|
+
round(sub_sig.size / original_size, 3),
|
|
136
|
+
round(correct / total, 3)
|
|
137
|
+
])
|
|
138
|
+
|
|
139
|
+
compression_df = pd.DataFrame(comp_rows, columns=["Signature Rows Used", "Signature Size", "Compression Ratio", "Accuracy"])
|
|
140
|
+
print("\nSIGNATURE SIZE, COMPRESSION RATIO & ACCURACY TABLE")
|
|
141
|
+
print("=" * 60)
|
|
142
|
+
print(compression_df)
|
|
143
|
+
|
|
144
|
+
# =====================================================================
|
|
145
|
+
# 4. PERCENT CHANGE IN MEAN AVERAGE PRECISION ON TRAINING QUERIES
|
|
146
|
+
# =====================================================================
|
|
147
|
+
vectorizer = TfidfVectorizer()
|
|
148
|
+
tfidf = vectorizer.fit_transform(processed_docs)
|
|
149
|
+
|
|
150
|
+
training_queries = ["information retrieval", "query expansion", "search engines", "duplicate detection"]
|
|
151
|
+
|
|
152
|
+
# Ground truth relevant documents mapping for training queries
|
|
153
|
+
query_relevance = {
|
|
154
|
+
"information retrieval": {0, 1, 2, 3, 4},
|
|
155
|
+
"query expansion": {5, 6, 7},
|
|
156
|
+
"search engines": {1, 3, 8},
|
|
157
|
+
"duplicate detection": {8, 9}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
# (Alpha, Beta, Gamma) settings for Term Reweighting
|
|
161
|
+
settings = [(1.0, 0.75, 0.15), (1.0, 0.50, 0.25), (1.0, 1.00, 0.50)]
|
|
162
|
+
|
|
163
|
+
def avg_precision(score_vector, relevant_ids):
|
|
164
|
+
ranked = np.argsort(score_vector)[::-1]
|
|
165
|
+
hits, s = 0, 0
|
|
166
|
+
for rank, d in enumerate(ranked, 1):
|
|
167
|
+
if d in relevant_ids:
|
|
168
|
+
hits += 1
|
|
169
|
+
s += hits / rank
|
|
170
|
+
return s / len(relevant_ids) if len(relevant_ids) > 0 else 0
|
|
171
|
+
|
|
172
|
+
map_rows = []
|
|
173
|
+
for a, b, g in settings:
|
|
174
|
+
before_list, after_list = [], []
|
|
175
|
+
for tq in training_queries:
|
|
176
|
+
tq_vec = vectorizer.transform([" ".join(preprocess(tq))])
|
|
177
|
+
base = cosine_similarity(tq_vec, tfidf)[0]
|
|
178
|
+
|
|
179
|
+
# Pseudo-relevance for Rocchio on this query
|
|
180
|
+
top = base.argsort()[::-1][:3]
|
|
181
|
+
rel = tfidf[top]
|
|
182
|
+
nonrel = tfidf[[i for i in range(len(docs)) if i not in top]]
|
|
183
|
+
|
|
184
|
+
rq = a * tq_vec + b * np.asarray(rel.mean(axis=0)) - g * np.asarray(nonrel.mean(axis=0))
|
|
185
|
+
updated = cosine_similarity(np.asarray(rq), tfidf)[0]
|
|
186
|
+
|
|
187
|
+
# Calculate Average Precision
|
|
188
|
+
before_list.append(avg_precision(base, query_relevance[tq]))
|
|
189
|
+
after_list.append(avg_precision(updated, query_relevance[tq]))
|
|
190
|
+
|
|
191
|
+
mb, ma = np.mean(before_list), np.mean(after_list)
|
|
192
|
+
change = ((ma - mb) / mb) * 100 if mb else 0
|
|
193
|
+
map_rows.append([a, b, g, round(mb, 3), round(ma, 3), round(change, 3)])
|
|
194
|
+
|
|
195
|
+
map_df = pd.DataFrame(map_rows, columns=["Alpha", "Beta", "Gamma", "MAP Before", "MAP After", "Percent Change in MAP"])
|
|
196
|
+
print("\nPERCENT CHANGE IN MEAN AVERAGE PRECISION ON TRAINING QUERIES")
|
|
197
|
+
print("=" * 60)
|
|
198
|
+
print(map_df)
|
|
199
|
+
|
|
200
|
+
# =====================================================================
|
|
201
|
+
# 5. VISUALIZATIONS
|
|
202
|
+
# =====================================================================
|
|
203
|
+
plt.figure(figsize=(12, 5))
|
|
204
|
+
plt.subplot(1, 2, 1)
|
|
205
|
+
plt.plot(bucket_df["Bucket Size"], bucket_df["Precision"], marker='o', label="Precision")
|
|
206
|
+
plt.plot(bucket_df["Bucket Size"], bucket_df["Recall"], marker='s', label="Recall")
|
|
207
|
+
plt.plot(bucket_df["Bucket Size"], bucket_df["Fscore"], marker='^', label="Fscore")
|
|
208
|
+
plt.title("PRF vs Bucket Size")
|
|
209
|
+
plt.xlabel("Bucket Size")
|
|
210
|
+
plt.ylabel("Value")
|
|
211
|
+
plt.legend()
|
|
212
|
+
|
|
213
|
+
plt.subplot(1, 2, 2)
|
|
214
|
+
plt.plot(compression_df["Signature Rows Used"], compression_df["Compression Ratio"], marker='o', label="Compression")
|
|
215
|
+
plt.plot(compression_df["Signature Rows Used"], compression_df["Accuracy"], marker='s', label="Accuracy")
|
|
216
|
+
plt.title("Compression Ratio and Accuracy")
|
|
217
|
+
plt.xlabel("Signature Rows Used")
|
|
218
|
+
plt.ylabel("Value")
|
|
219
|
+
plt.legend()
|
|
220
|
+
|
|
221
|
+
plt.tight_layout()
|
|
222
|
+
plt.savefig("eval_metrics_plots.png", dpi=150)
|
|
223
|
+
print("\nMetrics plots saved to 'eval_metrics_plots.png'.")
|
|
224
|
+
plt.show()
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import random
|
|
4
|
+
import hashlib
|
|
5
|
+
import nltk
|
|
6
|
+
from itertools import combinations
|
|
7
|
+
from nltk.corpus import stopwords
|
|
8
|
+
from nltk.tokenize import word_tokenize
|
|
9
|
+
from nltk.stem import PorterStemmer
|
|
10
|
+
|
|
11
|
+
nltk.download('punkt_tab', quiet=True)
|
|
12
|
+
nltk.download('stopwords', quiet=True)
|
|
13
|
+
|
|
14
|
+
random.seed(42)
|
|
15
|
+
np.random.seed(42)
|
|
16
|
+
|
|
17
|
+
# =====================================================================
|
|
18
|
+
# READING FROM CORPUS (Example Code)
|
|
19
|
+
# =====================================================================
|
|
20
|
+
"""
|
|
21
|
+
To read documents from a local corpus directory instead of the hardcoded list below,
|
|
22
|
+
you can use the following snippet:
|
|
23
|
+
|
|
24
|
+
import os
|
|
25
|
+
corpus_dir = "path/to/your/corpus/folder"
|
|
26
|
+
docs = []
|
|
27
|
+
for filename in os.listdir(corpus_dir):
|
|
28
|
+
if filename.endswith(".txt"):
|
|
29
|
+
with open(os.path.join(corpus_dir, filename), "r", encoding="utf-8") as f:
|
|
30
|
+
docs.append(f.read())
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
docs = [
|
|
34
|
+
"information retrieval is the process of obtaining relevant documents",
|
|
35
|
+
"search engines use ranking algorithms for information retrieval",
|
|
36
|
+
"information retrieval systems index and rank documents",
|
|
37
|
+
"retrieval models help search engines find relevant documents",
|
|
38
|
+
"inverted index is widely used in information retrieval",
|
|
39
|
+
"query expansion improves retrieval effectiveness",
|
|
40
|
+
"query expansion adds related terms to the query",
|
|
41
|
+
"expansion techniques improve search results",
|
|
42
|
+
"duplicate documents appear frequently in search engines",
|
|
43
|
+
"near duplicate detection improves indexing"
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
stop_words = set(stopwords.words('english'))
|
|
47
|
+
stemmer = PorterStemmer()
|
|
48
|
+
|
|
49
|
+
def preprocess(text):
|
|
50
|
+
return [stemmer.stem(w) for w in word_tokenize(text.lower()) if w.isalnum() and w not in stop_words]
|
|
51
|
+
|
|
52
|
+
def sim_df(mat, title):
|
|
53
|
+
df = pd.DataFrame(np.round(np.asarray(mat), 3),
|
|
54
|
+
index=[f"Doc{i}" for i in range(len(docs))],
|
|
55
|
+
columns=[f"Doc{i}" for i in range(len(docs))])
|
|
56
|
+
print(f"\n{title}")
|
|
57
|
+
print(df)
|
|
58
|
+
return df
|
|
59
|
+
|
|
60
|
+
processed_docs = [" ".join(preprocess(doc)) for doc in docs]
|
|
61
|
+
shingles = [set(preprocess(doc)) for doc in docs]
|
|
62
|
+
|
|
63
|
+
# =====================================================================
|
|
64
|
+
# 1. MINHASH
|
|
65
|
+
# =====================================================================
|
|
66
|
+
num_hash, max_shingle = 50, 1000
|
|
67
|
+
hash_funcs = [(random.randint(1, max_shingle), random.randint(0, max_shingle)) for _ in range(num_hash)]
|
|
68
|
+
vocab = list(set(word for doc in shingles for word in doc))
|
|
69
|
+
shingle_index = {w: i for i, w in enumerate(vocab)}
|
|
70
|
+
|
|
71
|
+
def h(x, a, b): return (a * x + b) % max_shingle
|
|
72
|
+
|
|
73
|
+
signature = np.full((num_hash, len(docs)), np.inf)
|
|
74
|
+
for d, doc in enumerate(shingles):
|
|
75
|
+
for word in doc:
|
|
76
|
+
idx = shingle_index[word]
|
|
77
|
+
for i, (a, b) in enumerate(hash_funcs):
|
|
78
|
+
signature[i, d] = min(signature[i, d], h(idx, a, b))
|
|
79
|
+
signature = signature.astype(int)
|
|
80
|
+
|
|
81
|
+
minhash_sim = np.matrix([[np.mean(signature[:, i] == signature[:, j]) for j in range(len(docs))] for i in range(len(docs))])
|
|
82
|
+
sim_df(minhash_sim, "MinHash Similarity Table")
|
|
83
|
+
|
|
84
|
+
# =====================================================================
|
|
85
|
+
# 2. LOCALITY SENSITIVE HASHING (LSH)
|
|
86
|
+
# =====================================================================
|
|
87
|
+
def get_lsh_candidates(sig, bands):
|
|
88
|
+
rows = sig.shape[0] // bands
|
|
89
|
+
buckets, candidates = {}, set()
|
|
90
|
+
for b in range(bands):
|
|
91
|
+
for d in range(sig.shape[1]):
|
|
92
|
+
band = tuple(sig[b * rows:(b + 1) * rows, d])
|
|
93
|
+
key = hashlib.md5(str(band).encode()).hexdigest()
|
|
94
|
+
buckets.setdefault((b, key), []).append(d)
|
|
95
|
+
for group in buckets.values():
|
|
96
|
+
if len(group) > 1:
|
|
97
|
+
for pair in combinations(group, 2):
|
|
98
|
+
candidates.add(tuple(sorted(pair)))
|
|
99
|
+
return candidates
|
|
100
|
+
|
|
101
|
+
bands = 10
|
|
102
|
+
candidates = get_lsh_candidates(signature, bands)
|
|
103
|
+
lsh_df = pd.DataFrame([(f"Doc{i}", f"Doc{j}") for i, j in sorted(candidates)], columns=["Document 1", "Document 2"])
|
|
104
|
+
print(f"\nLSH Candidate Pairs Table (Bands={bands})")
|
|
105
|
+
print(lsh_df)
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pandas as pd
|
|
3
|
+
import nltk
|
|
4
|
+
from nltk.corpus import stopwords
|
|
5
|
+
from nltk.tokenize import word_tokenize
|
|
6
|
+
from nltk.stem import PorterStemmer
|
|
7
|
+
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
8
|
+
from sklearn.metrics.pairwise import cosine_similarity
|
|
9
|
+
|
|
10
|
+
nltk.download('punkt_tab', quiet=True)
|
|
11
|
+
nltk.download('stopwords', quiet=True)
|
|
12
|
+
|
|
13
|
+
# =====================================================================
|
|
14
|
+
# READING FROM CORPUS (Example Code)
|
|
15
|
+
# =====================================================================
|
|
16
|
+
"""
|
|
17
|
+
To read documents from a local corpus directory:
|
|
18
|
+
|
|
19
|
+
import os
|
|
20
|
+
corpus_dir = "path/to/your/corpus/folder"
|
|
21
|
+
docs = []
|
|
22
|
+
for filename in os.listdir(corpus_dir):
|
|
23
|
+
if filename.endswith(".txt"):
|
|
24
|
+
with open(os.path.join(corpus_dir, filename), "r", encoding="utf-8") as f:
|
|
25
|
+
docs.append(f.read())
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
docs = [
|
|
29
|
+
"information retrieval is the process of obtaining relevant documents",
|
|
30
|
+
"search engines use ranking algorithms for information retrieval",
|
|
31
|
+
"information retrieval systems index and rank documents",
|
|
32
|
+
"retrieval models help search engines find relevant documents",
|
|
33
|
+
"inverted index is widely used in information retrieval",
|
|
34
|
+
"query expansion improves retrieval effectiveness",
|
|
35
|
+
"query expansion adds related terms to the query",
|
|
36
|
+
"expansion techniques improve search results",
|
|
37
|
+
"duplicate documents appear frequently in search engines",
|
|
38
|
+
"near duplicate detection improves indexing"
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
stop_words = set(stopwords.words('english'))
|
|
42
|
+
stemmer = PorterStemmer()
|
|
43
|
+
|
|
44
|
+
def preprocess(text):
|
|
45
|
+
return [stemmer.stem(w) for w in word_tokenize(text.lower()) if w.isalnum() and w not in stop_words]
|
|
46
|
+
|
|
47
|
+
processed_docs = [" ".join(preprocess(doc)) for doc in docs]
|
|
48
|
+
|
|
49
|
+
vectorizer = TfidfVectorizer()
|
|
50
|
+
tfidf = vectorizer.fit_transform(processed_docs)
|
|
51
|
+
query = "information retrieval"
|
|
52
|
+
processed_query = " ".join(preprocess(query))
|
|
53
|
+
q_vec = vectorizer.transform([processed_query])
|
|
54
|
+
|
|
55
|
+
# =====================================================================
|
|
56
|
+
# 1. ROCCHIO'S FEEDBACK ALGORITHM
|
|
57
|
+
# =====================================================================
|
|
58
|
+
scores = cosine_similarity(q_vec, tfidf)[0]
|
|
59
|
+
|
|
60
|
+
# Pseudo-relevance assumption: top 3 docs are relevant (in a real system, top 10-20)
|
|
61
|
+
num_pseudo_relevant = 3
|
|
62
|
+
top_docs = scores.argsort()[::-1][:num_pseudo_relevant]
|
|
63
|
+
|
|
64
|
+
alpha, beta, gamma = 1.0, 0.75, 0.15
|
|
65
|
+
|
|
66
|
+
relevant = tfidf[top_docs]
|
|
67
|
+
non_relevant = tfidf[[i for i in range(len(docs)) if i not in top_docs]]
|
|
68
|
+
|
|
69
|
+
new_query = alpha * q_vec + beta * np.asarray(relevant.mean(axis=0)) - gamma * np.asarray(non_relevant.mean(axis=0))
|
|
70
|
+
new_scores = cosine_similarity(np.asarray(new_query), tfidf)[0]
|
|
71
|
+
|
|
72
|
+
rocchio_df = pd.DataFrame({
|
|
73
|
+
"Document": [f"Doc{i}" for i in range(len(docs))],
|
|
74
|
+
"Original Score": np.round(scores, 3),
|
|
75
|
+
"Updated Score (Rocchio)": np.round(new_scores, 3)
|
|
76
|
+
})
|
|
77
|
+
print("\nROCCHIO ALGORITHM SCORE TABLE")
|
|
78
|
+
print("=" * 60)
|
|
79
|
+
print(rocchio_df)
|
|
80
|
+
|
|
81
|
+
# =====================================================================
|
|
82
|
+
# 2. LOCAL CONTEXT ANALYSIS (LCA)
|
|
83
|
+
# =====================================================================
|
|
84
|
+
# Measures the co-occurrence of a term with all query terms based on information
|
|
85
|
+
# from pseudo-relevant documents (top 10-20 documents returned by initial search).
|
|
86
|
+
# Since our corpus is small, we'll use top 5 pseudo-relevant documents.
|
|
87
|
+
|
|
88
|
+
num_lca_pseudo_relevant = 5
|
|
89
|
+
top_k_lca = scores.argsort()[::-1][:num_lca_pseudo_relevant]
|
|
90
|
+
top_docs_lca = [processed_docs[i] for i in top_k_lca]
|
|
91
|
+
|
|
92
|
+
term_freq = {}
|
|
93
|
+
for doc in top_docs_lca:
|
|
94
|
+
for word in doc.split():
|
|
95
|
+
term_freq[word] = term_freq.get(word, 0) + 1
|
|
96
|
+
|
|
97
|
+
# Extract top expanded terms from pseudo-relevant docs
|
|
98
|
+
num_expansion_terms = 5
|
|
99
|
+
expanded_terms = sorted(term_freq, key=term_freq.get, reverse=True)[:num_expansion_terms]
|
|
100
|
+
expanded_query = processed_query + " " + " ".join(expanded_terms)
|
|
101
|
+
|
|
102
|
+
expanded_vec = vectorizer.transform([expanded_query])
|
|
103
|
+
expanded_scores = cosine_similarity(expanded_vec, tfidf)[0]
|
|
104
|
+
|
|
105
|
+
print("\nLOCAL CONTEXT ANALYSIS (LCA)")
|
|
106
|
+
print("=" * 60)
|
|
107
|
+
print(f"Original Query: {processed_query}")
|
|
108
|
+
print(f"Expanded Query: {expanded_query}")
|
|
109
|
+
|
|
110
|
+
lca_df = pd.DataFrame({
|
|
111
|
+
"Document": [f"Doc{i}" for i in range(len(docs))],
|
|
112
|
+
"Original Score": np.round(scores, 3),
|
|
113
|
+
"LCA Expanded Score": np.round(expanded_scores, 3)
|
|
114
|
+
})
|
|
115
|
+
print("\nLCA SCORE TABLE")
|
|
116
|
+
print(lca_df)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/adaboost.py
RENAMED
|
File without changes
|
|
File without changes
|
{bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/all_hunts_tree
RENAMED
|
File without changes
|
{bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/all_vis.py
RENAMED
|
File without changes
|
{bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/apriori.py
RENAMED
|
File without changes
|
{bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/bagging.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/hunts_test.py
RENAMED
|
File without changes
|
|
File without changes
|
{bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/id3_test.py
RENAMED
|
File without changes
|
{bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/lib_doc.py
RENAMED
|
File without changes
|
{bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/metrics.py
RENAMED
|
File without changes
|
{bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/preprocessing.py
RENAMED
|
File without changes
|
{bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/DM/sources/tennis.csv
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{bm_preprocessing-1.1.0 → bm_preprocessing-1.2.0}/src/bm_preprocessing/IR/sources/all_vis.py
RENAMED
|
File without changes
|
|
File without changes
|