scimapper 1.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- core/__init__.py +11 -0
- core/clusterer.py +178 -0
- core/collector.py +153 -0
- core/config_template.json +48 -0
- core/doc_builder.py +316 -0
- core/embedder.py +70 -0
- core/selector.py +129 -0
- core/utils.py +79 -0
- core/visualizer.py +271 -0
- scimapper-1.0.1.dist-info/METADATA +252 -0
- scimapper-1.0.1.dist-info/RECORD +15 -0
- scimapper-1.0.1.dist-info/WHEEL +5 -0
- scimapper-1.0.1.dist-info/entry_points.txt +2 -0
- scimapper-1.0.1.dist-info/licenses/LICENSE +21 -0
- scimapper-1.0.1.dist-info/top_level.txt +1 -0
core/visualizer.py
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
"""
|
|
2
|
+
AutoSMS Framework - Módulo de Visualización Avanzada (Visualizer)
|
|
3
|
+
===============================================================
|
|
4
|
+
Construye visualizaciones interactivas de última generación en Plotly:
|
|
5
|
+
1. Vista Dual: Mapa Conceptual 2D de Dispersión + Capa de Contorno de Densidad (KDE).
|
|
6
|
+
2. Panel Lateral de Evolución Temporal (2018–2026) con cálculo de tasas de crecimiento.
|
|
7
|
+
3. Selector interactivo de modos de visualización (Híbrido, Puntos, Densidad Topográfica).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import pandas as pd
|
|
12
|
+
import numpy as np
|
|
13
|
+
import plotly.graph_objects as go
|
|
14
|
+
from plotly.subplots import make_subplots
|
|
15
|
+
|
|
16
|
+
class AutoVisualizer:
|
|
17
|
+
def __init__(self, config: dict):
|
|
18
|
+
self.config = config
|
|
19
|
+
self.output_dir = config.get("project_metadata", {}).get("output_dir", "./output")
|
|
20
|
+
self.title = config.get("project_metadata", {}).get("title", "Mapeo Sistemático de Literatura")
|
|
21
|
+
|
|
22
|
+
def build_interactive_map(self, df: pd.DataFrame, topics_info: dict) -> str:
|
|
23
|
+
print("\n🌐 Construyendo Dashboard Dual Interactivo (Mapa 2D KDE + Trayectorias Temporales)...")
|
|
24
|
+
|
|
25
|
+
colores = [
|
|
26
|
+
"#1f77b4", "#ff7f0e", "#2ca02c", "#d62728",
|
|
27
|
+
"#9467bd", "#8c564b", "#e377c2", "#7f7f7f", "#bcbd22", "#17becf"
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
# Crear figura con 2 subplots en paralelo: Mapa 2D (58%) y Líneas Temporales (42%)
|
|
31
|
+
fig = make_subplots(
|
|
32
|
+
rows=1, cols=2,
|
|
33
|
+
column_widths=[0.58, 0.42],
|
|
34
|
+
subplot_titles=[
|
|
35
|
+
"<b>Mapa Semántico 2D y Densidad de Investigación (t-SNE + KDE)</b>",
|
|
36
|
+
"<b>Dinámica y Trayectoria Temporal Anual</b>"
|
|
37
|
+
],
|
|
38
|
+
horizontal_spacing=0.08
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
# 1. Capa Base: Mapa de Contorno / Densidad (KDE 2D)
|
|
42
|
+
kde_trace = go.Histogram2dContour(
|
|
43
|
+
x=df["coord_x"],
|
|
44
|
+
y=df["coord_y"],
|
|
45
|
+
name="Densidad Topográfica (KDE)",
|
|
46
|
+
colorscale="Blues",
|
|
47
|
+
reversescale=False,
|
|
48
|
+
showscale=False,
|
|
49
|
+
ncontours=22,
|
|
50
|
+
opacity=0.30,
|
|
51
|
+
hoverinfo="skip"
|
|
52
|
+
)
|
|
53
|
+
fig.add_trace(kde_trace, row=1, col=1)
|
|
54
|
+
|
|
55
|
+
# 2. Capas de Clusters (Envolventes Convexas, Puntos y Centroides)
|
|
56
|
+
from scipy.spatial import ConvexHull
|
|
57
|
+
|
|
58
|
+
def hex_to_rgb(hex_str):
|
|
59
|
+
hex_str = hex_str.lstrip('#')
|
|
60
|
+
return tuple(int(hex_str[i:i+2], 16) for i in (0, 2, 4))
|
|
61
|
+
|
|
62
|
+
unique_topics = sorted(df["topic_id"].unique())
|
|
63
|
+
total_docs = len(df)
|
|
64
|
+
hull_trace_indices = []
|
|
65
|
+
scatter_trace_indices = []
|
|
66
|
+
centroid_trace_indices = []
|
|
67
|
+
|
|
68
|
+
for idx, tid in enumerate(unique_topics):
|
|
69
|
+
sub = df[df["topic_id"] == tid]
|
|
70
|
+
color = colores[idx % len(colores)]
|
|
71
|
+
r, g, b = hex_to_rgb(color)
|
|
72
|
+
t_info = topics_info.get(str(tid), topics_info.get(tid, {}))
|
|
73
|
+
t_name = t_info.get("assigned_name", f"Tópico {tid}")
|
|
74
|
+
|
|
75
|
+
# Dinámica temporal para tooltip
|
|
76
|
+
t_dyn = t_info.get("temporal_dynamics", {})
|
|
77
|
+
badge = t_dyn.get("badge", "⚖️ Maduro")
|
|
78
|
+
cagr = t_dyn.get("cagr_percent", 0.0)
|
|
79
|
+
kws = ", ".join(t_info.get("keywords", [])[:5])
|
|
80
|
+
|
|
81
|
+
pts = sub[["coord_x", "coord_y"]].values
|
|
82
|
+
center = np.mean(pts, axis=0)
|
|
83
|
+
cx, cy = float(center[0]), float(center[1])
|
|
84
|
+
|
|
85
|
+
# A. Envolvente Convexa del Cluster
|
|
86
|
+
dists = np.linalg.norm(pts - center, axis=1)
|
|
87
|
+
mask_core = dists <= np.percentile(dists, 92)
|
|
88
|
+
pts_core = pts[mask_core]
|
|
89
|
+
|
|
90
|
+
if len(pts_core) >= 3:
|
|
91
|
+
hull = ConvexHull(pts_core)
|
|
92
|
+
hull_pts = pts_core[hull.vertices]
|
|
93
|
+
hull_pts = np.vstack([hull_pts, hull_pts[0]])
|
|
94
|
+
hx = hull_pts[:, 0]
|
|
95
|
+
hy = hull_pts[:, 1]
|
|
96
|
+
else:
|
|
97
|
+
hx = [cx]
|
|
98
|
+
hy = [cy]
|
|
99
|
+
|
|
100
|
+
hull_trace = go.Scatter(
|
|
101
|
+
x=hx,
|
|
102
|
+
y=hy,
|
|
103
|
+
mode="lines",
|
|
104
|
+
fill="toself",
|
|
105
|
+
fillcolor=f"rgba({r}, {g}, {b}, 0.16)",
|
|
106
|
+
line=dict(color=f"rgba({r}, {g}, {b}, 0.85)", width=2, dash="dash", shape="spline", smoothing=1.3),
|
|
107
|
+
name=f"Área {badge} {t_name}",
|
|
108
|
+
legendgroup=f"topic_{tid}",
|
|
109
|
+
showlegend=False,
|
|
110
|
+
hoverinfo="skip"
|
|
111
|
+
)
|
|
112
|
+
fig.add_trace(hull_trace, row=1, col=1)
|
|
113
|
+
hull_trace_indices.append(len(fig.data) - 1)
|
|
114
|
+
|
|
115
|
+
# B. Puntos individuales
|
|
116
|
+
hover_texts = []
|
|
117
|
+
for _, r_doc in sub.iterrows():
|
|
118
|
+
snip = (r_doc['abstract'][:240] + "...") if len(r_doc['abstract']) > 240 else r_doc['abstract']
|
|
119
|
+
txt = (
|
|
120
|
+
f"<b>{r_doc['title']}</b><br>"
|
|
121
|
+
f"<i>Autores:</i> {r_doc['authors']}<br>"
|
|
122
|
+
f"<i>Año:</i> {r_doc['publication_year']} | <i>Venue:</i> {r_doc['venue']}<br>"
|
|
123
|
+
f"<i>Citas:</i> {r_doc.get('cited_by_count', 0)} | <b>Estado:</b> {badge} (CAGR: {cagr}%)<br>"
|
|
124
|
+
f"<hr style='margin:4px 0;'>"
|
|
125
|
+
f"<b>Abstract:</b> {snip}"
|
|
126
|
+
)
|
|
127
|
+
hover_texts.append(txt)
|
|
128
|
+
|
|
129
|
+
scatter_trace = go.Scatter(
|
|
130
|
+
x=sub["coord_x"],
|
|
131
|
+
y=sub["coord_y"],
|
|
132
|
+
mode="markers",
|
|
133
|
+
name=f"{badge} {t_name} ({len(sub)})",
|
|
134
|
+
text=hover_texts,
|
|
135
|
+
hoverinfo="text",
|
|
136
|
+
legendgroup=f"topic_{tid}",
|
|
137
|
+
marker=dict(size=8, color=color, opacity=0.88, line=dict(width=1, color="#222222"))
|
|
138
|
+
)
|
|
139
|
+
fig.add_trace(scatter_trace, row=1, col=1)
|
|
140
|
+
scatter_trace_indices.append(len(fig.data) - 1)
|
|
141
|
+
|
|
142
|
+
# C. Centroide y Etiqueta de Cluster
|
|
143
|
+
centroid_hover = (
|
|
144
|
+
f"<b>🏛️ CLUSTER {tid}: {t_name}</b><br>"
|
|
145
|
+
f"<b>Estado:</b> {badge} (CAGR: {cagr}%)<br>"
|
|
146
|
+
f"<b>Volumen:</b> {len(sub)} artículos ({round(len(sub)/total_docs*100, 1)}%)<br>"
|
|
147
|
+
f"<b>Palabras Clave (c-TF-IDF):</b> {kws}<br>"
|
|
148
|
+
f"<i>Centroide: ({cx:.2f}, {cy:.2f})</i>"
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
centroid_trace = go.Scatter(
|
|
152
|
+
x=[cx],
|
|
153
|
+
y=[cy],
|
|
154
|
+
mode="markers+text",
|
|
155
|
+
text=[f"<b>C{tid}</b>"],
|
|
156
|
+
textposition="top center",
|
|
157
|
+
textfont=dict(size=12, color="#0f172a", family="Arial Black, Segoe UI, sans-serif"),
|
|
158
|
+
hovertext=[centroid_hover],
|
|
159
|
+
hoverinfo="text",
|
|
160
|
+
legendgroup=f"topic_{tid}",
|
|
161
|
+
showlegend=False,
|
|
162
|
+
marker=dict(size=16, symbol="diamond", color=color, line=dict(width=2, color="#ffffff"))
|
|
163
|
+
)
|
|
164
|
+
fig.add_trace(centroid_trace, row=1, col=1)
|
|
165
|
+
centroid_trace_indices.append(len(fig.data) - 1)
|
|
166
|
+
|
|
167
|
+
# 3. Subplot 2: Evolución Temporal Anual (Líneas de tendencia)
|
|
168
|
+
years_series = pd.to_numeric(df["publication_year"], errors="coerce").dropna().astype(int)
|
|
169
|
+
min_year = int(years_series.min()) if not years_series.empty else 2018
|
|
170
|
+
max_year = int(years_series.max()) if not years_series.empty else 2026
|
|
171
|
+
all_years = list(range(min_year, max_year + 1))
|
|
172
|
+
|
|
173
|
+
for idx, tid in enumerate(unique_topics):
|
|
174
|
+
color = colores[idx % len(colores)]
|
|
175
|
+
t_info = topics_info.get(str(tid), topics_info.get(tid, {}))
|
|
176
|
+
t_name = t_info.get("assigned_name", f"Tópico {tid}")
|
|
177
|
+
t_dyn = t_info.get("temporal_dynamics", {})
|
|
178
|
+
y_dist = t_dyn.get("yearly_distribution", {})
|
|
179
|
+
|
|
180
|
+
y_values = [y_dist.get(y, y_dist.get(str(y), 0)) for y in all_years]
|
|
181
|
+
|
|
182
|
+
line_trace = go.Scatter(
|
|
183
|
+
x=all_years,
|
|
184
|
+
y=y_values,
|
|
185
|
+
mode="lines+markers",
|
|
186
|
+
name=f"{t_name}",
|
|
187
|
+
legendgroup=f"topic_{tid}",
|
|
188
|
+
showlegend=False,
|
|
189
|
+
line=dict(color=color, width=2.8),
|
|
190
|
+
marker=dict(size=7, color=color),
|
|
191
|
+
hovertemplate=f"<b>{t_name}</b><br>Año %{{x}}: %{{y}} artículos<extra></extra>"
|
|
192
|
+
)
|
|
193
|
+
fig.add_trace(line_trace, row=1, col=2)
|
|
194
|
+
|
|
195
|
+
# 4. Botones de Control para Alternar Modos de Visualización (Updatemenus)
|
|
196
|
+
total_traces = len(fig.data)
|
|
197
|
+
|
|
198
|
+
vis_all = [True] * total_traces
|
|
199
|
+
vis_regions = [True] * total_traces
|
|
200
|
+
for p_idx in scatter_trace_indices:
|
|
201
|
+
vis_regions[p_idx] = False
|
|
202
|
+
|
|
203
|
+
vis_points = [False] * total_traces
|
|
204
|
+
for p_idx in scatter_trace_indices + centroid_trace_indices:
|
|
205
|
+
vis_points[p_idx] = True
|
|
206
|
+
for l_idx in range(len(fig.data) - len(unique_topics), len(fig.data)):
|
|
207
|
+
vis_points[l_idx] = True
|
|
208
|
+
|
|
209
|
+
fig.update_layout(
|
|
210
|
+
autosize=True,
|
|
211
|
+
height=860,
|
|
212
|
+
template="plotly_white",
|
|
213
|
+
updatemenus=[
|
|
214
|
+
dict(
|
|
215
|
+
type="buttons",
|
|
216
|
+
direction="right",
|
|
217
|
+
x=1.0,
|
|
218
|
+
xanchor="right",
|
|
219
|
+
y=1.14,
|
|
220
|
+
showactive=True,
|
|
221
|
+
bgcolor="#F8FAFC",
|
|
222
|
+
bordercolor="#CBD5E1",
|
|
223
|
+
font=dict(size=11, color="#1E293B"),
|
|
224
|
+
buttons=[
|
|
225
|
+
dict(label="🌟 Vista Completa (Regiones + Puntos + Densidad)", method="update", args=[{"visible": vis_all}]),
|
|
226
|
+
dict(label="🛡️ Solo Regiones y Fronteras de Clusters", method="update", args=[{"visible": vis_regions}]),
|
|
227
|
+
dict(label="📍 Solo Puntos de Artículos", method="update", args=[{"visible": vis_points}])
|
|
228
|
+
]
|
|
229
|
+
)
|
|
230
|
+
],
|
|
231
|
+
title={
|
|
232
|
+
'text': f"<b>{self.title}</b><br><span style='font-size:13px; color:#64748B; font-weight:normal;'>Mapeo Semántico 2D (t-SNE) + Densidad Topográfica (KDE) + Radar de Dinámica Temporal | {len(df)} Artículos Indexados</span>",
|
|
233
|
+
'x': 0.01,
|
|
234
|
+
'xanchor': 'left',
|
|
235
|
+
'y': 0.98,
|
|
236
|
+
'font': dict(size=17, color="#0F172A", family="Segoe UI, Inter, Arial, sans-serif")
|
|
237
|
+
},
|
|
238
|
+
xaxis=dict(title="Dimensión Semántica 1 (t-SNE)", showgrid=True, gridcolor="#F1F5F9", zeroline=False),
|
|
239
|
+
yaxis=dict(title="Dimensión Semántica 2 (t-SNE)", showgrid=True, gridcolor="#F1F5F9", zeroline=False),
|
|
240
|
+
xaxis2=dict(
|
|
241
|
+
title="Año de Publicación",
|
|
242
|
+
dtick=1,
|
|
243
|
+
range=[min_year - 0.5, max_year + 0.5],
|
|
244
|
+
showgrid=True,
|
|
245
|
+
gridcolor="#F1F5F9"
|
|
246
|
+
),
|
|
247
|
+
yaxis2=dict(title="Volumen de Publicaciones (N)", showgrid=True, gridcolor="#F1F5F9"),
|
|
248
|
+
legend=dict(
|
|
249
|
+
title=dict(text="<b>Taxonomías y Estado de Madurez</b>", font=dict(size=12, color="#0F172A")),
|
|
250
|
+
orientation="h",
|
|
251
|
+
y=-0.16,
|
|
252
|
+
x=0.5,
|
|
253
|
+
xanchor="center",
|
|
254
|
+
bgcolor="rgba(255,255,255,0.95)",
|
|
255
|
+
bordercolor="#CBD5E1",
|
|
256
|
+
borderwidth=1,
|
|
257
|
+
font=dict(size=11, color="#334155")
|
|
258
|
+
),
|
|
259
|
+
margin=dict(l=60, r=50, t=130, b=120)
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
out_html = os.path.join(self.output_dir, "mapa_interactivo.html")
|
|
263
|
+
fig.write_html(out_html, config={"responsive": True, "displayModeBar": True})
|
|
264
|
+
print(f"✔ Dashboard Dual interactivo generado en: {out_html}")
|
|
265
|
+
return out_html
|
|
266
|
+
|
|
267
|
+
def save_final_dataset(self, df: pd.DataFrame) -> str:
|
|
268
|
+
out_csv = os.path.join(self.output_dir, "corpus_con_topicos.csv")
|
|
269
|
+
df.to_csv(out_csv, index=False, encoding="utf-8-sig")
|
|
270
|
+
print(f"✔ Dataset consolidado guardado en: {out_csv}")
|
|
271
|
+
return out_csv
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: scimapper
|
|
3
|
+
Version: 1.0.1
|
|
4
|
+
Summary: An automated scientific mapping and systematic literature review framework with dense semantic clustering and PRISMA 2020 compliance.
|
|
5
|
+
Author-email: Gustavo Andrés Futo <gustavoandresfu@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/gustavoandresfu/SciMapper
|
|
8
|
+
Project-URL: Bug Tracker, https://github.com/gustavoandresfu/SciMapper/issues
|
|
9
|
+
Keywords: systematic-literature-review,systematic-mapping-study,scimapper,scientific-mapping,topic-modeling,sentence-transformers,bertopic,prisma-2020,openalex,evidence-synthesis,nlp
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Requires-Python: >=3.9
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
21
|
+
Requires-Dist: ruff>=0.4.0; extra == "dev"
|
|
22
|
+
Provides-Extra: web
|
|
23
|
+
Requires-Dist: streamlit>=1.30.0; extra == "web"
|
|
24
|
+
Provides-Extra: dependencies
|
|
25
|
+
Requires-Dist: requests>=2.28.0; extra == "dependencies"
|
|
26
|
+
Requires-Dist: pandas>=1.5.0; extra == "dependencies"
|
|
27
|
+
Requires-Dist: numpy>=1.23.0; extra == "dependencies"
|
|
28
|
+
Requires-Dist: scikit-learn>=1.2.0; extra == "dependencies"
|
|
29
|
+
Requires-Dist: scipy>=1.10.0; extra == "dependencies"
|
|
30
|
+
Requires-Dist: sentence-transformers>=2.2.0; extra == "dependencies"
|
|
31
|
+
Requires-Dist: torch>=2.0.0; extra == "dependencies"
|
|
32
|
+
Requires-Dist: plotly>=5.13.0; extra == "dependencies"
|
|
33
|
+
Requires-Dist: python-docx>=0.8.11; extra == "dependencies"
|
|
34
|
+
Requires-Dist: tqdm>=4.65.0; extra == "dependencies"
|
|
35
|
+
Dynamic: license-file
|
|
36
|
+
|
|
37
|
+
<div align="center">
|
|
38
|
+
|
|
39
|
+
# 🚀 SciMapper
|
|
40
|
+
### Automated Scientific Mapping & Systematic Literature Reviews with Dense Semantic Clustering
|
|
41
|
+
|
|
42
|
+
[](https://www.python.org/)
|
|
43
|
+
[](https://opensource.org/licenses/MIT)
|
|
44
|
+
[](https://www.prisma-statement.org/)
|
|
45
|
+
[](https://openalex.org/)
|
|
46
|
+
[](https://www.sbert.net/)
|
|
47
|
+
|
|
48
|
+
**An end-to-end, domain-agnostic framework designed to conduct reproducible Systematic Literature Reviews (SLR) and Systematic Mapping Studies (SMS).**
|
|
49
|
+
It automates academic corpus harvesting, deduplication, dense semantic embedding, unsupervised clustering, taxonomic discovery, interactive 2D cartography, and manuscript drafting—while keeping **researchers firmly in the loop**.
|
|
50
|
+
|
|
51
|
+
🌐 **Languages:** **English** • [Español (Guía en Castellano)](README_ES.md)
|
|
52
|
+
|
|
53
|
+
[Quickstart](#-quickstart-in-3-steps) • [Architecture](#-framework-architecture) • [Comparison](#-comparison-with-the-state-of-the-art) • [Theoretical Foundation](#-theoretical-foundations) • [Citation](#-citation)
|
|
54
|
+
|
|
55
|
+
</div>
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## 🌟 Key Features
|
|
60
|
+
|
|
61
|
+
* 📡 **Automated Scholarly Harvesting:** Direct connection to the **OpenAlex API** (*Polite Pool* with 250M+ works) executing boolean queries and reconstructing inverted-index abstracts in $O(N \log N)$.
|
|
62
|
+
* 🧹 **Bidirectional Deduplication:** Collapses duplicates across arXiv, IEEE, ACM, Springer, and PubMed via canonical lowercase DOI and normalized lexical title hashing.
|
|
63
|
+
* 🧠 **Domain-Specific Transformer Presets:** Plug-and-play dense representations via `Sentence-Transformers`:
|
|
64
|
+
* `software_engineering`: `sentence-transformers/all-MiniLM-L6-v2`
|
|
65
|
+
* `biomedical_health`: `microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract`
|
|
66
|
+
* `scientific_general`: `allenai/scibert_scivocab_uncased`
|
|
67
|
+
* `finance_business`: `ProsusAI/finbert`
|
|
68
|
+
* `multilingual`: `sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2`
|
|
69
|
+
* 🔬 **Unsupervised Topic Discovery ($c$-TF-IDF):** Discovers latent research themes without generative hallucination, extracting mathematically distinctive n-grams using *Class-based TF-IDF*.
|
|
70
|
+
* 🗺️ **Interactive Cartography & Cluster Hulls:** Produces publication-grade 2D HTML dashboards (Plotly) featuring:
|
|
71
|
+
* Regional boundaries via **Convex Hulls** with smoothed spline contours.
|
|
72
|
+
* Cluster centroids with real-time metadata cards ($N$, percentage, top keywords).
|
|
73
|
+
* Layer switching (*Full View*, *Cluster Boundaries Only*, *Paper Scatter Points*).
|
|
74
|
+
* 📈 **Temporal Dynamics & Hot Topic Radar:** Computes Compound Annual Growth Rates (CAGR) per cluster to classify topics into 🚀 *Emerging*, ⚖️ *Mature*, or 📉 *Declining*.
|
|
75
|
+
* 🛡️ **Mathematical PRISMA 2020 Audit:** Objective isolation of out-of-scope or foreign language clusters as documented exclusion criteria.
|
|
76
|
+
* 📝 **Automated Manuscript Generation:** Compiles an executive Microsoft Word (`.docx`) template complete with cover page, research questions (RQs), PRISMA summary tables, taxonomy distribution, and structured discussion sections.
|
|
77
|
+
|
|
78
|
+
---
|
|
79
|
+
|
|
80
|
+
## 🏗️ Framework Architecture
|
|
81
|
+
|
|
82
|
+
```mermaid
|
|
83
|
+
graph TD
|
|
84
|
+
A["⚙️ 1. Project Configuration<br><code>scimapper_config.json</code>"] --> B["📡 2. Harvester & Deduplicator<br><code>collector.py (OpenAlex API)</code>"]
|
|
85
|
+
B --> C["🧠 3. Semantic Embeddings<br><code>embedder.py (SBERT / MiniLM)</code>"]
|
|
86
|
+
C --> D["🔬 4. Clustering & c-TF-IDF<br><code>clusterer.py (PCA + t-SNE)</code>"]
|
|
87
|
+
D --> E1["🗺️ 5a. Interactive Cartography<br><code>visualizer.py (Plotly 2D + Hulls)</code>"]
|
|
88
|
+
D --> E2["⚖️ 5b. Human-in-the-Loop Review<br><code>discovered_topics.json</code>"]
|
|
89
|
+
E2 --> F["📝 6. Automated Manuscript<br><code>doc_builder.py (.docx Output)</code>"]
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
## ⚡ Quickstart in 3 Steps
|
|
95
|
+
|
|
96
|
+
### 1. Installation
|
|
97
|
+
|
|
98
|
+
Clone the repository and install the dependencies:
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
git clone https://github.com/gustavoandresfu/SciMapper.git
|
|
102
|
+
cd SciMapper
|
|
103
|
+
pip install -r requirements.txt
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
*(Optional)* Install as an editable command-line utility:
|
|
107
|
+
```bash
|
|
108
|
+
pip install -e .
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
---
|
|
112
|
+
|
|
113
|
+
### 2. Initialize a Project
|
|
114
|
+
|
|
115
|
+
Launch the interactive configuration wizard (prompts for domain preset, search queries, and project metadata):
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
python main.py --init
|
|
119
|
+
# Or if installed via pip:
|
|
120
|
+
scimapper --init
|
|
121
|
+
```
|
|
122
|
+
*(Or use `python main.py --init --quick` to generate the baseline template directly without prompts).*
|
|
123
|
+
|
|
124
|
+
The generated `sms_config.json` is ready to inspect and refine:
|
|
125
|
+
|
|
126
|
+
```json
|
|
127
|
+
{
|
|
128
|
+
"project_metadata": {
|
|
129
|
+
"title": "Systematic Literature Review: Generative AI in Healthcare",
|
|
130
|
+
"short_name": "SciMapper_GenAI_Health",
|
|
131
|
+
"authors": ["Your Name"],
|
|
132
|
+
"temporal_window": [2019, 2026],
|
|
133
|
+
"output_dir": "./output"
|
|
134
|
+
},
|
|
135
|
+
"research_questions": {
|
|
136
|
+
"RQ1": "What foundational architectures and LLMs are predominantly adopted in clinical workflows?",
|
|
137
|
+
"RQ2": "What empirical metrics and validation benchmarks are reported for diagnostic accuracy?",
|
|
138
|
+
"RQ3": "What open challenges regarding hallucinations and ethical safety remain unaddressed?"
|
|
139
|
+
},
|
|
140
|
+
"search_strategy": {
|
|
141
|
+
"openalex_email": "researcher@university.edu",
|
|
142
|
+
"max_results_per_query": 100,
|
|
143
|
+
"queries": [
|
|
144
|
+
"\"generative ai\" AND \"clinical diagnostics\"",
|
|
145
|
+
"\"large language models\" AND \"medical decision support\""
|
|
146
|
+
]
|
|
147
|
+
},
|
|
148
|
+
"nlp_tuning": {
|
|
149
|
+
"domain_preset": "biomedical_health",
|
|
150
|
+
"n_clusters": 5
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
---
|
|
156
|
+
|
|
157
|
+
### 3. Run the Automated Pipeline
|
|
158
|
+
|
|
159
|
+
Execute the discovery phase:
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
python main.py --run-discovery --config sms_config.json
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
1. Review and refine taxonomic names in `output/discovered_topics.json`.
|
|
166
|
+
2. Inspect the pre-selected seminal papers in `output/candidatos_discusion.csv`.
|
|
167
|
+
3. Compile the interactive dashboard and Word document:
|
|
168
|
+
|
|
169
|
+
```bash
|
|
170
|
+
python main.py --compile --config sms_config.json
|
|
171
|
+
python main.py --build-doc --config sms_config.json
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
Or execute everything end-to-end in a single run:
|
|
175
|
+
|
|
176
|
+
```bash
|
|
177
|
+
python main.py --all --config sms_config.json
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
---
|
|
181
|
+
|
|
182
|
+
## 📊 Comparison with the State-of-the-Art
|
|
183
|
+
|
|
184
|
+
| Capability | ASReview | Generic BERTopic | Manual Review Tools (Covidence / Rayyan) | **SciMapper** |
|
|
185
|
+
| :--- | :---: | :---: | :---: | :---: |
|
|
186
|
+
| **Academic Ingestion** | ❌ Manual (CSV/RIS) | ❌ None | ❌ Manual Import | **✅ Automated OpenAlex API** |
|
|
187
|
+
| **Deduplication** | ⚠️ Basic | ❌ None | ⚠️ Exact match | **✅ Canonical DOI + Title Hash** |
|
|
188
|
+
| **Semantic Representations** | ✅ Optional | ✅ Native | ❌ None | **✅ Pre-tuned Domain Transformers** |
|
|
189
|
+
| **Topic Modeling ($c$-TF-IDF)** | ❌ Binary only | ✅ Yes | ❌ None | **✅ Mathematical $c$-TF-IDF** |
|
|
190
|
+
| **Objective PRISMA Exclusion** | ❌ Human judgment | ❌ None | ❌ Human labeling | **✅ Algorithmic Outlier Clustering** |
|
|
191
|
+
| **Cartography with Convex Hulls** | ❌ None | ⚠️ Scatter only | ❌ None | **✅ 2D Hulls + Centroids + Density** |
|
|
192
|
+
| **Temporal CAGR Tracking** | ❌ None | ❌ Manual | ❌ None | **✅ Automated Hot-Topic Radar** |
|
|
193
|
+
| **Automated Manuscript (.docx)** | ❌ None | ❌ None | ❌ Export tables | **✅ Formatted PRISMA Word Paper** |
|
|
194
|
+
|
|
195
|
+
---
|
|
196
|
+
|
|
197
|
+
## 🔬 Theoretical Foundations
|
|
198
|
+
|
|
199
|
+
The framework's methodological and algorithmic foundations are grounded in established literature:
|
|
200
|
+
|
|
201
|
+
1. **Topic Modeling & Semantic Representation:**
|
|
202
|
+
Based on the modular architecture described by **Jay Alammar and Maarten Grootendorst** in *Hands-On Large Language Models* (O'Reilly, Chapter 5: *Text Clustering and Topic Modeling*), decoupling dense vector embeddings, dimensionality reduction, clustering, and class-based TF-IDF ($c$-TF-IDF).
|
|
203
|
+
2. **Systematic Mapping Methodology:**
|
|
204
|
+
Follows the seminal guidelines of **Petersen et al. (2008)** (*Systematic Mapping Studies in Software Engineering*) and **Kitchenham & Charters (2007)**.
|
|
205
|
+
3. **Auditability & Reporting Standards:**
|
|
206
|
+
Strict adherence to the **PRISMA 2020** statement (*Preferred Reporting Items for Systematic Reviews and Meta-Analyses*), ensuring transparent and reproducible record accounting.
|
|
207
|
+
|
|
208
|
+
---
|
|
209
|
+
|
|
210
|
+
## 📁 Repository Structure
|
|
211
|
+
|
|
212
|
+
```text
|
|
213
|
+
SciMapper/
|
|
214
|
+
├── config_template.json # Master template for project setup
|
|
215
|
+
├── test_config.json # Quick verification test suite
|
|
216
|
+
├── main.py # Unified CLI orchestrator
|
|
217
|
+
├── requirements.txt # Python dependency specification
|
|
218
|
+
├── pyproject.toml # Standard packaging & CLI entry point (scimapper)
|
|
219
|
+
├── CITATION.cff # Academic citation metadata
|
|
220
|
+
├── LICENSE # MIT Open Source License
|
|
221
|
+
└── core/
|
|
222
|
+
├── collector.py # OpenAlex API ingestion & abstract reconstruction
|
|
223
|
+
├── embedder.py # Multi-domain Sentence-Transformer pipeline
|
|
224
|
+
├── clusterer.py # Dimensionality reduction (PCA+t-SNE) & c-TF-IDF
|
|
225
|
+
├── selector.py # Stratified seminal paper recommendation
|
|
226
|
+
├── visualizer.py # Plotly 2D interactive maps with Convex Hulls
|
|
227
|
+
└── doc_builder.py # Automated PRISMA Word manuscript generation
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
---
|
|
231
|
+
|
|
232
|
+
## 📖 Citation
|
|
233
|
+
|
|
234
|
+
If you use **SciMapper** in your research, academic reviews, or dissertations, please cite it as:
|
|
235
|
+
|
|
236
|
+
```bibtex
|
|
237
|
+
@software{futo_scimapper_2026,
|
|
238
|
+
author = {Gustavo Andrés Futo},
|
|
239
|
+
title = {SciMapper: Automated Scientific Mapping & Systematic Literature Reviews with Dense Semantic Clustering and PRISMA 2020 Compliance},
|
|
240
|
+
year = {2026},
|
|
241
|
+
publisher = {GitHub},
|
|
242
|
+
journal = {GitHub repository},
|
|
243
|
+
howpublished = {\url{https://github.com/gustavoandresfu/SciMapper}}
|
|
244
|
+
}
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
---
|
|
248
|
+
|
|
249
|
+
## 📄 License
|
|
250
|
+
|
|
251
|
+
This project is licensed under the **MIT License** — see the [LICENSE](LICENSE) file for details.
|
|
252
|
+
Author: **Gustavo Andrés Futo** (2026).
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
core/__init__.py,sha256=E2r3eKcI1M1FfgvWdfFt55GrRc2ZA9sazMXLQEKU54k,262
|
|
2
|
+
core/clusterer.py,sha256=H-rwlbPq3lCr8dHuP0Q42C0JSYOBd8KMH4e0hKxUcZ8,7499
|
|
3
|
+
core/collector.py,sha256=AxfF6LE_ISmomKKUiyzjTat-uoppyVsRnQj6Bxi9r-8,6024
|
|
4
|
+
core/config_template.json,sha256=jySu7Lc81gcGscAoQJ8olvBqYBnfrqFg6Y8Mjji6oNE,1627
|
|
5
|
+
core/doc_builder.py,sha256=ykb1a03qwkOes8DFAlgw2H9OaOsoMa-HsPoE1huFJts,17108
|
|
6
|
+
core/embedder.py,sha256=RDnuRokw3p75fn0DfjpF3T1cx4VjYO4lzJO09Cpk5xI,2998
|
|
7
|
+
core/selector.py,sha256=VRRaKaieP1q1BQB1F9Ot54Amfuzz9umbJUoYr3J1arQ,6559
|
|
8
|
+
core/utils.py,sha256=xthpQeghPjxW2JkYRE9qWa5kwefweUD0-rVEH2QQHwg,2866
|
|
9
|
+
core/visualizer.py,sha256=92kC5AOVuChOH6ecquiM4nRfSjGvWMpqZyWbM4KyVhU,11848
|
|
10
|
+
scimapper-1.0.1.dist-info/licenses/LICENSE,sha256=ml5HlDG7GbJ-jdXvUiYOaXv9rrrVuv6l1Ja_TUS92e8,1077
|
|
11
|
+
scimapper-1.0.1.dist-info/METADATA,sha256=GvwG_di99DJ1QTdDMbxwTSjz2hvERF6dpL3bbf8zC5Q,12106
|
|
12
|
+
scimapper-1.0.1.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
13
|
+
scimapper-1.0.1.dist-info/entry_points.txt,sha256=2km7mWcGb_HIMK35IP1iZB8KlCe7NR4WJ6cMVxM9cEk,40
|
|
14
|
+
scimapper-1.0.1.dist-info/top_level.txt,sha256=0ebp8k35dOTHVwbY3UcokRsjC7cWLN_iaJ70tIcmODU,5
|
|
15
|
+
scimapper-1.0.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Gustavo Andrés Futo
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
core
|