topiclayers 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- topiclayers/__init__.py +3 -0
- topiclayers/adapters/__init__.py +7 -0
- topiclayers/adapters/csv.py +91 -0
- topiclayers/adapters/registry.py +47 -0
- topiclayers/adapters/twitter.py +73 -0
- topiclayers/cli.py +192 -0
- topiclayers/config.py +84 -0
- topiclayers/core/__init__.py +26 -0
- topiclayers/core/data.py +217 -0
- topiclayers/core/network.py +281 -0
- topiclayers/core/topic.py +712 -0
- topiclayers/manifest.py +88 -0
- topiclayers/schema.py +27 -0
- topiclayers/status.py +214 -0
- topiclayers/support/__init__.py +1 -0
- topiclayers/support/cache.py +11 -0
- topiclayers/support/repro.py +16 -0
- topiclayers-0.2.0.dist-info/METADATA +201 -0
- topiclayers-0.2.0.dist-info/RECORD +22 -0
- topiclayers-0.2.0.dist-info/WHEEL +4 -0
- topiclayers-0.2.0.dist-info/entry_points.txt +3 -0
- topiclayers-0.2.0.dist-info/licenses/LICENSE +21 -0
topiclayers/manifest.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Run manifest: reproducibility metadata for each pipeline run."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from dataclasses import dataclass, asdict
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Optional
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass
|
|
12
|
+
class RunManifest:
|
|
13
|
+
topiclayers_version: str
|
|
14
|
+
git_hash: Optional[str]
|
|
15
|
+
python_version: str
|
|
16
|
+
library_versions: dict
|
|
17
|
+
params: dict
|
|
18
|
+
input_hash: str
|
|
19
|
+
row_counts: dict
|
|
20
|
+
dropped_edges: int
|
|
21
|
+
topic_distribution: dict
|
|
22
|
+
status: str
|
|
23
|
+
timestamp: str
|
|
24
|
+
|
|
25
|
+
@classmethod
|
|
26
|
+
def create(cls, params, row_counts, topic_dist, dropped, status):
|
|
27
|
+
import hashlib
|
|
28
|
+
import subprocess
|
|
29
|
+
import sys
|
|
30
|
+
from datetime import datetime, timezone
|
|
31
|
+
|
|
32
|
+
import topiclayers
|
|
33
|
+
|
|
34
|
+
git_hash = None
|
|
35
|
+
try:
|
|
36
|
+
result = subprocess.run(
|
|
37
|
+
["git", "rev-parse", "HEAD"],
|
|
38
|
+
capture_output=True, text=True, timeout=5,
|
|
39
|
+
)
|
|
40
|
+
if result.returncode == 0:
|
|
41
|
+
git_hash = result.stdout.strip()
|
|
42
|
+
except Exception:
|
|
43
|
+
pass
|
|
44
|
+
|
|
45
|
+
library_versions = _gather_versions()
|
|
46
|
+
|
|
47
|
+
params_serialized = json.dumps(params, sort_keys=True, default=str)
|
|
48
|
+
input_hash = hashlib.sha256(params_serialized.encode()).hexdigest()[:16]
|
|
49
|
+
|
|
50
|
+
return cls(
|
|
51
|
+
topiclayers_version=topiclayers.__version__,
|
|
52
|
+
git_hash=git_hash,
|
|
53
|
+
python_version=sys.version.split()[0],
|
|
54
|
+
library_versions=library_versions,
|
|
55
|
+
params=params,
|
|
56
|
+
input_hash=input_hash,
|
|
57
|
+
row_counts=row_counts,
|
|
58
|
+
dropped_edges=dropped,
|
|
59
|
+
topic_distribution=topic_dist,
|
|
60
|
+
status=status,
|
|
61
|
+
timestamp=datetime.now(timezone.utc).isoformat(),
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
def write(self, path: Path) -> None:
|
|
65
|
+
with open(path, "w") as f:
|
|
66
|
+
json.dump(asdict(self), f, indent=2, default=str)
|
|
67
|
+
|
|
68
|
+
@classmethod
|
|
69
|
+
def load(cls, path: Path) -> "RunManifest":
|
|
70
|
+
with open(path) as f:
|
|
71
|
+
data = json.load(f)
|
|
72
|
+
return cls(**data)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _gather_versions() -> dict:
|
|
76
|
+
deps = [
|
|
77
|
+
"numpy", "pandas", "networkx", "scipy",
|
|
78
|
+
"bertopic", "umap", "sklearn", "sentence_transformers",
|
|
79
|
+
"openai", "qdrant_client",
|
|
80
|
+
]
|
|
81
|
+
versions: dict = {}
|
|
82
|
+
for pkg in deps:
|
|
83
|
+
try:
|
|
84
|
+
mod = __import__(pkg)
|
|
85
|
+
versions[pkg] = getattr(mod, "__version__", "unknown")
|
|
86
|
+
except ImportError:
|
|
87
|
+
versions[pkg] = "not installed"
|
|
88
|
+
return versions
|
topiclayers/schema.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Generic post schema: Post dataclass and InteractionType enum."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from enum import Enum
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class InteractionType(str, Enum):
|
|
10
|
+
ORIGINAL = "original"
|
|
11
|
+
REPOST = "repost"
|
|
12
|
+
QUOTE = "quote"
|
|
13
|
+
REPLY = "reply"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class Post:
|
|
18
|
+
post_id: str
|
|
19
|
+
user_id: str
|
|
20
|
+
text: str
|
|
21
|
+
created_at: Optional[str] = None
|
|
22
|
+
lang: Optional[str] = None
|
|
23
|
+
interaction_type: Optional[InteractionType] = None
|
|
24
|
+
target_post_id: Optional[str] = None
|
|
25
|
+
target_user_id: Optional[str] = None
|
|
26
|
+
mentions: list[str] = field(default_factory=list)
|
|
27
|
+
extra: dict = field(default_factory=dict)
|
topiclayers/status.py
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"""Run status: live stage tracking and ETA for long-running pipeline runs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import threading
|
|
8
|
+
import time
|
|
9
|
+
from datetime import datetime, timezone
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Optional
|
|
12
|
+
|
|
13
|
+
STAGES = ["load", "data", "embed", "topic_model", "label", "network", "manifest"]
|
|
14
|
+
_STATUS_FILE = "run_status.json"
|
|
15
|
+
_TIMINGS_FILE = "stage_timings.json"
|
|
16
|
+
_MIN_UPDATE_INTERVAL = 1.0
|
|
17
|
+
_HEARTBEAT_INTERVAL = 5.0
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class RunStatus:
|
|
21
|
+
def __init__(self, output_dir: Path, name: str):
|
|
22
|
+
self.output_dir = Path(output_dir)
|
|
23
|
+
self.name = name
|
|
24
|
+
self.path = self.output_dir / _STATUS_FILE
|
|
25
|
+
self.timings_path = self.output_dir / "cache" / _TIMINGS_FILE
|
|
26
|
+
self.current_stage: Optional[str] = None
|
|
27
|
+
self.stage_index: Optional[int] = None
|
|
28
|
+
self.stage_total: Optional[int] = None
|
|
29
|
+
self.processed: Optional[int] = None
|
|
30
|
+
self.started_at = datetime.now(timezone.utc)
|
|
31
|
+
self.stage_started_at: Optional[datetime] = None
|
|
32
|
+
self._last_write = 0.0
|
|
33
|
+
self._stop_heartbeat = threading.Event()
|
|
34
|
+
self._heartbeat_thread: Optional[threading.Thread] = None
|
|
35
|
+
|
|
36
|
+
def start(self) -> None:
|
|
37
|
+
self._start_heartbeat()
|
|
38
|
+
self._write(status="running")
|
|
39
|
+
|
|
40
|
+
def _start_heartbeat(self) -> None:
|
|
41
|
+
# keeps stage_elapsed_seconds fresh during opaque stages (e.g. UMAP +
|
|
42
|
+
# HDBSCAN) that never call update()
|
|
43
|
+
def beat():
|
|
44
|
+
while not self._stop_heartbeat.wait(_HEARTBEAT_INTERVAL):
|
|
45
|
+
if self.current_stage is not None:
|
|
46
|
+
self._write(status="running", force=True)
|
|
47
|
+
|
|
48
|
+
self._heartbeat_thread = threading.Thread(target=beat, daemon=True)
|
|
49
|
+
self._heartbeat_thread.start()
|
|
50
|
+
|
|
51
|
+
def stage(self, name: str, total: Optional[int] = None) -> None:
|
|
52
|
+
now = datetime.now(timezone.utc)
|
|
53
|
+
if self.current_stage is not None and self.stage_started_at is not None:
|
|
54
|
+
duration = (now - self.stage_started_at).total_seconds()
|
|
55
|
+
self._record_timing(self.current_stage, duration)
|
|
56
|
+
self.current_stage = name
|
|
57
|
+
self.stage_index = STAGES.index(name) if name in STAGES else None
|
|
58
|
+
self.stage_total = total
|
|
59
|
+
self.processed = 0
|
|
60
|
+
self.stage_started_at = now
|
|
61
|
+
self._write(status="running", force=True)
|
|
62
|
+
|
|
63
|
+
def update(self, processed: int, total: Optional[int] = None) -> None:
|
|
64
|
+
self.processed = processed
|
|
65
|
+
if total is not None:
|
|
66
|
+
self.stage_total = total
|
|
67
|
+
self._write(status="running", throttle=True)
|
|
68
|
+
|
|
69
|
+
def flush(self) -> None:
|
|
70
|
+
self._write(status="running", force=True)
|
|
71
|
+
|
|
72
|
+
def finish(self, status: str = "success") -> None:
|
|
73
|
+
self._stop_heartbeat.set()
|
|
74
|
+
if self.current_stage is not None and self.stage_started_at is not None:
|
|
75
|
+
duration = (datetime.now(timezone.utc) - self.stage_started_at).total_seconds()
|
|
76
|
+
self._record_timing(self.current_stage, duration)
|
|
77
|
+
self._write(status=status, force=True)
|
|
78
|
+
|
|
79
|
+
def _eta_seconds(self) -> Optional[float]:
|
|
80
|
+
if (
|
|
81
|
+
self.processed is None
|
|
82
|
+
or self.stage_total is None
|
|
83
|
+
or self.stage_started_at is None
|
|
84
|
+
or self.processed <= 0
|
|
85
|
+
or self.stage_total <= self.processed
|
|
86
|
+
):
|
|
87
|
+
return None
|
|
88
|
+
elapsed = (datetime.now(timezone.utc) - self.stage_started_at).total_seconds()
|
|
89
|
+
if elapsed <= 0:
|
|
90
|
+
return None
|
|
91
|
+
rate = self.processed / elapsed
|
|
92
|
+
return (self.stage_total - self.processed) / rate
|
|
93
|
+
|
|
94
|
+
def _write(self, status: str, force: bool = False, throttle: bool = False) -> None:
|
|
95
|
+
now_monotonic = time.monotonic()
|
|
96
|
+
if throttle and not force:
|
|
97
|
+
if now_monotonic - self._last_write < _MIN_UPDATE_INTERVAL:
|
|
98
|
+
return
|
|
99
|
+
self._last_write = now_monotonic
|
|
100
|
+
|
|
101
|
+
now = datetime.now(timezone.utc)
|
|
102
|
+
payload = {
|
|
103
|
+
"name": self.name,
|
|
104
|
+
"status": status,
|
|
105
|
+
"stages": STAGES,
|
|
106
|
+
"current_stage": self.current_stage,
|
|
107
|
+
"stage_index": self.stage_index,
|
|
108
|
+
"stage_started_at": self.stage_started_at.isoformat() if self.stage_started_at else None,
|
|
109
|
+
"stage_elapsed_seconds": round((now - self.stage_started_at).total_seconds(), 2) if self.stage_started_at else None,
|
|
110
|
+
"total_elapsed_seconds": round((now - self.started_at).total_seconds(), 2),
|
|
111
|
+
"processed": self.processed,
|
|
112
|
+
"total": self.stage_total,
|
|
113
|
+
"eta_seconds": round(self._eta_seconds(), 2) if self._eta_seconds() is not None else None,
|
|
114
|
+
"updated_at": now.isoformat(),
|
|
115
|
+
}
|
|
116
|
+
self.output_dir.mkdir(parents=True, exist_ok=True)
|
|
117
|
+
tmp = self.path.with_suffix(f".tmp{threading.get_ident()}")
|
|
118
|
+
with open(tmp, "w") as f:
|
|
119
|
+
json.dump(payload, f, indent=2)
|
|
120
|
+
os.replace(tmp, self.path)
|
|
121
|
+
|
|
122
|
+
def _record_timing(self, stage: str, duration: float) -> None:
|
|
123
|
+
self.timings_path.parent.mkdir(parents=True, exist_ok=True)
|
|
124
|
+
records: list[dict] = []
|
|
125
|
+
if self.timings_path.exists():
|
|
126
|
+
try:
|
|
127
|
+
with open(self.timings_path) as f:
|
|
128
|
+
records = json.load(f)
|
|
129
|
+
except Exception:
|
|
130
|
+
records = []
|
|
131
|
+
records.append({
|
|
132
|
+
"stage": stage,
|
|
133
|
+
"duration_seconds": round(duration, 2),
|
|
134
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
135
|
+
})
|
|
136
|
+
tmp = self.timings_path.with_suffix(".tmp")
|
|
137
|
+
with open(tmp, "w") as f:
|
|
138
|
+
json.dump(records, f, indent=2)
|
|
139
|
+
os.replace(tmp, self.timings_path)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def load_status(path: Path) -> dict:
|
|
143
|
+
with open(path) as f:
|
|
144
|
+
return json.load(f)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def historical_eta(timings_path: Path, stage: str, last_n: int = 3) -> Optional[float]:
|
|
148
|
+
if not timings_path.exists():
|
|
149
|
+
return None
|
|
150
|
+
try:
|
|
151
|
+
with open(timings_path) as f:
|
|
152
|
+
records = json.load(f)
|
|
153
|
+
except Exception:
|
|
154
|
+
return None
|
|
155
|
+
durations = sorted(r["duration_seconds"] for r in records if r.get("stage") == stage)
|
|
156
|
+
if not durations:
|
|
157
|
+
return None
|
|
158
|
+
return durations[len(durations) // 2]
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def format_seconds(seconds: Optional[float]) -> str:
|
|
162
|
+
if seconds is None:
|
|
163
|
+
return "unknown"
|
|
164
|
+
seconds = int(round(seconds))
|
|
165
|
+
if seconds < 60:
|
|
166
|
+
return f"{seconds}s"
|
|
167
|
+
if seconds < 3600:
|
|
168
|
+
return f"{seconds // 60}m {seconds % 60:02d}s"
|
|
169
|
+
return f"{seconds // 3600}h {(seconds % 3600) // 60:02d}m"
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def print_status(output_dir: Path) -> int:
|
|
173
|
+
path = output_dir / _STATUS_FILE
|
|
174
|
+
if not path.exists():
|
|
175
|
+
print(f"No run_status.json found in {output_dir}.")
|
|
176
|
+
print("Either no run has started there, or the config pointed to a different output_dir.")
|
|
177
|
+
return 1
|
|
178
|
+
|
|
179
|
+
data = load_status(path)
|
|
180
|
+
status = data.get("status", "unknown")
|
|
181
|
+
stage = data.get("current_stage")
|
|
182
|
+
idx = data.get("stage_index")
|
|
183
|
+
n_stages = len(data.get("stages", STAGES))
|
|
184
|
+
|
|
185
|
+
if idx is not None and stage:
|
|
186
|
+
print(f"Stage {idx + 1}/{n_stages}: {stage}")
|
|
187
|
+
elif stage:
|
|
188
|
+
print(f"Stage: {stage}")
|
|
189
|
+
print(f"Status: {status}")
|
|
190
|
+
print(f"Run: {data.get('name')}")
|
|
191
|
+
|
|
192
|
+
if status == "running":
|
|
193
|
+
total = data.get("total")
|
|
194
|
+
processed = data.get("processed")
|
|
195
|
+
if total:
|
|
196
|
+
pct = 100 * processed / total if processed is not None else 0
|
|
197
|
+
print(f"Progress: {processed}/{total} ({pct:.0f}%)")
|
|
198
|
+
stage_elapsed = data.get("stage_elapsed_seconds")
|
|
199
|
+
if stage_elapsed is not None:
|
|
200
|
+
print(f"Stage elapsed: {format_seconds(stage_elapsed)}")
|
|
201
|
+
eta = data.get("eta_seconds")
|
|
202
|
+
if eta is not None:
|
|
203
|
+
print(f"Stage ETA: ~{format_seconds(eta)}")
|
|
204
|
+
else:
|
|
205
|
+
hist = historical_eta(output_dir / "cache" / _TIMINGS_FILE, stage)
|
|
206
|
+
if hist is not None:
|
|
207
|
+
print(f"No live ETA for this stage; past runs took ~{format_seconds(hist)} (median).")
|
|
208
|
+
else:
|
|
209
|
+
print("No live ETA for this stage (no per-item counter, no past runs yet).")
|
|
210
|
+
print(f"Total elapsed: {format_seconds(data.get('total_elapsed_seconds'))}")
|
|
211
|
+
elif status in ("success", "failed"):
|
|
212
|
+
print(f"Total elapsed: {format_seconds(data.get('total_elapsed_seconds'))}")
|
|
213
|
+
print(f"Updated: {data.get('updated_at')}")
|
|
214
|
+
return 0
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Shared small helpers: cache keys and reproducibility seeds."""
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def set_global_seed(seed: int) -> None:
|
|
5
|
+
import random
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
random.seed(seed)
|
|
10
|
+
np.random.seed(seed)
|
|
11
|
+
try:
|
|
12
|
+
import torch
|
|
13
|
+
|
|
14
|
+
torch.manual_seed(seed)
|
|
15
|
+
except ImportError:
|
|
16
|
+
pass
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: topiclayers
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: From any collection of posts to topical multilayer networks — hardened and social-scientist-friendly
|
|
5
|
+
License: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Keywords: topic-modeling,network-analysis,social-science,polarisation
|
|
8
|
+
Author: alessiogandelli
|
|
9
|
+
Author-email: alessiogandelli99@gmail.com
|
|
10
|
+
Requires-Python: >=3.12,<3.13
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Provides-Extra: labeling
|
|
17
|
+
Requires-Dist: bertopic (>=0.17.4,<0.18.0)
|
|
18
|
+
Requires-Dist: igraph (>=0.11.4)
|
|
19
|
+
Requires-Dist: jsonlines (>=4.0.0,<5.0.0)
|
|
20
|
+
Requires-Dist: litellm (>=1.40) ; extra == "labeling"
|
|
21
|
+
Requires-Dist: networkx (>=3.6,<4.0)
|
|
22
|
+
Requires-Dist: numpy (>=1.26,<3)
|
|
23
|
+
Requires-Dist: openai (>=3.0.0,<4.0.0)
|
|
24
|
+
Requires-Dist: pandas (>=2.2,<4)
|
|
25
|
+
Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
|
|
26
|
+
Requires-Dist: pyyaml (>=6.0,<7.0)
|
|
27
|
+
Requires-Dist: qdrant-client (>=1.19.0,<2.0.0)
|
|
28
|
+
Requires-Dist: scikit-learn (>=1.5)
|
|
29
|
+
Requires-Dist: sentence-transformers (>=6.0.0,<7.0.0)
|
|
30
|
+
Requires-Dist: umap-learn (>=0.5.12,<0.6.0)
|
|
31
|
+
Requires-Dist: uunet (>=2.1.1,<3.0.0)
|
|
32
|
+
Project-URL: Changelog, https://github.com/alessiogandelli/topiclayers/blob/main/CHANGELOG.md
|
|
33
|
+
Project-URL: Issues, https://github.com/alessiogandelli/topiclayers/issues
|
|
34
|
+
Project-URL: Repository, https://github.com/alessiogandelli/topiclayers
|
|
35
|
+
Description-Content-Type: text/markdown
|
|
36
|
+
|
|
37
|
+
# topiclayers
|
|
38
|
+
|
|
39
|
+
> From posts to topical multilayer networks — hardened and social-scientist-friendly.
|
|
40
|
+
|
|
41
|
+
## Quickstart
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
# Requires Python 3.12
|
|
45
|
+
pip install topiclayers
|
|
46
|
+
topiclayers run examples/generic.yml
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
That's it. You get a GML network file in `./out/generic_toy/networks/`.
|
|
50
|
+
|
|
51
|
+
## What does it do?
|
|
52
|
+
|
|
53
|
+
1. **Loads** your posts (Twitter JSONL, CSV, Parquet).
|
|
54
|
+
2. **Cleans** text (removes URLs, @mentions, newlines).
|
|
55
|
+
3. **Models topics** using BERTopic with SentenceTransformer embeddings.
|
|
56
|
+
4. **Labels topics** with an LLM (local via Ollama by default) and propagates topic labels through retweet chains.
|
|
57
|
+
5. **Builds networks**: single-layer retweet, multilayer per-topic, and bipartite temporal-text networks.
|
|
58
|
+
|
|
59
|
+
## Minimal config
|
|
60
|
+
|
|
61
|
+
Save this as `my_config.yml`:
|
|
62
|
+
|
|
63
|
+
```yaml
|
|
64
|
+
input:
|
|
65
|
+
format: csv_posts
|
|
66
|
+
posts: my_data.csv
|
|
67
|
+
|
|
68
|
+
name: my_dataset
|
|
69
|
+
|
|
70
|
+
topic_model:
|
|
71
|
+
embedder: all-MiniLM-L6-v2
|
|
72
|
+
min_topic_size: 50
|
|
73
|
+
|
|
74
|
+
network:
|
|
75
|
+
type: multilayer_repost
|
|
76
|
+
|
|
77
|
+
output_dir: ./out/my_dataset
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Run it: `topiclayers run my_config.yml`
|
|
81
|
+
|
|
82
|
+
## Checking progress
|
|
83
|
+
|
|
84
|
+
Long runs write a live status file to `<output_dir>/run_status.json`. Check the current stage, elapsed time, and ETA from another terminal without touching the running job:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
topiclayers status out/my_dataset
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Example output:
|
|
91
|
+
|
|
92
|
+
```
|
|
93
|
+
Stage 3/7: embed
|
|
94
|
+
Status: running
|
|
95
|
+
Run: my_dataset
|
|
96
|
+
Progress: 412000/1200000 (34%)
|
|
97
|
+
Stage elapsed: 18m 12s
|
|
98
|
+
Stage ETA: ~35m 20s
|
|
99
|
+
Total elapsed: 21m 05s
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Stages are `load → data → embed → topic_model → label → network → manifest`. Stages with a per-item counter (data, embed, network) report a live ETA. Opaque stages (BERTopic's UMAP+HDBSCAN) fall back to the median duration of past runs, stored in `<output_dir>/cache/stage_timings.json`.
|
|
103
|
+
|
|
104
|
+
## Topic labeling with an LLM (optional)
|
|
105
|
+
|
|
106
|
+
After clustering, the pipeline can ask a local LLM to write one short, readable
|
|
107
|
+
label per topic (instead of raw keyword lists like `-1_proclamation_plante_trending`).
|
|
108
|
+
Default is **Ollama** — free, local, nothing leaves your machine:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
# one-time setup
|
|
112
|
+
pip install 'topiclayers[labeling]'
|
|
113
|
+
ollama pull qwen2.5:7b
|
|
114
|
+
|
|
115
|
+
# start the server in another terminal
|
|
116
|
+
ollama serve
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Then just run the pipeline as usual (`label_model` is on by default). Any
|
|
120
|
+
[LiteLLM](https://docs.litellm.ai/docs/providers) model string works:
|
|
121
|
+
|
|
122
|
+
```yaml
|
|
123
|
+
topic_model:
|
|
124
|
+
label_model: ollama/qwen2.5:7b # default
|
|
125
|
+
# label_model: gpt-4o-mini # OpenAI (needs OPENAI_API_KEY)
|
|
126
|
+
# label_model: anthropic/claude-3-haiku-20240307
|
|
127
|
+
# label_model: "" # disable labeling
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
### Custom / self-hosted OpenAI-compatible endpoints
|
|
131
|
+
|
|
132
|
+
Any server speaking the OpenAI protocol (Unsloth, llama.cpp, vLLM, LM Studio...)
|
|
133
|
+
works via `label_api_base`:
|
|
134
|
+
|
|
135
|
+
```yaml
|
|
136
|
+
topic_model:
|
|
137
|
+
label_model: openai/unsloth/Qwen3.8-27B-GGUF # note the openai/ prefix
|
|
138
|
+
label_api_base: http://localhost:8888/v1
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
with the token in `.env` (key resolution order: `LABEL_API_KEY` →
|
|
142
|
+
`OPENAI_LIKE_API_KEY` → `OPENAI_API_KEY`). You can also set `LABEL_API_BASE`
|
|
143
|
+
in `.env` instead of the YAML. Find the exact model name your server exposes
|
|
144
|
+
with `curl http://localhost:8888/v1/models`.
|
|
145
|
+
|
|
146
|
+
Labeling is non-fatal: if the model server is unreachable, topics keep their
|
|
147
|
+
keyword labels and everything else proceeds normally.
|
|
148
|
+
|
|
149
|
+
## Input formats
|
|
150
|
+
|
|
151
|
+
| Format | Extension | Description |
|
|
152
|
+
|--------|-----------|-------------|
|
|
153
|
+
| `twitter_jsonl` | `.json`, `.jsonl` | Twitter/X API v2 JSONL |
|
|
154
|
+
| `csv_posts` | `.csv` | Generic CSV with `post_id`, `user_id`, `text` columns |
|
|
155
|
+
| `parquet_posts` | `.parquet` | Same schema as CSV, Parquet format |
|
|
156
|
+
|
|
157
|
+
CSV optional columns: `created_at`, `lang`, `interaction_type` (`original`/`repost`/`quote`/`reply`), `target_post_id`, `mentions` (semicolon-separated), `extra_*` passthrough columns.
|
|
158
|
+
|
|
159
|
+
## What if something fails?
|
|
160
|
+
|
|
161
|
+
| Symptom | Likely cause | Fix |
|
|
162
|
+
|---------|-------------|-----|
|
|
163
|
+
| "All topics are -1" / "no topics found" | `min_topic_size` too high or dataset too small | Halve `min_topic_size` in config, or use a larger dataset (>100 posts) |
|
|
164
|
+
| "Module not found" | topiclayers not installed | `pip install -e .` |
|
|
165
|
+
| "OpenAI API key not set" | Using OpenAI embedder without key | Switch `embedder` to `all-MiniLM-L6-v2` (default, works offline) |
|
|
166
|
+
| "Qdrant connection refused" | Qdrant vector DB not running | Ignore — Qdrant is optional. Set `QDRANT_URL` in `.env` to enable |
|
|
167
|
+
| "Cannot create multilayer network" | All posts are outliers | Reduce `min_topic_size` or provide more data |
|
|
168
|
+
| "UMAP spectral layout failed" | Dataset too small (<10 posts) | Topic modeling needs more data; consider using topic labels from elsewhere |
|
|
169
|
+
|
|
170
|
+
## Output files
|
|
171
|
+
|
|
172
|
+
For a dataset named `<name>` (e.g. `my_dataset`), output goes to `<output_dir>/`:
|
|
173
|
+
|
|
174
|
+
```
|
|
175
|
+
<output_dir>/
|
|
176
|
+
├── run_manifest.json # Reproducibility metadata
|
|
177
|
+
├── run_status.json # Live stage/ETA tracking (while running)
|
|
178
|
+
├── cache/
|
|
179
|
+
│ └── data/
|
|
180
|
+
│ ├── tweets_<name>.pkl/.csv # Full tweet table
|
|
181
|
+
│ ├── retweet_labeled_<name>.pkl/.csv # Retweets with topic labels
|
|
182
|
+
│ └── manifest_<name>.json # Cache validity key
|
|
183
|
+
│ └── stage_timings.json # Per-stage durations for ETA prediction
|
|
184
|
+
└── networks/
|
|
185
|
+
├── <name>_retweet.gml # Single-layer retweet network
|
|
186
|
+
├── <name>_retweet_network_ml.gml # Multilayer (uunet format)
|
|
187
|
+
├── <name>_ttt.gml # Temporal-text bipartite network
|
|
188
|
+
└── projected/
|
|
189
|
+
└── <name>__prj_<topic>.gml # Per-topic projected networks
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
## Requirements
|
|
193
|
+
|
|
194
|
+
- Python 3.12 (uunet, used for multilayer networks, does not ship wheels for 3.13+ yet)
|
|
195
|
+
- Optional: Ollama + `topiclayers[labeling]` extra (LLM topic labeling), Docker (for Qdrant vector search), OpenAI API key (for OpenAI embeddings)
|
|
196
|
+
|
|
197
|
+
## Planned: JOSS software paper
|
|
198
|
+
|
|
199
|
+
Once the API stabilises and the PLOS ONE core paper results are regenerated with this library,
|
|
200
|
+
a short JOSS (Journal of Open Source Software) paper will be submitted with a Zenodo DOI.
|
|
201
|
+
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
topiclayers/__init__.py,sha256=4XbKMp3QGlTC1rsUqiPIyoPTfpf8bNj6ffFWUvxIx8A,88
|
|
2
|
+
topiclayers/adapters/__init__.py,sha256=VxMgBam893e87VG6eiLb9TPOfk7tOnsqRvIOq8biUzQ,221
|
|
3
|
+
topiclayers/adapters/csv.py,sha256=US0EDAKyI3_s4DeE0jlItMoi_UT4QaV0gjCYy2upPKM,3142
|
|
4
|
+
topiclayers/adapters/registry.py,sha256=PApnvL7404j1ZZYvYSPmkvkFyzkhGCbWiv8jkCgMyZM,1361
|
|
5
|
+
topiclayers/adapters/twitter.py,sha256=SD8fa0B_UaPj9MbXoHJaQMjNyPbMovrvT9e8KjWI0-k,2504
|
|
6
|
+
topiclayers/cli.py,sha256=EKC7plJK9T-gOzonhILh3AsqaiQViIQL9ta5e7ODWA8,7053
|
|
7
|
+
topiclayers/config.py,sha256=LvbIslvYxKfaCiE__nR7fmN9m7LBtRVdVn86cEpr7AI,2964
|
|
8
|
+
topiclayers/core/__init__.py,sha256=f4h5d3Gdj8tp0wDQYHXqZd8OMaE-NECYryFq5RAQH7A,781
|
|
9
|
+
topiclayers/core/data.py,sha256=fgwk9leMZa7dHkvDP6URWgPwTW2rrdBSnudFPbLRx_c,8563
|
|
10
|
+
topiclayers/core/network.py,sha256=kd65b5gROJQ6nxN_C63DgR-poiM5crJZtW3Io7eASvs,10130
|
|
11
|
+
topiclayers/core/topic.py,sha256=bCRruIWU89xh06FM_zkNK2lG8lH-N5bacqvduzvq78s,28360
|
|
12
|
+
topiclayers/manifest.py,sha256=rBsPgU3IPvCz5O57NDCwUWn_HYb0a9iNWAqq4dbCw8Q,2485
|
|
13
|
+
topiclayers/schema.py,sha256=Mu2qIq8BkHGmgVsqgnoyD_3v8cuhTwOoFRo3yPlbR14,697
|
|
14
|
+
topiclayers/status.py,sha256=onv7Fm1-aqUk10v8pycSKbV01wOk1WCqbj4-RZAL1TI,8336
|
|
15
|
+
topiclayers/support/__init__.py,sha256=GCJXFQQljeyHJSTt1u6mY7qSsQzPBgBx4mNY5DEY-O8,66
|
|
16
|
+
topiclayers/support/cache.py,sha256=zaup-4QmZElh-twARyPKo-uS0almLu0jJUf6Q7fjfqM,232
|
|
17
|
+
topiclayers/support/repro.py,sha256=Cp_IIGl92IsATJTHSetInbRMx3MAVlyQL8iaq4sN42k,267
|
|
18
|
+
topiclayers-0.2.0.dist-info/METADATA,sha256=Ku9Oj5NbC6U_BHkuBuNrg2_2GUVxJjGO8FTv4JxjaEM,7602
|
|
19
|
+
topiclayers-0.2.0.dist-info/WHEEL,sha256=EGEvSphFYqXKs23-kQBeyNoJP1nrT8ZJKQoi5p5DYL8,88
|
|
20
|
+
topiclayers-0.2.0.dist-info/entry_points.txt,sha256=d2gMl2jDXQpI9Rm_DFRBwpq0GvcYzXODV6YDsq3D55w,52
|
|
21
|
+
topiclayers-0.2.0.dist-info/licenses/LICENSE,sha256=jcGcNA49w2iZWy8w_zBayHfzn4HjDXXf8XxLIDRJt2I,1073
|
|
22
|
+
topiclayers-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 Alessio Gandelli
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|