atomzero 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,108 @@
1
+ """Local neural sentence-embedding reranker (optional candidate).
2
+
3
+ Uses a local `sentence-transformers/all-MiniLM-L6-v2` directory and plain
4
+ transformers/torch; `sentence-transformers` is not required. This is a retrieval
5
+ candidate, not part of the 50M core model.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import threading
10
+ from pathlib import Path
11
+
12
+ import torch
13
+ import torch.nn.functional as F
14
+
15
+ ROOT = Path(__file__).resolve().parents[1]
16
+ DEFAULT_MODEL = ROOT / "models/all-MiniLM-L6-v2"
17
+
18
+
19
+ class NeuralEmbedderError(RuntimeError):
20
+ pass
21
+
22
+
23
+ class NeuralReranker:
24
+ def __init__(self, model_path: str | Path = DEFAULT_MODEL,
25
+ device: str = "cpu", encoder=None, batch_size: int = 32,
26
+ max_length: int = 256):
27
+ self.model_path = Path(model_path)
28
+ self.device = torch.device(device)
29
+ self.batch_size = max(1, int(batch_size))
30
+ self.max_length = max(32, min(int(max_length), 512))
31
+ self._encoder = encoder
32
+ self._tokenizer = None
33
+ self._model = None
34
+ self._lock = threading.RLock()
35
+
36
+ @property
37
+ def cache_key(self) -> str:
38
+ try:
39
+ stamp = self.model_path.stat().st_mtime
40
+ except OSError:
41
+ stamp = 0.0
42
+ return f"neural:{self.model_path.resolve()}:{stamp:.0f}"
43
+
44
+ def _load(self) -> None:
45
+ if self._encoder is not None or self._model is not None:
46
+ return
47
+ if not self.model_path.exists():
48
+ raise NeuralEmbedderError(
49
+ f"embedding model not found: {self.model_path}")
50
+ try:
51
+ from transformers import AutoModel, AutoTokenizer
52
+ except ImportError as exc:
53
+ raise NeuralEmbedderError(
54
+ "transformers is required for neural reranking") from exc
55
+ with self._lock:
56
+ if self._model is not None:
57
+ return
58
+ self._tokenizer = AutoTokenizer.from_pretrained(
59
+ str(self.model_path))
60
+ self._model = AutoModel.from_pretrained(
61
+ str(self.model_path)).to(self.device)
62
+ self._model.eval()
63
+
64
+ def embed(self, texts: list[str]) -> torch.Tensor:
65
+ if self._encoder is not None:
66
+ vectors = self._encoder(texts)
67
+ if not isinstance(vectors, torch.Tensor):
68
+ vectors = torch.tensor(vectors, dtype=torch.float32)
69
+ return F.normalize(vectors.float(), p=2, dim=1)
70
+ self._load()
71
+ rows = []
72
+ with self._lock, torch.inference_mode():
73
+ for start in range(0, len(texts), self.batch_size):
74
+ batch = texts[start:start + self.batch_size]
75
+ encoded = self._tokenizer(
76
+ batch, padding=True, truncation=True,
77
+ max_length=self.max_length, return_tensors="pt")
78
+ encoded = {key: value.to(self.device)
79
+ for key, value in encoded.items()}
80
+ output = self._model(**encoded)
81
+ mask = encoded["attention_mask"].unsqueeze(-1)
82
+ summed = (output.last_hidden_state * mask).sum(dim=1)
83
+ counts = mask.sum(dim=1).clamp(min=1)
84
+ rows.append(F.normalize(summed / counts, p=2, dim=1))
85
+ return torch.cat(rows, dim=0)
86
+
87
+ def rerank(self, query: str, hits: list[dict], k: int = 5) -> list[dict]:
88
+ if not hits:
89
+ return []
90
+ if all("neural_score" in hit for hit in hits):
91
+ scores = [float(hit["neural_score"]) for hit in hits]
92
+ else:
93
+ texts = [
94
+ f"{hit.get('title', '')} {hit.get('section', '')} "
95
+ f"{hit.get('chunk', '')}".strip()
96
+ for hit in hits
97
+ ]
98
+ query_vector = self.embed([str(query or "")])
99
+ doc_vectors = self.embed(texts)
100
+ scores = (doc_vectors @ query_vector.T).flatten().tolist()
101
+ ranked = sorted(enumerate(hits),
102
+ key=lambda item: (-scores[item[0]], item[0]))
103
+ output = []
104
+ for index, hit in ranked[:k]:
105
+ item = dict(hit)
106
+ item["neural_score"] = float(scores[index])
107
+ output.append(item)
108
+ return output
@@ -0,0 +1,330 @@
1
+ """Reviewed research queue: pending -> approved -> Wikipedia -> DocMemory.
2
+
3
+ Only approved topics are fetched. A local scheduler or the HTTP
4
+ ``/v1/research/queue/run`` endpoint can process them without further user
5
+ interaction; rejected or pending items never touch the network.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import copy
10
+ import json
11
+ import re
12
+ import threading
13
+ import time
14
+ from pathlib import Path
15
+
16
+ from atomzero.teacher_notes import add_teacher_note
17
+
18
+
19
+ class ResearchQueueError(ValueError):
20
+ pass
21
+
22
+
23
+ class ItemNotFound(ResearchQueueError):
24
+ pass
25
+
26
+
27
+ def _now() -> str:
28
+ return time.strftime("%Y-%m-%dT%H:%M:%S")
29
+
30
+
31
+ def _clean_topic(topic: str) -> str:
32
+ topic = str(topic or "").strip()
33
+ if not topic:
34
+ raise ResearchQueueError("topic is required")
35
+ if len(topic) > 200:
36
+ raise ResearchQueueError("topic is too long (max 200 chars)")
37
+ if any(ord(ch) < 32 for ch in topic):
38
+ raise ResearchQueueError("topic contains control characters")
39
+ return topic
40
+
41
+
42
+ def _clean_lang(lang: str) -> str:
43
+ lang = str(lang or "en").strip().lower()
44
+ if not re.fullmatch(r"[a-z]{2,10}(-[a-z]{2,10})?", lang):
45
+ raise ResearchQueueError("invalid language code")
46
+ return lang
47
+
48
+
49
+ def _clean_pages(pages) -> int:
50
+ try:
51
+ pages = int(pages)
52
+ except (TypeError, ValueError) as exc:
53
+ raise ResearchQueueError("pages must be an integer") from exc
54
+ if not 1 <= pages <= 5:
55
+ raise ResearchQueueError("pages must be between 1 and 5")
56
+ return pages
57
+
58
+
59
+ def _page_row(page) -> dict:
60
+ page = page if isinstance(page, dict) else {}
61
+ try:
62
+ chunks = int(page.get("chunks", 0))
63
+ except (TypeError, ValueError):
64
+ chunks = 0
65
+ return {"source": str(page.get("source", "")),
66
+ "title": str(page.get("title", "")),
67
+ "chunks": chunks,
68
+ "preview": str(page.get("preview", ""))[:200]}
69
+
70
+
71
+ def _teacher_prompt(topic: str, pages: list) -> str:
72
+ blocks = []
73
+ for index, page in enumerate(pages[:5], 1):
74
+ page = page if isinstance(page, dict) else {}
75
+ context = str(page.get("context") or page.get("preview") or "").strip()
76
+ if not context:
77
+ continue
78
+ blocks.append(
79
+ f"[{index}] {page.get('title', '')}\n"
80
+ f"URL: {page.get('source', '')}\n{context[:4000]}")
81
+ return (
82
+ f"Write a concise factual synthesis about: {topic}\n"
83
+ "Use only the sources below. Do not invent facts. If the sources "
84
+ "disagree or are incomplete, say so.\n\n" + "\n\n".join(blocks))
85
+
86
+
87
+ class ResearchQueue:
88
+ def __init__(self, path: str | Path = "data/research_queue.json"):
89
+ self.path = Path(path)
90
+ self.path.parent.mkdir(parents=True, exist_ok=True)
91
+ self._lock = threading.RLock()
92
+ self.items = self._load()
93
+
94
+ def _load(self) -> list[dict]:
95
+ if not self.path.exists():
96
+ return []
97
+ try:
98
+ data = json.loads(self.path.read_text(encoding="utf-8"))
99
+ except (OSError, json.JSONDecodeError) as exc:
100
+ raise ResearchQueueError(
101
+ f"cannot read research queue: {exc}") from exc
102
+ if not isinstance(data, list):
103
+ raise ResearchQueueError("research queue must be a JSON list")
104
+ return [item for item in data if isinstance(item, dict)]
105
+
106
+ def _reload(self) -> None:
107
+ self.items = self._load()
108
+
109
+ def _save(self) -> None:
110
+ tmp = self.path.with_suffix(self.path.suffix + ".tmp")
111
+ tmp.write_text(
112
+ json.dumps(self.items, indent=2, ensure_ascii=False) + "\n",
113
+ encoding="utf-8")
114
+ tmp.replace(self.path)
115
+
116
+ def _find(self, item_id: str) -> dict:
117
+ item_id = str(item_id or "").strip()
118
+ for item in self.items:
119
+ if item.get("id") == item_id:
120
+ return item
121
+ raise ItemNotFound(f"research item not found: {item_id}")
122
+
123
+ def _next_id(self) -> str:
124
+ nums = []
125
+ for item in self.items:
126
+ match = re.fullmatch(r"r(\d+)", str(item.get("id", "")))
127
+ if match:
128
+ nums.append(int(match.group(1)))
129
+ return f"r{max(nums, default=0) + 1:04d}"
130
+
131
+ def add(self, topic: str, lang: str = "en", pages: int = 2,
132
+ kind: str = "research", origin_id: str = "") -> dict:
133
+ topic = _clean_topic(topic)
134
+ lang = _clean_lang(lang)
135
+ pages = _clean_pages(pages)
136
+ kind = "curiosity" if str(kind).lower() == "curiosity" else "research"
137
+ origin_id = str(origin_id or "")[:64]
138
+ with self._lock:
139
+ self._reload()
140
+ for item in self.items:
141
+ if (item.get("status") in ("pending", "approved", "running")
142
+ and str(item.get("topic", "")).lower() == topic.lower()):
143
+ raise ResearchQueueError("topic is already queued")
144
+ item = {
145
+ "id": self._next_id(),
146
+ "topic": topic,
147
+ "status": "pending",
148
+ "lang": lang,
149
+ "max_pages": pages,
150
+ "kind": kind,
151
+ "origin_id": origin_id,
152
+ "created": _now(),
153
+ "approved_at": "",
154
+ "started_at": "",
155
+ "finished_at": "",
156
+ "sources": [],
157
+ "error": "",
158
+ }
159
+ self.items.append(item)
160
+ self._save()
161
+ return copy.deepcopy(item)
162
+
163
+ def approve(self, item_id: str) -> dict:
164
+ with self._lock:
165
+ self._reload()
166
+ item = self._find(item_id)
167
+ if item.get("status") not in ("pending", "approved"):
168
+ raise ResearchQueueError(
169
+ f"cannot approve a {item.get('status')} item")
170
+ item["status"] = "approved"
171
+ item["approved_at"] = _now()
172
+ item["error"] = ""
173
+ self._save()
174
+ return copy.deepcopy(item)
175
+
176
+ def reject(self, item_id: str) -> dict:
177
+ with self._lock:
178
+ self._reload()
179
+ item = self._find(item_id)
180
+ if item.get("status") not in ("pending", "approved"):
181
+ raise ResearchQueueError(
182
+ f"cannot reject a {item.get('status')} item")
183
+ item["status"] = "rejected"
184
+ item["finished_at"] = _now()
185
+ self._save()
186
+ return copy.deepcopy(item)
187
+
188
+ def list(self) -> list[dict]:
189
+ with self._lock:
190
+ self._reload()
191
+ return copy.deepcopy(self.items)
192
+
193
+ def stats(self) -> dict:
194
+ items = self.list()
195
+ counts: dict[str, int] = {}
196
+ for item in items:
197
+ status = str(item.get("status", "unknown"))
198
+ counts[status] = counts.get(status, 0) + 1
199
+ return {"total": len(items), **counts}
200
+
201
+ def recover_stale_running(self, stale_seconds: int = 21600) -> list[str]:
202
+ """Return interrupted ``running`` items to ``approved`` for a retry."""
203
+ cutoff = time.time() - max(60, int(stale_seconds))
204
+ recovered: list[str] = []
205
+ with self._lock:
206
+ self._reload()
207
+ for item in self.items:
208
+ if item.get("status") != "running":
209
+ continue
210
+ try:
211
+ started = time.mktime(time.strptime(
212
+ str(item.get("started_at", "")),
213
+ "%Y-%m-%dT%H:%M:%S"))
214
+ except (TypeError, ValueError, OverflowError):
215
+ started = 0.0
216
+ if started <= cutoff:
217
+ item["status"] = "approved"
218
+ item["started_at"] = ""
219
+ item["error"] = "recovered stale running item"
220
+ recovered.append(str(item.get("id", "")))
221
+ if recovered:
222
+ self._save()
223
+ return recovered
224
+
225
+ def run_approved(self, docs, limit: int = 1, fetcher=None,
226
+ teacher=None) -> list[dict]:
227
+ """Fetch approved topics into DocMemory; pending items never run.
228
+
229
+ If ``teacher`` is supplied, its synthesis of the fetched Wikipedia
230
+ context is stored as an unverified teacher note.
231
+ """
232
+ if docs is None:
233
+ raise ResearchQueueError("no document memory configured")
234
+ if fetcher is None:
235
+ from atomzero.web_research import wikipedia_research
236
+ fetcher = wikipedia_research
237
+ try:
238
+ limit = max(1, min(int(limit), 5))
239
+ except (TypeError, ValueError) as exc:
240
+ raise ResearchQueueError("limit must be an integer") from exc
241
+
242
+ selected: list[str] = []
243
+ with self._lock:
244
+ self._reload()
245
+ for item in self.items:
246
+ if (item.get("status") == "approved"
247
+ and len(selected) < limit):
248
+ item["status"] = "running"
249
+ item["started_at"] = _now()
250
+ item["error"] = ""
251
+ selected.append(str(item["id"]))
252
+ self._save()
253
+
254
+ results: list[dict] = []
255
+ for item_id in selected:
256
+ with self._lock:
257
+ self._reload()
258
+ item = self._find(item_id)
259
+ topic = str(item.get("topic", ""))
260
+ lang = str(item.get("lang", "en"))
261
+ pages = int(item.get("max_pages", 2))
262
+ try:
263
+ found = fetcher(docs, topic, lang, pages)
264
+ except Exception as exc:
265
+ with self._lock:
266
+ self._reload()
267
+ item = self._find(item_id)
268
+ item["status"] = "error"
269
+ item["error"] = str(exc)[:500]
270
+ item["finished_at"] = _now()
271
+ item["sources"] = []
272
+ self._save()
273
+ results.append(copy.deepcopy(item))
274
+ else:
275
+ teacher_result = None
276
+ if teacher is not None:
277
+ label = str(getattr(teacher, "label", "provider"))
278
+ model = str(getattr(teacher, "model", ""))
279
+ if not found:
280
+ teacher_result = {
281
+ "label": label, "model": model, "source": "",
282
+ "title": "", "verified": False,
283
+ "answer_preview": "",
284
+ "error": "no source pages to synthesize",
285
+ }
286
+ else:
287
+ try:
288
+ prompt = _teacher_prompt(topic, found)
289
+ answer = teacher.ask(
290
+ prompt,
291
+ system=("You are a concise research assistant. "
292
+ "Use only the supplied sources. "
293
+ "State uncertainty."),
294
+ max_tokens=512)
295
+ source_url = next(
296
+ (str(page.get("source", ""))
297
+ for page in found
298
+ if str(page.get("source", "")).startswith(
299
+ ("http://", "https://"))), "")
300
+ note = add_teacher_note(
301
+ docs, label, model, topic, answer,
302
+ source_url)
303
+ teacher_result = {
304
+ "label": label, "model": model,
305
+ "source": note["source"],
306
+ "title": note["title"],
307
+ "verified": False,
308
+ "answer_preview": answer[:500],
309
+ "error": "",
310
+ }
311
+ except Exception as exc:
312
+ teacher_result = {
313
+ "label": label, "model": model, "source": "",
314
+ "title": "", "verified": False,
315
+ "answer_preview": "",
316
+ "error": str(exc)[:500],
317
+ }
318
+ with self._lock:
319
+ self._reload()
320
+ item = self._find(item_id)
321
+ item["status"] = "done"
322
+ item["finished_at"] = _now()
323
+ item["error"] = ""
324
+ item["sources"] = [
325
+ _page_row(page) for page in (found or [])]
326
+ if teacher is not None:
327
+ item["teacher"] = teacher_result or {}
328
+ self._save()
329
+ results.append(copy.deepcopy(item))
330
+ return results
atomzero/safety.py ADDED
@@ -0,0 +1,226 @@
1
+ """Opt-in safety filter for Atom Zero.
2
+
3
+ The filter is intentionally narrow and conservative: it targets explicit
4
+ requests for harmful instructions, not discussion, prevention or news. It is a
5
+ guardrail, not a complete moderation system. Enable it for hosted or shared
6
+ deployments; local research defaults stay unchanged.
7
+
8
+ filter = SafetyFilter()
9
+ verdict = filter.check("how to make a bomb") # {"category": "violence", ...}
10
+ text, verdict = filter.filter_reply(model_output)
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import re
15
+
16
+ VERSION = "0.2"
17
+
18
+ REFUSALS = {
19
+ "sexual_minors": "I can't help with sexual content involving minors.",
20
+ "self_harm": ("I can't help with instructions for self-harm. If you are in "
21
+ "immediate danger, contact local emergency services or a "
22
+ "crisis line. I can help with coping or finding support."),
23
+ "violence": "I can't help with instructions for violence or weapons.",
24
+ "hate": "I can't help with hateful or violent content targeting a group.",
25
+ "harassment": "I can't help with harassment, doxxing or blackmail.",
26
+ "illegal": ("I can't help with illegal instructions. I can help with harm "
27
+ "reduction or lawful alternatives."),
28
+ "privacy": ("I can't help with finding or exposing someone's private "
29
+ "information."),
30
+ "malware": "I can't help with malware or unauthorized access.",
31
+ "generic": ("I can't help with that request. I can help with a safe, "
32
+ "lawful alternative."),
33
+ }
34
+
35
+ # Narrow patterns. Each one needs an explicit harmful target or instruction;
36
+ # topic words alone (suicide prevention, child abuse reporting, genocide
37
+ # history) must not match.
38
+ _PATTERNS = {
39
+ "sexual_minors": [
40
+ r"\b(child|minor|underage)\s+(porn|pornography|nudes?|sex|erotica)\b",
41
+ r"\b(porn|nudes?|sex|erotica)\s+(with|involving)\s+(a\s+)?"
42
+ r"(child|minor|underage)\b",
43
+ r"\b(sexual|explicit)\s+(story|content|roleplay|fantasy)\b"
44
+ r".{0,30}\b(child|minor|underage)\b",
45
+ r"\bcsam\b",
46
+ ],
47
+ "self_harm": [
48
+ r"\bhow (do|can) i (kill|hurt|cut|harm) myself\b",
49
+ r"\bhow to (kill|hurt|cut|harm) (myself|yourself)\b",
50
+ r"\b(suicide|self[- ]harm)\s+(method|methods|instruction|instructions)\b",
51
+ r"\bhow to (commit|die by) suicide\b",
52
+ r"\bwie\s+bringe\s+ich\s+mich\s+um\b",
53
+ ],
54
+ "violence": [
55
+ r"\bhow to (make|build|construct)\b.{0,40}\b(bomb|explosive|gun|weapon)\b",
56
+ r"\b(bomb|explosive)\b.{0,30}\b(recipe|instructions|how to make)\b",
57
+ r"\b(shoot|attack)\s+(up|a)\s+(school|church|mosque|synagogue|mall)\b",
58
+ r"\bhow to (kill|murder|stab|shoot)\b",
59
+ r"\bpoison\s+(someone|a person|people)\b",
60
+ r"\bmass shooting plan\b",
61
+ r"\b(plan|planning)\s+(a\s+)?mass shooting\b",
62
+ ],
63
+ "hate": [
64
+ r"\b(kill|exterminate|gas|lynch)\s+(all\s+)?"
65
+ r"(jews|muslims|christians|blacks|whites|asians|gays|trans|"
66
+ r"immigrants|women)\b",
67
+ r"\b(ethnic cleansing|genocide)\b.{0,30}\b(how|plan|support|justify)\b",
68
+ r"\b(how|why|justify|support)\b.{0,30}\b(ethnic cleansing|genocide)\b",
69
+ ],
70
+ "harassment": [
71
+ r"\b(dox|doxx|swat)\s+(someone|him|her|them|a person)\b",
72
+ r"\bblackmail\s+(someone|him|her|them)\b",
73
+ r"\bsextortion\b",
74
+ r"\bhow to (stalk|harass|threaten)\b",
75
+ ],
76
+ "illegal": [
77
+ r"\bhow to (make|synthesize|cook)\s+"
78
+ r"(meth|methamphetamine|fentanyl|heroin|cocaine|explosives)\b",
79
+ r"\bhow to (launder|launder money)\b",
80
+ r"\bcredit card (fraud|skimming)\b",
81
+ r"\bhow to (steal|hotwire)\s+(a car|a credit card|money)\b",
82
+ r"\bbuy (drugs|weapons) online\b",
83
+ ],
84
+ "privacy": [
85
+ r"\bhow to (find|get)\s+someone'?s\s+"
86
+ r"(address|ssn|social security|phone number)\b",
87
+ r"\b(dox|doxx)\s+(someone|him|her|them|a person)\b",
88
+ ],
89
+ "malware": [
90
+ r"\bhow to (write|create|build)\s+"
91
+ r"(ransomware|malware|a virus|a trojan)\b",
92
+ r"\binstall\s+(a\s+)?keylogger\b",
93
+ ],
94
+ }
95
+
96
+ _COMPILED = {
97
+ category: [re.compile(pattern, re.IGNORECASE) for pattern in patterns]
98
+ for category, patterns in _PATTERNS.items()
99
+ }
100
+
101
+ # Language-agnostic, order-independent intent rules. These are still lexical
102
+ # rules (not an ML classifier), but they catch paraphrases and translated
103
+ # requests that the English regexes miss.
104
+ _INSTRUCTION = ("how", "steps", "explain", "instructions", "nasıl", "nasil",
105
+ "wie", "comment", "como", "cómo", "anleitung", "instrucciones",
106
+ "talimat", "erklär", "come", "hoe", "как", "walk", "guide",
107
+ "show", "recipe")
108
+ _BUILD = ("make", "build", "construct", "yap", "bau", "fabriqu", "hac",
109
+ "constru", "herstell", "costru", "faz", "maak", "сдела", "assembl",
110
+ "recipe", "makin")
111
+ _WEAPON = ("bomb", "explos", "patlay", "spreng", "gun", "weapon", "silah",
112
+ "arma", "bom", "бомб")
113
+ _SELF = ("myself", "yourself", "kendim", "kendin", "mich", "mir", "me",
114
+ "moi", "mi", "себя")
115
+ _KILL = ("kill", "hurt", "cut", "harm", "öldür", "oldur", "umbring", "matar",
116
+ "suicid", "tuer", "uccid", "уби")
117
+ _SUICIDE = ("suicid", "intihar", "selbstmord")
118
+ _CHILD = ("child", "minor", "underage", "çocuk", "cocuk", "kind", "menor",
119
+ "mineur", "enfant", "infant", "дет")
120
+ _SEXUAL = ("sex", "porn", "nude", "erotic", "cinsel", "seks", "porno",
121
+ "sexuel", "sexual")
122
+ _BENIGN_SEXUAL = ("abuse", "report", "prevent", "protect", "istismar",
123
+ "ihbar", "koru", "missbrauch", "melden", "abuso",
124
+ "denunci", "preven", "schutz", "segnal", "minori")
125
+ _BENIGN_SELF = ("help", "friend", "support", "prevent", "warning", "signs",
126
+ "destek", "yardım", "arkadaş", "önle", "belirti", "helfen",
127
+ "freund", "verhindern", "anzeichen", "ayudar", "amigo",
128
+ "preven", "señales", "aider", "ami", "prévenir", "signes",
129
+ "aiut", "amico", "segnal", "помощ", "помоч", "друг",
130
+ "предупред")
131
+ _HATE_GROUP = ("jews", "muslims", "christians", "immigrants", "women", "gays",
132
+ "trans", "mülteci", "multeci", "muslime", "juden", "inmigrant",
133
+ "mujer", "femme", "kadın", "kadin", "frau", "immigrat",
134
+ "migrant")
135
+ _HATE_ACTION = ("kill", "extermin", "gas", "lynch", "öldür", "oldur",
136
+ "matar", "töt", "yok", "uccid")
137
+ _HARASS_ACTION = ("dox", "swat", "blackmail", "sextort", "şantaj", "santaj",
138
+ "chantaj", "erpress", "ricatt")
139
+ _HARASS_TARGET = ("someone", "him", "her", "them", "person", "birini",
140
+ "birine", "jemand", "alguien", "qualcuno", "quelqu")
141
+ _DRUG = ("meth", "fentanyl", "heroin", "cocaine", "metamfetamin",
142
+ "uyuşturucu", "uyusturucu", "drogen", "droga", "drogue",
143
+ "metanfetamina", "метамф")
144
+ _ILLEGAL_ACTION = ("make", "synthes", "cook", "yap", "herstell", "fabriqu",
145
+ "hac", "buy", "satın", "kauf", "comprar", "produr",
146
+ "сдела")
147
+ _PRIVATE_ACTION = ("find", "get", "bul", "finden", "encontr", "trouv",
148
+ "trovar", "найд")
149
+ _PRIVATE_TARGET = ("address", "ssn", "phone", "adres", "adresse",
150
+ "dirección", "direccion", "telefon", "password",
151
+ "contraseña", "contrase", "indirizz", "адрес")
152
+ _MALWARE = ("ransomware", "malware", "virus", "trojan", "fidye",
153
+ "schadsoftware", "malveillant")
154
+ _MALWARE_ACTION = ("write", "create", "build", "install", "yaz", "kur",
155
+ "schreib", "écri", "crear", "instalar", "scriv", "напиш")
156
+
157
+ _SEMANTIC_RULES = [
158
+ {"category": "sexual_minors", "all": (_CHILD, _SEXUAL),
159
+ "none": _BENIGN_SEXUAL},
160
+ {"category": "self_harm", "all": (_INSTRUCTION, _SUICIDE),
161
+ "none": _BENIGN_SELF},
162
+ {"category": "self_harm", "all": (_INSTRUCTION, _KILL, _SELF),
163
+ "none": _BENIGN_SELF},
164
+ {"category": "violence", "all": (_INSTRUCTION, _BUILD, _WEAPON)},
165
+ {"category": "hate", "all": (_HATE_GROUP, _HATE_ACTION)},
166
+ {"category": "harassment", "all": (_HARASS_ACTION, _HARASS_TARGET)},
167
+ {"category": "illegal", "all": (_INSTRUCTION, _DRUG, _ILLEGAL_ACTION)},
168
+ {"category": "privacy", "all": (_PRIVATE_ACTION, _PRIVATE_TARGET)},
169
+ {"category": "malware", "all": (_MALWARE, _MALWARE_ACTION,
170
+ _INSTRUCTION)},
171
+ ]
172
+
173
+
174
+ def _tokens(value: str) -> list[str]:
175
+ normalized = str(value or "").replace("İ", "i").replace("I", "ı")
176
+ return re.findall(r"[^\W_]+", normalized.lower(), re.UNICODE)
177
+
178
+
179
+ def _has(tokens: list[str], stems) -> bool:
180
+ return any(token.startswith(stem) for token in tokens
181
+ for stem in stems)
182
+
183
+
184
+ def _semantic_check(value: str) -> dict | None:
185
+ tokens = _tokens(value)
186
+ if not tokens:
187
+ return None
188
+ for rule in _SEMANTIC_RULES:
189
+ if not all(_has(tokens, stems) for stems in rule["all"]):
190
+ continue
191
+ if "none" in rule and _has(tokens, rule["none"]):
192
+ continue
193
+ return {
194
+ "category": rule["category"],
195
+ "action": "refuse",
196
+ "match": "semantic:" + rule["category"],
197
+ }
198
+ return None
199
+
200
+
201
+ class SafetyFilter:
202
+ version = VERSION
203
+
204
+ def check(self, text: str) -> dict | None:
205
+ value = str(text or "")
206
+ if not value.strip():
207
+ return None
208
+ for category, patterns in _COMPILED.items():
209
+ for pattern in patterns:
210
+ match = pattern.search(value)
211
+ if match:
212
+ return {
213
+ "category": category,
214
+ "action": "refuse",
215
+ "match": match.group(0)[:80],
216
+ }
217
+ return _semantic_check(value)
218
+
219
+ def refusal(self, category: str) -> str:
220
+ return REFUSALS.get(category, REFUSALS["generic"])
221
+
222
+ def filter_reply(self, text: str) -> tuple[str, dict | None]:
223
+ verdict = self.check(text)
224
+ if verdict is None:
225
+ return text, None
226
+ return self.refusal(verdict["category"]), verdict