kd-engine 1.0.0__py3-none-any.whl

ytk/dedup/service.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+
+from ytk.models.content import KnowledgeAtomV2
+from ytk.models.knowledge import KnowledgeAtom
+
+
+def _normalize_content(text: str) -> str:
+    return re.sub(r"[^\w\s]", "", text.casefold().strip())
+
+
+#: Negation markers.  Two atoms whose content differs only by one of these must
+#: never be merged (T41.1: "保留否定、条件和相反结论").
+NEGATION_TOKENS: frozenset[str] = frozenset(
+    {
+        "not",
+        "no",
+        "never",
+        "none",
+        "without",
+        "avoid",
+        "cannot",
+        "cant",
+        "can't",
+        "dont",
+        "don't",
+        "doesnt",
+        "doesn't",
+        "isnt",
+        "isn't",
+        "arent",
+        "aren't",
+        "wont",
+        "won't",
+        "shouldnt",
+        "shouldn't",
+        "mustnt",
+        "mustn't",
+        "禁止",
+        "不要",
+        "不能",
+        "不得",
+        "无需",
+        "避免",
+        "不可",
+        "严禁",
+    }
+)
+
+#: Condition / precondition markers.  "Do X" and "Do X only if Y" are
+#: different conclusions even when the action text matches.
+CONDITION_TOKENS: frozenset[str] = frozenset(
+    {
+        "if",
+        "unless",
+        "when",
+        "whenever",
+        "only",
+        "before",
+        "after",
+        "provided",
+        "otherwise",
+        "else",
+        "如果",
+        "只有",
+        "除非",
+        "当",
+        "之前",
+        "之后",
+        "否则",
+        "前提",
+    }
+)
+
+_WORD_RE = re.compile(r"[a-z][a-z'\u2019_-]*")
+_CJK_RE = re.compile(r"[\u4e00-\u9fff]+")
+
+
+def _qualifier_tokens(text: str, vocabulary: frozenset[str]) -> frozenset[str]:
+    """Tokens from ``vocabulary`` present in ``text`` (latin words + CJK runs)."""
+    lowered = str(text).casefold()
+    words = set(_WORD_RE.findall(lowered))
+    tokens = {entry for entry in vocabulary if entry in words}
+    cjk_entries = [entry for entry in vocabulary if _CJK_RE.search(entry)]
+    if cjk_entries:
+        runs = _CJK_RE.findall(lowered)
+        tokens.update(
+            entry for entry in cjk_entries if any(entry in run for run in runs)
+        )
+    return frozenset(tokens)
+
+
+def _ordered_union(left: Iterable[str], right: Iterable[str]) -> list[str]:
+    merged: list[str] = []
+    for value in (*left, *right):
+        if value not in merged:
+            merged.append(value)
+    return merged
+
+
+class DeduplicationService:
+    def __init__(
+        self,
+        embedding_client: object | None = None,
+        semantic_threshold: float = 0.85,
+    ) -> None:
+        self.embedding_client = embedding_client
+        self.semantic_threshold = semantic_threshold
+
+    def merge_v2(self, atoms: list[KnowledgeAtomV2]) -> list[KnowledgeAtomV2]:
+        """Merge v2 atoms by type + normalized content (+ qualifier signature).
+
+        A merge unions ``source_ids``/``evidence_ids``/``topic_path`` and keeps
+        the **maximum** confidence of the inputs.  Confidence is never raised
+        because more sources carry the same claim (a repost is not evidence),
+        and atoms that differ in negation or condition never merge.
+        """
+        merged: dict[tuple[object, ...], KnowledgeAtomV2] = {}
+        for atom in atoms:
+            key: tuple[object, ...] = (
+                atom.atom_type,
+                _normalize_content(atom.content),
+                _qualifier_tokens(atom.content, NEGATION_TOKENS),
+                _qualifier_tokens(atom.content, CONDITION_TOKENS),
+            )
+            if atom.atom_type == "core_concept":
+                # Concept content is often only a name. Its definition carries
+                # the claim, including negation, conditions and numeric values.
+                key += (" ".join(atom.explanation.casefold().split()),)
+            current = merged.get(key)
+            if current is None:
+                merged[key] = atom
+                continue
+            merged[key] = current.model_copy(
+                update={
+                    "source_ids": _ordered_union(current.source_ids, atom.source_ids),
+                    "evidence_ids": _ordered_union(current.evidence_ids, atom.evidence_ids),
+                    "topic_path": _ordered_union(current.topic_path, atom.topic_path),
+                    # max of the *individual* judgments; not count-boosted
+                    "confidence": max(current.confidence, atom.confidence),
+                    "metadata": {**atom.metadata, **current.metadata},
+                }
+            )
+        return list(merged.values())
+
+    def merge(self, atoms: list[KnowledgeAtom]) -> list[KnowledgeAtom]:
+        merged: dict[tuple[str, ...], KnowledgeAtom] = {}
+        for atom in atoms:
+            key: tuple[str, ...] = (atom.atom_type, _normalize_content(atom.content))
+            if atom.atom_type == "core_concept":
+                key += (" ".join(atom.explanation.casefold().split()),)
+            if key not in merged:
+                merged[key] = atom
+                continue
+
+            current = merged[key]
+            merged[key] = current.model_copy(
+                update={
+                    "timestamps": sorted(set(current.timestamps + atom.timestamps)),
+                    "evidence": current.evidence + atom.evidence,
+                    "confidence": max(current.confidence, atom.confidence),
+                }
+            )
+        return list(merged.values())