orbitkit 0.9.22__tar.gz → 0.9.23__tar.gz

{orbitkit-0.9.22/orbitkit.egg-info → orbitkit-0.9.23}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.2
 Name: orbitkit
-Version: 0.9.22
+Version: 0.9.23
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao
@@ -45,7 +45,6 @@ Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: license
-Dynamic: license-file
 Dynamic: maintainer
 Dynamic: maintainer-email
 Dynamic: platform

orbitkit-0.9.23/orbitkit/VERSION

@@ -0,0 +1 @@
+0.9.23

{orbitkit-0.9.22 → orbitkit-0.9.23}/orbitkit/pdf_validator/filters/repeated_text.py

@@ -32,6 +32,14 @@ VLM/OCR 偶发在一个 block 里把某个短语连续复读很多次，例如::
 filter**，因此质量闸门的复读退化率始终反映真实情况。它同时给出每个退化块的
 **命中详情**（复读短语 / 重复次数 / 文本片段），进 ``x_parse_record`` 后
 「退化率 0.054 该不该判死」才成为可复核的判断，而不是只有一个数字。
+
+**不算复读的重复**（表格 / 图表 markdown 里成行出现是常态，不是模型退化）：
+
+1. markdown 表格分隔行 ``| --- | --- | --- |``——语法而非内容，整行不参与判定；
+2. 占位符 / 空值 token（``-`` ``—`` ``na`` ``N/A`` ``none`` ``|`` 纯标点等）
+   对检测**透明**：既不计入短语，也不打断短语；
+3. 复读短语必须含至少一个**实词**（≥2 个字母 / 汉字），纯数字、纯符号的
+   重复（``0 0 0 0``、``... ... ...``）不算。
 """
 
 from __future__ import annotations
@@ -57,6 +65,21 @@ _MIN_BLOCK_WORDS = 4
 
 _WORD_RE = re.compile(r"\S+")
 
+# markdown 表格分隔行：| --- | :---: | ---: |（也允许无首尾竖线）
+_MD_SEPARATOR_LINE_RE = re.compile(r"^\s*\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)*\|?\s*$")
+
+# 占位符 / 空值 token（小写比较）：表格里成行出现是常态，对复读检测透明
+_PLACEHOLDER_TOKENS = frozenset(
+    {
+        "na", "n/a", "n.a.", "n.a", "nan", "none", "non", "null", "nil",
+        "nm", "n.m.", "tbd", "tba", "n/d", "nd", "无", "暂无", "不适用",
+        "0", "0.0", "0.00", "0%", "0.0%", "0.00%",
+    }
+)
+
+# 实词：至少 2 个字母 / 汉字（含带重音的拉丁字母、日文、韩文等）
+_REAL_WORD_RE = re.compile(r"[^\W\d_]{2,}")
+
 # 样本片段截断长度：只为让人判断「是真复读还是页眉/表头」，不做内容留存
 _SAMPLE_CHARS = 160
 
@@ -78,6 +101,38 @@ def _tokenize(text: str) -> Tuple[List[str], List[Tuple[int, int]]]:
     return words, spans
 
 
+def _is_placeholder(word: str) -> bool:
+    """占位符 / 空值 / 纯标点 token：对复读检测透明。"""
+    if not any(ch.isalnum() for ch in word):
+        return True  # 纯符号：| - — … / * ○ √ × ...
+    return word.lower() in _PLACEHOLDER_TOKENS
+
+
+def _content_tokens(text: str) -> Tuple[List[str], List[Tuple[int, int]]]:
+    """复读检测用的词序列：剔除 markdown 分隔行与占位符 token。
+
+    返回的 spans 仍是**原串**字符区间，trim 据此在原串上挖段；被剔除的 token
+    不出现在序列里，因此既不计入短语也不打断短语。
+    """
+    words: List[str] = []
+    spans: List[Tuple[int, int]] = []
+    offset = 0
+    for line in text.splitlines(keepends=True):
+        if not _MD_SEPARATOR_LINE_RE.match(line):
+            for m in _WORD_RE.finditer(line):
+                word = m.group(0)
+                if _is_placeholder(word):
+                    continue
+                words.append(word)
+                spans.append((offset + m.start(), offset + m.end()))
+        offset += len(line)
+    return words, spans
+
+
+def _has_real_word(phrase: Sequence[str]) -> bool:
+    return any(_REAL_WORD_RE.search(w) for w in phrase)
+
+
 def _find_runaway_repeats(
     words: List[str], *, max_n: int, min_reps: int
 ) -> List[Tuple[int, int, int]]:
@@ -99,7 +154,7 @@ def _find_runaway_repeats(
                 while j + n <= n_words and words[j : j + n] == words[i : i + n]:
                     reps += 1
                     j += n
-                if reps >= min_reps:
+                if reps >= min_reps and _has_real_word(words[i : i + n]):
                     candidates.append((reps * n, i, n))
                     reps_of[(i, n)] = reps
                 i = j  # 跳过整段，避免重复检测子段
@@ -139,7 +194,7 @@ def has_runaway_repetition(
     只判定、不改文本，供统计「退化块占比」用。短于 ``min_words`` 的文本一律
     返回 ``False``（真实短句可能有正常重复）。
     """
-    words, _ = _tokenize(text or "")
+    words, _ = _content_tokens(text or "")
     if len(words) < min_words:
         return False
     return bool(_find_runaway_repeats(words, max_n=max_n, min_reps=min_reps))
@@ -215,7 +270,7 @@ def find_degenerate_blocks(blocks: Sequence[dict]) -> Dict[int, Sample]:
     found: Dict[int, Sample] = {}
     for i, b in enumerate(blocks):
         text = b.get("sentence") or ""
-        words, spans = _tokenize(text)
+        words, spans = _content_tokens(text)
         if len(words) < _MIN_WORDS:
             continue
         run = _find_runaway_repeat(words, max_n=_MAX_N, min_reps=_MIN_REPS)
@@ -297,7 +352,7 @@ def trim_repeated_text(block: dict) -> Optional[dict]:
     （可能是合法用词），删掉其余重复。
     """
     text = block.get("sentence") or ""
-    words, spans = _tokenize(text)
+    words, spans = _content_tokens(text)
     if len(words) < _MIN_WORDS:
         return block
 
@@ -321,7 +376,7 @@ def trim_repeated_text(block: dict) -> Optional[dict]:
 def drop_repeated_text(block: dict) -> Optional[dict]:
     """复读段主导整块（覆盖词数占比 ≥ 阈值）时整块丢弃，否则保留。"""
     text = block.get("sentence") or ""
-    words, _spans = _tokenize(text)
+    words, _spans = _content_tokens(text)
     if len(words) < _MIN_WORDS:
         return block
 
@@ -332,4 +387,4 @@ def drop_repeated_text(block: dict) -> Optional[dict]:
     covered = sum(n * reps for _start, n, reps in runs)
     if covered / len(words) >= _DROP_COVER_FRACTION:
         return None
-    return block
+    return block

{orbitkit-0.9.22 → orbitkit-0.9.23/orbitkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.2
 Name: orbitkit
-Version: 0.9.22
+Version: 0.9.23
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao
@@ -45,7 +45,6 @@ Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: license
-Dynamic: license-file
 Dynamic: maintainer
 Dynamic: maintainer-email
 Dynamic: platform

orbitkit-0.9.22/orbitkit/VERSION

@@ -1 +0,0 @@
-0.9.22