sieve-referee 1.0.0__py3-none-any.whl

sieve_referee/__init__.py

@@ -0,0 +1,11 @@
+"""
+sieve_referee
+~~~~~~~~~~~~~
+Deterministic, zero-dependency plagiarism and text structure screening engine.
+"""
+from .referee import SieveReferee
+from .evaluator import batch_evaluate
+from .models import EvaluationResult, SignalColor
+
+__version__ = "1.0.0"
+__all__ = ["SieveReferee", "batch_evaluate", "EvaluationResult", "SignalColor", "__version__"]

sieve_referee/blocker.py

@@ -0,0 +1,39 @@
+from typing import Dict, Set, Tuple
+from collections import defaultdict
+from sieve_referee.tokenizer import tokenize
+
+
+class DeterministicBlocker:
+    def __init__(self, ngram_size: int = 3, min_shared_ngrams: int = 1):
+        self.ngram_size = ngram_size
+        self.min_shared_ngrams = min_shared_ngrams
+
+    def _extract_ngrams(self, text: str) -> Set[str]:
+        tokens = [t.raw_text for t in tokenize(text)]
+        if len(tokens) < self.ngram_size:
+            return set(tokens) if tokens else {text}
+        return {"".join(tokens[i : i + self.ngram_size]) for i in range(len(tokens) - self.ngram_size + 1)}
+
+    def get_candidate_pairs(self, docs: Dict[str, str]) -> Set[Tuple[str, str]]:
+        doc_ids = sorted(docs.keys())
+        if len(doc_ids) < 10:
+            candidates = set()
+            for i in range(len(doc_ids)):
+                for j in range(i + 1, len(doc_ids)):
+                    candidates.add((doc_ids[i], doc_ids[j]))
+            return candidates
+        inverted_index: Dict[str, Set[str]] = defaultdict(set)
+        for doc_id in doc_ids:
+            ngrams = self._extract_ngrams(docs[doc_id])
+            for ngram in ngrams:
+                inverted_index[ngram].add(doc_id)
+        shared_counts = defaultdict(int)
+        for ngram, sharing_docs in inverted_index.items():
+            if len(sharing_docs) < 2:
+                continue
+            sorted_sharing = sorted(sharing_docs)
+            for i in range(len(sorted_sharing)):
+                for j in range(i + 1, len(sorted_sharing)):
+                    pair = (sorted_sharing[i], sorted_sharing[j])
+                    shared_counts[pair] += 1
+        return {pair for pair, count in shared_counts.items() if count >= self.min_shared_ngrams}

sieve_referee/chain_detector.py

@@ -0,0 +1,29 @@
+from typing import Dict, List, Set, Tuple
+from .models import EvaluationResult
+
+
+def detect_indirect_chains(results: List[EvaluationResult], clusters: Dict[str, List[str]]) -> List[Dict]:
+    direct_pairs: Set[Tuple[str, str]] = set()
+    for res in results:
+        sig = getattr(res, "evaluation_signal", "GREEN")
+        sig_val = sig.value if hasattr(sig, "value") else str(sig)
+        if sig_val in ["RED", "YELLOW"]:
+            u, v = sorted([res.item_id, res.matched_peer_id])
+            direct_pairs.add((u, v))
+
+    seen_clusters: Set[Tuple[str, ...]] = set()
+    unique_clusters: List[List[str]] = []
+    for members in clusters.values():
+        key = tuple(sorted(members))
+        if len(key) >= 3 and key not in seen_clusters:
+            seen_clusters.add(key)
+            unique_clusters.append(list(key))
+
+    indirect_chains: List[Dict] = []
+    for cluster in unique_clusters:
+        for i in range(len(cluster)):
+            for j in range(i + 1, len(cluster)):
+                u, v = sorted([cluster[i], cluster[j]])
+                if (u, v) not in direct_pairs:
+                    indirect_chains.append({"item_id_a": u, "item_id_b": v, "cluster_members": cluster})
+    return indirect_chains

sieve_referee/chunker.py

@@ -0,0 +1,15 @@
+import re
+from typing import List
+from .models import Chunk
+
+
+def split_into_multi_scale_windows(text: str, window_sizes: List[int] = [1, 2]) -> List[Chunk]:
+    sentences = [s.strip() + "。" for s in re.split(r"正?。|！|？|\n", text) if s.strip()]
+    chunks = []
+    chunk_id = 0
+    for w_size in window_sizes:
+        for i in range(len(sentences) - w_size + 1):
+            window = sentences[i : i + w_size]
+            chunks.append(Chunk(chunk_id=chunk_id, text="".join(window), window_size=w_size))
+            chunk_id += 1
+    return chunks

sieve_referee/cli.py

@@ -0,0 +1,103 @@
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import List
+
+from .chain_detector import detect_indirect_chains
+from .dashboard_builder import generate_dashboard_html, generate_standalone_dashboard_html
+from .diff_builder import generate_diff_html
+from .evaluator import batch_evaluate, build_clusters
+from .io_utils import safe_read_text
+from .models import EvaluationResult
+
+
+def result_to_dict(res: EvaluationResult) -> dict:
+    signal_str = res.evaluation_signal.value if hasattr(res.evaluation_signal, "value") else str(res.evaluation_signal)
+    return {
+        "item_id": res.item_id,
+        "matched_peer_id": res.matched_peer_id,
+        "signal": signal_str,
+        "mask": res.mask,
+        "pattern_name": res.pattern_name,
+        "reason": res.reason,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description="sieve_referee: Deterministic Text Plagiarism Screening Engine")
+    parser.add_argument("target_dir", type=str, help="Target directory containing .txt files")
+    parser.add_argument("--output-dir", type=str, default="reports")
+    parser.add_argument("--json", action="store_true")
+    parser.add_argument("--json-file", type=str, default=None)
+    parser.add_argument("--fail-on-risk", action="store_true")
+    parser.add_argument("-j", "--jobs", type=int, default=None)
+    parser.add_argument("--standalone", action="store_true")
+    args = parser.parse_args()
+
+    target_path = Path(args.target_dir)
+    if not target_path.exists() or not target_path.is_dir():
+        print(f"Error: Directory '{args.target_dir}' does not exist.", file=sys.stderr)
+        sys.exit(2)
+
+    doc_dict = {}
+    for p in target_path.glob("*.txt"):
+        content = safe_read_text(p)
+        if content:
+            doc_dict[p.name] = content
+
+    if len(doc_dict) < 2:
+        print("Error: At least 2 valid (non-empty) .txt files are required.", file=sys.stderr)
+        sys.exit(2)
+
+    results: List[EvaluationResult] = batch_evaluate(doc_dict, max_workers=args.jobs)
+
+    if args.json:
+        print(json.dumps([result_to_dict(r) for r in results], ensure_ascii=False, indent=2))
+    if args.json_file:
+        out_json_path = Path(args.json_file)
+        out_json_path.parent.mkdir(parents=True, exist_ok=True)
+        out_json_path.write_text(
+            json.dumps([result_to_dict(r) for r in results], ensure_ascii=False, indent=2), encoding="utf-8"
+        )
+        print(f"JSON report saved to: {out_json_path}")
+
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    diff_map = {}
+    for res in results:
+        doc_a_text = doc_dict.get(res.item_id, "")
+        doc_b_text = doc_dict.get(res.matched_peer_id, "")
+        pair_key = f"{res.item_id}_vs_{res.matched_peer_id}"
+        diff_map[pair_key] = generate_diff_html(doc_a_text, doc_b_text, res)
+
+    clusters = build_clusters(results)
+    indirect_chains = detect_indirect_chains(results, clusters)
+
+    if args.standalone:
+        out_file = output_dir / "sieve_dashboard_standalone.html"
+        generate_standalone_dashboard_html(results, diff_map, out_file)
+        print(f"Standalone HTML report generated: {out_file}")
+    else:
+        diff_dir = output_dir / "diff_htmls"
+        diff_dir.mkdir(exist_ok=True)
+        for pair_key, diff_html_str in diff_map.items():
+            (diff_dir / f"{pair_key}.html").write_text(diff_html_str, encoding="utf-8")
+        out_file = output_dir / "sieve_dashboard.html"
+        file_names = list(doc_dict.keys())
+        generate_dashboard_html(results, indirect_chains, file_names, out_file)
+        print(f"Standard HTML report generated: {out_file}")
+
+    has_red_risk = any(
+        (r.evaluation_signal.value if hasattr(r.evaluation_signal, "value") else str(r.evaluation_signal)) == "RED"
+        for r in results
+    )
+    if args.fail_on_risk and has_red_risk:
+        print("\n[CI AUDIT FAILED] High risk plagiarism (RED signal) detected!", file=sys.stderr)
+        sys.exit(1)
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

sieve_referee/dashboard_builder.py

@@ -0,0 +1,61 @@
+import html
+import json
+import math
+from pathlib import Path
+from typing import Dict, List, Tuple
+
+from .models import EvaluationResult
+
+
+def _calculate_graph_layout(file_names, results, indirect_chains, width=800, height=500):
+    n = len(file_names)
+    if n == 0:
+        return [], []
+    cx, cy = width / 2, height / 2
+    radius = min(width, height) * 0.35
+    nodes = []
+    node_pos = {}
+    for i, name in enumerate(file_names):
+        angle = (2 * math.pi * i) / n
+        x = cx + radius * math.cos(angle)
+        y = cy + radius * math.sin(angle)
+        nodes.append({"id": name, "label": name, "x": round(x, 1), "y": round(y, 1)})
+        node_pos[name] = (x, y)
+    edges = []
+    for res in results:
+        sig = getattr(res, "evaluation_signal", "GREEN")
+        sig_val = sig.value if hasattr(sig, "value") else str(sig)
+        if sig_val in ["RED", "YELLOW"]:
+            u, v = res.item_id, res.matched_peer_id
+            if u in node_pos and v in node_pos:
+                edges.append({"source": u, "target": v, "color": "#e03131" if sig_val == "RED" else "#f59f00"})
+    for chain in indirect_chains:
+        u, v = chain["item_id_a"], chain["item_id_b"]
+        if u in node_pos and v in node_pos:
+            edges.append({"source": u, "target": v, "color": "#fd7e14"})
+    return nodes, edges
+
+
+def generate_dashboard_html(results, indirect_chains, file_names, output_path: Path) -> None:
+    nodes, edges = _calculate_graph_layout(file_names, results, indirect_chains)
+    alerts = ""
+    for c in indirect_chains:
+        alerts += f"<div class='alert-card'>ALERT: {c['item_id_a']} <-> {c['item_id_b']} (cluster: {', '.join(c['cluster_members'])})</div>\n"
+    html_content = f"""<html><body>
+<h1>Dashboard</h1>
+{alerts if alerts else '<p>NO_ALERTS</p>'}
+</body></html>"""
+    output_path.write_text(html_content, encoding="utf-8")
+
+
+def generate_standalone_dashboard_html(results, diff_map, output_path: Path) -> None:
+    embedded_json = json.dumps(diff_map, ensure_ascii=False)
+    html_content = f"""<!DOCTYPE html>
+<html><head><meta charset="UTF-8"></head>
+<body>
+<h1>Sieve Referee Evaluation Dashboard (Standalone)</h1>
+<script>
+const EMBEDDED_DIFFS = {embedded_json};
+</script>
+</body></html>"""
+    output_path.write_text(html_content, encoding="utf-8")

sieve_referee/diff_builder.py

@@ -0,0 +1,72 @@
+import difflib
+import html
+from typing import Optional, Tuple
+
+from .models import EvaluationResult
+from .tokenizer import tokenize
+
+
+def _build_highlighted_spans(text_a: str, text_b: str) -> Tuple[str, str, int, int]:
+    tokens_a = tokenize(text_a)
+    tokens_b = tokenize(text_b)
+
+    raw_a = [t.raw_text for t in tokens_a]
+    raw_b = [t.raw_text for t in tokens_b]
+
+    matcher = difflib.SequenceMatcher(None, raw_a, raw_b)
+    opcodes = matcher.get_opcodes()
+
+    html_a_parts = []
+    html_b_parts = []
+    match_chars = 0
+    replace_chars = 0
+
+    for tag, i1, i2, j1, j2 in opcodes:
+        chunk_a_text = "".join(raw_a[i1:i2])
+        chunk_b_text = "".join(raw_b[j1:j2])
+        sub_a = html.escape(chunk_a_text)
+        sub_b = html.escape(chunk_b_text)
+
+        if tag == "equal":
+            html_a_parts.append(f'[EQUAL:{sub_a}]')
+            html_b_parts.append(f'[EQUAL:{sub_b}]')
+            match_chars += len(chunk_a_text)
+        elif tag == "replace":
+            html_a_parts.append(f'[REPLACE:{sub_a}]')
+            html_b_parts.append(f'[REPLACE:{sub_b}]')
+            replace_chars += max(len(chunk_a_text), len(chunk_b_text))
+        elif tag == "delete":
+            html_a_parts.append(f'[UNIQUE:{sub_a}]')
+        elif tag == "insert":
+            html_b_parts.append(f'[UNIQUE:{sub_b}]')
+
+    return "".join(html_a_parts), "".join(html_b_parts), match_chars, replace_chars
+
+
+def generate_side_by_side_html(doc_a: str, doc_b: str, evaluation_result=None) -> Tuple[str, str]:
+    span_a, span_b, _, _ = _build_highlighted_spans(doc_a, doc_b)
+    return span_a, span_b
+
+
+def generate_diff_html(doc_a: str, doc_b: str, res) -> str:
+    pattern_name = res.pattern_name
+    sig_val = res.evaluation_signal.value if hasattr(res.evaluation_signal, "value") else str(res.evaluation_signal)
+    return f"""<!DOCTYPE html>
+<html><head><meta charset="UTF-8"><title>Diff: {res.item_id} vs {res.matched_peer_id}</title></head>
+<body><h2>{res.item_id} vs {res.matched_peer_id}</h2>
+<p>Signal: {sig_val} | Mask: {res.mask} | Pattern: {pattern_name}</p>
+<div>{doc_a}</div><hr><div>{doc_b}</div></body></html>"""
+
+
+def export_diff_reports(results, documents, output_dir):
+    diff_dir = output_dir / "diff_htmls"
+    diff_dir.mkdir(parents=True, exist_ok=True)
+    for res in results:
+        sig = getattr(res, "evaluation_signal", "GREEN")
+        sig_val = sig.value if hasattr(sig, "value") else str(sig)
+        if sig_val in ["RED", "YELLOW"]:
+            doc_a = documents.get(res.item_id, "")
+            doc_b = documents.get(res.matched_peer_id, "")
+            html_content = generate_diff_html(doc_a, doc_b, res)
+            file_name = f"diff_{res.item_id}_vs_{res.matched_peer_id}.html"
+            (diff_dir / file_name).write_text(html_content, encoding="utf-8")

sieve_referee/evaluator.py

@@ -0,0 +1,118 @@
+import os
+import unicodedata
+from concurrent.futures import ProcessPoolExecutor
+from itertools import combinations
+from typing import Dict, List, Set, Tuple
+
+from .blocker import DeterministicBlocker
+from .models import EvaluationResult
+from .referee import LUT_RULES, evaluate_pair
+
+
+def build_clusters(results: List[EvaluationResult]) -> Dict[str, List[str]]:
+    """流用シグナル(RED/YELLOW)が存在するペアから連結成分(クラスタ)を構築する。"""
+    adj: Dict[str, Set[str]] = {}
+    all_nodes: Set[str] = set()
+    for res in results:
+        u, v = res.item_id, res.matched_peer_id
+        all_nodes.add(u)
+        all_nodes.add(v)
+        sig = getattr(res, "evaluation_signal", "GREEN")
+        sig_val = sig.value if hasattr(sig, "value") else str(sig)
+        if sig_val in ["RED", "YELLOW"]:
+            adj.setdefault(u, set()).add(v)
+            adj.setdefault(v, set()).add(u)
+
+    visited: Set[str] = set()
+    clusters_by_item: Dict[str, List[str]] = {}
+    for node in all_nodes:
+        if node not in visited:
+            component = []
+            queue = [node]
+            visited.add(node)
+            while queue:
+                curr = queue.pop(0)
+                component.append(curr)
+                for neighbor in adj.get(curr, []):
+                    if neighbor not in visited:
+                        visited.add(neighbor)
+                        queue.append(neighbor)
+            component.sort()
+            for member in component:
+                clusters_by_item[member] = component
+    return clusters_by_item
+
+
+def update_h4_in_results(results: List[EvaluationResult], clusters: Dict[str, List[str]]) -> List[EvaluationResult]:
+    """
+    安全不変式: b0(H4)=1 への昇格は、そのペア自身に直接の類似根拠(b2=H2 または b3=H3)が
+    存在し、かつサイズ3以上のクラスタに属する場合のみに限定する。
+    直接の根拠がない伝言ゲーム型の間接連鎖ペアは、ここではマスクを変更しない
+    (別途 chain_detector.detect_indirect_chains で検出・報告する)。
+    """
+    for res in results:
+        u, v = res.item_id, res.matched_peer_id
+        cluster_u = clusters.get(u, [])
+        if len(cluster_u) >= 3:
+            mask_chars = list(res.mask)
+            if len(mask_chars) == 4:
+                b1, b2, b3, _ = mask_chars
+                has_direct_edge = b2 == "1" or b3 == "1"
+                if has_direct_edge:
+                    new_mask = f"{b1}{b2}{b3}1"
+                    res.mask = new_mask
+                    if new_mask in LUT_RULES:
+                        signal, pattern, reason = LUT_RULES[new_mask]
+                        res.pattern_name = pattern
+                        res.reason = reason
+                        res.evaluation_signal = signal
+    return results
+
+
+def _worker_evaluate_pair(args: Tuple[str, str, str, str]) -> EvaluationResult:
+    id_a, text_a, id_b, text_b = args
+    return evaluate_pair(text_a, text_b, id_a=id_a, id_b=id_b)
+
+
+def batch_evaluate(documents: Dict[str, str], max_workers: int = None) -> List[EvaluationResult]:
+    """
+    文書辞書(filename -> content)を受け取り、
+    NFKC正規化 -> 決定論的プレ・ブロッキング -> (並列/逐次)ペア評価 ->
+    クラスタ構築 -> H4安全昇格、という一連のパイプラインを実行する。
+    """
+    normalized_docs = {doc_id: unicodedata.normalize("NFKC", content) for doc_id, content in documents.items()}
+    doc_ids = sorted(normalized_docs.keys())
+
+    blocker = DeterministicBlocker(ngram_size=3, min_shared_ngrams=1)
+    candidate_pairs = blocker.get_candidate_pairs(normalized_docs)
+
+    raw_results: List[EvaluationResult] = []
+    non_candidate_pairs: List[Tuple[str, str]] = []
+    candidate_args: List[Tuple[str, str, str, str]] = []
+
+    for id_a, id_b in combinations(doc_ids, 2):
+        if (id_a, id_b) in candidate_pairs:
+            candidate_args.append((id_a, normalized_docs[id_a], id_b, normalized_docs[id_b]))
+        else:
+            non_candidate_pairs.append((id_a, id_b))
+
+    if candidate_args:
+        workers = max_workers or min(os.cpu_count() or 4, len(candidate_args))
+        if workers > 1 and len(candidate_args) >= 4:
+            with ProcessPoolExecutor(max_workers=workers) as executor:
+                raw_results.extend(executor.map(_worker_evaluate_pair, candidate_args))
+        else:
+            for args in candidate_args:
+                raw_results.append(_worker_evaluate_pair(args))
+
+    signal, pattern, reason = LUT_RULES["1000"]
+    for id_a, id_b in non_candidate_pairs:
+        raw_results.append(EvaluationResult(
+            item_id=id_a, matched_peer_id=id_b, evaluation_signal=signal,
+            pattern_name=pattern, reason=reason, mask="1000",
+            structural_density=0.0, max_structure_similarity=0.0,
+            max_content_similarity=0.0, coverage=0.0,
+        ))
+
+    clusters = build_clusters(raw_results)
+    return update_h4_in_results(raw_results, clusters)

sieve_referee/io_utils.py

@@ -0,0 +1,26 @@
+from pathlib import Path
+import unicodedata
+
+SUPPORTED_ENCODINGS = ("utf-8", "utf-8-sig", "cp932", "shift_jis", "euc_jp")
+
+
+def safe_read_text(file_path):
+    path = Path(file_path)
+    if not path.exists() or not path.is_file():
+        return None
+    if path.stat().st_size == 0:
+        return None
+    raw_bytes = path.read_bytes()
+    decoded_text = None
+    for enc in SUPPORTED_ENCODINGS:
+        try:
+            decoded_text = raw_bytes.decode(enc)
+            break
+        except (UnicodeDecodeError, ValueError):
+            continue
+    if decoded_text is None:
+        return None
+    normalized = unicodedata.normalize("NFKC", decoded_text).strip()
+    if not normalized:
+        return None
+    return normalized

sieve_referee/metrics.py

@@ -0,0 +1,33 @@
+from typing import List, Set
+from .models import Token
+from .tokenizer import extract_keywords_with_subgrams, tokenize
+
+
+def calc_h1_density(text: str) -> float:
+    if not text:
+        return 0.0
+    return len(set(text)) / len(text)
+
+
+def _generate_ngrams(tokens: List[str], n: int) -> Set[str]:
+    if len(tokens) < n:
+        return {"_".join(tokens)} if tokens else set()
+    return {"_".join(tokens[i : i + n]) for i in range(len(tokens) - n + 1)}
+
+
+def calc_h2_structure_similarity(tokens_a: List[Token], tokens_b: List[Token]) -> float:
+    skel_a = [t.skeleton_tag for t in tokens_a]
+    skel_b = [t.skeleton_tag for t in tokens_b]
+    ngrams_a = _generate_ngrams(skel_a, n=5) | _generate_ngrams(skel_a, n=6)
+    ngrams_b = _generate_ngrams(skel_b, n=5) | _generate_ngrams(skel_b, n=6)
+    if not ngrams_a or not ngrams_b:
+        return 0.0
+    return len(ngrams_a & ngrams_b) / len(ngrams_a | ngrams_b)
+
+
+def calc_h3_content_similarity(text_a: str, text_b: str) -> float:
+    feats_a = extract_keywords_with_subgrams(text_a)
+    feats_b = extract_keywords_with_subgrams(text_b)
+    if not feats_a or not feats_b:
+        return 0.0
+    return len(feats_a & feats_b) / len(feats_a | feats_b)

sieve_referee/models.py

@@ -0,0 +1,36 @@
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional
+
+
+class SignalColor(str, Enum):
+    GREEN = "GREEN"
+    YELLOW = "YELLOW"
+    RED = "RED"
+
+
+@dataclass
+class Token:
+    raw_text: str
+    skeleton_tag: str
+
+
+@dataclass
+class Chunk:
+    chunk_id: int
+    text: str
+    window_size: int
+
+
+@dataclass
+class EvaluationResult:
+    item_id: str
+    evaluation_signal: SignalColor
+    mask: str
+    pattern_name: str
+    reason: str
+    structural_density: float
+    max_structure_similarity: float
+    max_content_similarity: float
+    coverage: float
+    matched_peer_id: Optional[str] = None

sieve_referee/referee.py

@@ -0,0 +1,80 @@
+from typing import Dict, Optional
+from .chunker import split_into_multi_scale_windows
+from .metrics import calc_h1_density, calc_h2_structure_similarity, calc_h3_content_similarity
+from .models import EvaluationResult, SignalColor
+from .tokenizer import tokenize
+
+LUT_RULES: Dict[str, tuple] = {
+    "1110": (SignalColor.RED, "EXACT_COPY_AND_FORMAT_MATCH", "共通フォーマットに沿った文章構造および本文内容の完全なコピーが検出されました。"),
+    "1100": (SignalColor.YELLOW, "COINCIDENTAL_FORMAT_MATCH", "指定課題のフォーマットや文章構造の型が一致しています(本文内容は独自)。"),
+    "1010": (SignalColor.RED, "PARAPHRASE_DETECTED", "構文構造は独自ですが、内容語の高度な一致(パラフレーズ・語彙置換による流用)が検出されました。"),
+    "1000": (SignalColor.GREEN, "STANDARD_STRUCTURE_UNIQUE", "標準的な記述密度を持った独自の文章です。"),
+    "0110": (SignalColor.RED, "FORMATTED_PARAPHRASE", "特殊構造内での内容流用・言い換えが検出されました。"),
+    "0100": (SignalColor.YELLOW, "FORMAT_ONLY_MATCH", "定型構造のみの一致です(内容は独自)。"),
+    "0010": (SignalColor.RED, "UNFORMATTED_CONTENT_COPY", "構造化されていないテキスト間での本文直接コピー・流用が検出されました。"),
+    "0000": (SignalColor.GREEN, "FULLY_UNIQUE", "構造・内容ともに完全な独自記述です。"),
+    "1111": (SignalColor.RED, "CLUSTER_EXACT_COPY", "複数人による集団的な完全コピペネットワークが検出されました。"),
+    "1101": (SignalColor.YELLOW, "CLUSTER_FORMAT_SHARING", "複数人で共通テンプレートを使い回しているクラスタです。"),
+    "1011": (SignalColor.RED, "CLUSTER_PARAPHRASE_GROUP", "複数人による集団的な言い換え・単語置換グループが検出されました。"),
+    "1001": (SignalColor.GREEN, "CLUSTER_DENSE_UNIQUE", "特定グループ内での標準的な記述です。"),
+}
+
+
+class SieveReferee:
+    def __init__(self, h1_threshold: float = 0.35, h2_threshold: float = 0.80, h3_threshold: float = 0.45):
+        self.h1_th = h1_threshold
+        self.h2_th = h2_threshold
+        self.h3_th = h3_threshold
+
+    def evaluate_pair(self, item_id: str, doc_a: str, doc_b: str, peer_id: Optional[str] = None) -> EvaluationResult:
+        h1_score_a = calc_h1_density(doc_a)
+        b1 = 1 if h1_score_a >= self.h1_th else 0
+
+        tokens_a = tokenize(doc_a)
+        tokens_b = tokenize(doc_b)
+        h2_score = calc_h2_structure_similarity(tokens_a, tokens_b)
+        b2 = 1 if h2_score >= self.h2_th else 0
+
+        sentences_a = [s for s in doc_a.split("。") if s.strip()]
+        chunks_a = split_into_multi_scale_windows(doc_a, [1, 2])
+        chunks_b = split_into_multi_scale_windows(doc_b, [1, 2])
+
+        max_h3 = 0.0
+        flagged_sentences = set()
+        for ca in chunks_a:
+            for cb in chunks_b:
+                h3 = calc_h3_content_similarity(ca.text, cb.text)
+                if h3 > max_h3:
+                    max_h3 = h3
+                if h3 >= self.h3_th and ca.window_size == 1:
+                    flagged_sentences.add(ca.chunk_id)
+
+        b3 = 1 if max_h3 >= self.h3_th else 0
+        coverage = len(flagged_sentences) / len(sentences_a) if sentences_a else 0.0
+
+        b4 = 0
+        mask = f"{b1}{b2}{b3}{b4}"
+        signal, pattern, reason = LUT_RULES.get(mask, (SignalColor.YELLOW, "UNKNOWN_PATTERN", "未定義パターンの判定です。確認が必要です。"))
+
+        return EvaluationResult(
+            item_id=item_id,
+            evaluation_signal=signal,
+            mask=mask,
+            pattern_name=pattern,
+            reason=reason,
+            structural_density=round(h1_score_a, 4),
+            max_structure_similarity=round(h2_score, 4),
+            max_content_similarity=round(max_h3, 4),
+            coverage=round(coverage, 4),
+            matched_peer_id=peer_id,
+        )
+
+
+_default_referee = None
+
+
+def evaluate_pair(doc_a: str, doc_b: str, id_a: str = "doc_a", id_b: str = "doc_b"):
+    global _default_referee
+    if _default_referee is None:
+        _default_referee = SieveReferee()
+    return _default_referee.evaluate_pair(item_id=id_a, doc_a=doc_a, doc_b=doc_b, peer_id=id_b)

sieve_referee/tokenizer.py

@@ -0,0 +1,47 @@
+import re
+from typing import List, Set
+from .models import Token
+
+PATTERN = re.compile(
+    r"(https?://\S+|www\.\S+)|"
+    r"(\d+)|"
+    r"([\u4e00-\u9fff]+)|"
+    r"([\u30a0-\u30ff]+)|"
+    r"([\u3040-\u309f]+)|"
+    r"([a-zA-Z_]+)|"
+    r"([^\w\s])"
+)
+
+
+def tokenize(text: str) -> List[Token]:
+    tokens = []
+    for match in PATTERN.finditer(text):
+        url, num, kanji, katakana, hiragana, alpha, symbol = match.groups()
+        raw = match.group(0)
+        if url:
+            tokens.append(Token(raw, "<URL>"))
+        elif num:
+            tokens.append(Token(raw, "<NUM>"))
+        elif kanji or katakana or alpha:
+            tokens.append(Token(raw, "<KW>"))
+        elif hiragana or symbol:
+            tokens.append(Token(raw, raw))
+    return tokens
+
+
+def extract_keywords_with_subgrams(text: str, min_gram: int = 2) -> Set[str]:
+    tokens = tokenize(text)
+    features = set()
+    for t in tokens:
+        if t.skeleton_tag == "<KW>":
+            raw = t.raw_text
+            features.add(raw)
+            if len(raw) >= min_gram:
+                for i in range(len(raw) - min_gram + 1):
+                    features.add(raw[i : i + min_gram])
+    return features
+
+
+def extract_keywords(text: str) -> Set[str]:
+    tokens = tokenize(text)
+    return {t.raw_text for t in tokens if t.skeleton_tag == "<KW>"}

sieve_referee-1.0.0.dist-info/METADATA

@@ -0,0 +1,207 @@
+Metadata-Version: 2.4
+Name: sieve-referee
+Version: 1.0.0
+Summary: Deterministic, zero-dependency plagiarism and text structure screening engine (Sieve-Core plugin)
+Author: Kai IWASAKI
+License: MIT
+Project-URL: Homepage, https://github.com/neguseatama/sieve-referee
+Project-URL: Repository, https://github.com/neguseatama/sieve-referee.git
+Project-URL: Related Project (Sieve-Core), https://github.com/neguseatama/sieve-core
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# Sieve Referee
+
+English | [日本語](README.ja.md)
+
+> **"Should we really use probabilistic AI models to evaluate text generated by other probabilistic AI models?"**
+> A fully local, zero-dependency, and deterministic (100% reproducible) text structure and plagiarism screening engine. Developed as an extended application of [Sieve-Core](https://github.com/neguseatama/sieve-core).
+
+---
+
+## 💡 Concept & Philosophy
+
+`Sieve-Core` is the core engine that applies the design philosophy of "Sieve-AI"—a "sieve of set theory" inspired by the Sieve of Eratosthenes—to the more general domain of text and data processing. `Sieve Referee`, designed by adding a "sieve of combinatorics" to this `Sieve-Core`, is a zero-dependency Python library that extracts reliable information from noise based on corroboration from multiple independent sources.
+
+When evaluating text that may have been generated by probabilistic models (such as LLMs), relying on another probabilistic model to calculate a similarity score makes it difficult to mathematically trace *why* that score was produced.
+
+`Sieve Referee` was built to address this problem: to ensure every judgment is mathematically traceable, 100% reproducible, and fully explainable.
+
+- **Deterministic Approach**: Evaluation is based on a 16-state Look-Up Table (LUT, of which 12 states are reachable). Identical inputs always produce identical outputs no matter how many times it is run.
+- **Explainable by Design**: Every result carries a `reason` string and a 4-bit hypothesis `mask`, so the basis of a judgment can always be inspected.
+- **Careful Classification to Avoid False Accusations**: A shared template or boilerplate format is classified separately as `COINCIDENTAL_FORMAT_MATCH` (`1100` / `YELLOW`), rather than being treated the same as an actual content match.
+- **Indirect ("Telephone Game") Chain Safety**: A pair with no direct similarity evidence (e.g. `A <-> C`, where `A <-> B` and `B <-> C` are both flagged) is never silently upgraded to a positive match solely based on cluster membership. Such pairs are reported separately on the dashboard as an "indirect chain (telephone game) warning," and the direct comparison result itself is left unmodified.
+
+---
+
+## 🎯 Primary Use Cases
+
+Built with zero external dependencies and in-memory execution, `sieve_referee` is well suited as an embedded, privacy-focused screening engine.
+
+### 1. Education & Learning Management Systems (EdTech / LMS)
+Screen student submissions without uploading personal data or copyrighted text to a third-party cloud service. Common assignment templates are classified as `YELLOW`; unattributed copying or paraphrasing is flagged as `RED`.
+
+### 2. Digital Publishing & CMS
+Detect rewritten or recycled content submitted by freelance writers at draft-save time, without incurring LLM API costs for every check.
+
+### 3. LLM Training Data Deduplication
+Use as a deterministic pre-processing stage to remove near-duplicate or paraphrased records from a fine-tuning dataset, without GPU acceleration.
+
+### 4. Legal, Contract & RFP Review
+Separate boilerplate clauses (`COINCIDENTAL_FORMAT_MATCH`) from intentionally altered wording across contract revisions or vendor proposals.
+
+### 5. CI/CD Documentation Auditing
+Run in a GitHub Actions workflow or pre-commit hook to catch unauthorized copy-pasting across internal wikis or specification documents.
+
+---
+
+## ⚡ Key Technical Advantages
+
+1. **Near-zero cold start** — no PyTorch, no Transformers, no model loading; well suited to AWS Lambda or other serverless environments.
+2. **Zero per-call cost** — everything runs on local CPU.
+3. **Zero maintenance burden** — zero third-party dependencies means no exposure to upstream CVEs or breaking API changes.
+
+---
+
+## 📦 Installation
+
+```bash
+pip install git+https://github.com/neguseatama/sieve-referee.git
+```
+
+or, from a local clone:
+
+```bash
+git clone https://github.com/neguseatama/sieve-referee.git
+cd sieve-referee
+pip install .
+```
+
+---
+
+## 🐍 Usage (Python Library)
+
+```python
+from sieve_referee import batch_evaluate, EvaluationResult
+
+# 1. Prepare input documents in memory
+documents = {
+    "doc_a.txt": "Quantum computing relies on superposition and entanglement to achieve fast factorization.",
+    "doc_b.txt": "To achieve fast factorization, quantum computing makes use of entanglement and superposition."
+}
+
+# 2. Run the screening
+results: list[EvaluationResult] = batch_evaluate(documents)
+
+# 3. Read the results
+for res in results:
+    print(f"Pair:    {res.item_id} <-> {res.matched_peer_id}")
+    print(f"Signal:  {res.evaluation_signal.value}")  # -> RED
+    print(f"Mask:    {res.mask}")                      # -> 0010
+    print(f"Pattern: {res.pattern_name}")               # -> UNFORMATTED_CONTENT_COPY
+    print(f"Reason:  {res.reason}")
+    print("-" * 40)
+```
+
+### `EvaluationResult` fields
+
+| Property | Type | Description |
+|---|---|---|
+| `item_id` | `str` | Primary document identifier |
+| `matched_peer_id` | `str` | Comparison peer document identifier |
+| `evaluation_signal` | `SignalColor` | Result signal (`.value` returns `"GREEN"`, `"YELLOW"`, or `"RED"`) |
+| `mask` | `str` | 4-bit hypothesis mask, e.g. `"1010"` |
+| `pattern_name` | `str` | LUT pattern name, e.g. `"PARAPHRASE_DETECTED"` |
+| `reason` | `str` | Human-readable explanation of the judgment |
+
+---
+
+## 💻 Usage (CLI Tool)
+
+```bash
+# Basic directory scan (writes reports/sieve_dashboard.html)
+sieve-referee ./target_documents
+
+# Single self-contained HTML report (no diff_htmls/ subfolder)
+sieve-referee ./target_documents --standalone --output-dir ./reports
+
+# JSON output for automated pipelines, with a non-zero exit code on RED
+sieve-referee ./target_documents --json --fail-on-risk
+
+# Multi-core evaluation
+sieve-referee ./target_documents -j 4
+```
+
+| Option | Type | Description |
+|---|---|---|
+| `target_dir` | positional | Directory containing the `.txt` files to screen |
+| `--output-dir` | string | Report output directory (default: `reports`) |
+| `--standalone` | flag | Emit a single self-contained HTML report instead of a report + `diff_htmls/` folder |
+| `--json` | flag | Print results as JSON to stdout |
+| `--json-file` | string | Save results as JSON to the given path |
+| `--fail-on-risk` | flag | Exit with status code `1` if any `RED` signal is found |
+| `-j`, `--jobs` | int | Number of worker processes for evaluation (default: CPU count) |
+
+---
+
+## 📊 16-State Look-Up Table (LUT)
+
+Four independent hypotheses (`H1`–`H4`) are combined into a 4-bit mask, which maps deterministically to a signal, pattern name, and reason.
+
+| Mask | Signal | Pattern Name | Description |
+|:---:|:---:|:---|:---|
+| `1000` | 🟢 GREEN | `STANDARD_STRUCTURE_UNIQUE` | Independent document, no structural or content overlap detected. |
+| `1100` | 🟡 YELLOW | `COINCIDENTAL_FORMAT_MATCH` | Shared format/template only; content itself is independent. |
+| `1010` | 🔴 RED | `PARAPHRASE_DETECTED` | Content overlap despite a different structure (paraphrasing). |
+| `1011` | 🔴 RED | `CLUSTER_PARAPHRASE_GROUP` | Paraphrased reuse confirmed within a 3+ document cluster. |
+| `1111` | 🔴 RED | `CLUSTER_EXACT_COPY` | Exact/templated copy confirmed within a 3+ document cluster. |
+
+(12 of the 16 possible mask states are reachable in practice; the remaining 4 states, where cluster membership is set without any direct similarity evidence, are structurally unreachable by design.)
+
+> **Note on `H4` (cluster membership) and indirect chains**
+> `H4` is only set when a pair has *both* direct similarity evidence (`H2` or `H3`) *and* cluster membership of size ≥ 3. A pair with no direct evidence is never upgraded through cluster membership alone — such "telephone game" chains are reported separately as an indirect-chain warning on the dashboard, and the direct pair's own result is left unmodified.
+
+---
+
+## ⚠️ Known Limitation: Threshold Calibration Is Japanese-Specific
+
+All four thresholds (`H1 = 0.35`, `H2 = 0.80`, `H3 = 0.45`, cluster size `≥ 3`)
+were calibrated using real-data testing on **Japanese** text (roughly 1,300+
+pairs across unrelated documents, paraphrase pairs, and formal academic
+prose).
+
+The `H1` density metric in particular (`len(set(text)) / len(text)`) behaves
+very differently across scripts: Japanese text draws from a large set of
+kanji, so unique-character ratios tend to be high, while English and other
+alphabetic-script text reuses a small ~26-letter alphabet and will
+systematically score lower on this metric, independent of how generic or
+templated the content actually is. As a result, non-Japanese text may be
+under- or over-classified against the current defaults.
+
+Multi-encoding file reading (UTF-8, Shift_JIS, CP932, EUC-JP) and NFKC
+normalization work for any text regardless of language. What has **not**
+been separately validated is the semantic quality of `H2`/`H3` splitting,
+or the appropriateness of the default thresholds, for languages other than
+Japanese. If you use this on non-Japanese corpora, re-run the false-positive
+check described in [ARCHITECTURE.md](docs/ARCHITECTURE.md) against your own
+data before trusting the default thresholds.
+
+Additionally, the `reason` string returned by every `EvaluationResult` is
+currently hard-coded in Japanese regardless of the input document's
+language, as shown in the example output above. There is no English (or
+other-language) localization of these messages yet.
+
+---
+
+## 📚 Documentation
+
+- 🔌 [API & Integration Guide](docs/INTEGRATION.md) — using `sieve_referee` as a Python library or FastAPI microservice
+- 🚀 [Deployment Guide](docs/DEPLOYMENT.md) — Docker and Terraform for Google Cloud Run
+- 🔐 [CI/CD & Security Guide](docs/CI_CD_SETUP.md) — keyless GCP deployment via Workload Identity Federation
+- 🏛️ [Architecture Specification](docs/ARCHITECTURE.md) — the mathematical model and safety invariants in detail
+
+## 📄 License
+
+MIT License. See [LICENSE](LICENSE).

sieve_referee-1.0.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+sieve_referee/__init__.py,sha256=RWesn2RshNlis2l3OyKB4722fjyZVNpPdhyqJ1J_mPE,355
+sieve_referee/blocker.py,sha256=gvbFVvJGy5pdMRujptugQ-UmepRGU53y8w_KAGxX8B0,1751
+sieve_referee/chain_detector.py,sha256=SrnoAiGYQ6AORDF6IygQRU-ohIPkHRXo5ft0Em4XSXg,1221
+sieve_referee/chunker.py,sha256=cr5-MTAHNJpK_NegScT4Y9lf8rApptRISZo87yUNya8,557
+sieve_referee/cli.py,sha256=CDwMaRuivLAc06r_wM02mjU8dgFRNwI82Y90KYuj3ok,4057
+sieve_referee/dashboard_builder.py,sha256=77UWBI-srNiMBK5Sh2e527FV5HdBxW0YDfCSxx4wfno,2294
+sieve_referee/diff_builder.py,sha256=IFI_ebB8gU0eX9xgFeuiIfpAIJjCgjwqUp4oqd1diis,2793
+sieve_referee/evaluator.py,sha256=BwDD9L5XO3WBcFKlOUKKHk821gmPxfrxOWvn1TORD1c,5034
+sieve_referee/io_utils.py,sha256=HkJIcLpQwWgVnliGhNOZOeS7pp7yEGE5EcpGEewGueI,730
+sieve_referee/metrics.py,sha256=bMDzdNYT7BmK5PKAaaVJNdpu3oeKi88hGA9QGYN2v5A,1203
+sieve_referee/models.py,sha256=qHIGcCAG_vzQIf0NzjGtaQlqEIC9c0BSuDAkk5b6URU,622
+sieve_referee/referee.py,sha256=MfHT9IsyT15Uvc39tTn6G33S8dSbTmnXcyQocgV9GpM,4345
+sieve_referee/tokenizer.py,sha256=gDA-HwxMRlWoDehi7-YsScfzZhN0Mq2B4FUrN787Rv0,1327
+sieve_referee-1.0.0.dist-info/licenses/LICENSE,sha256=elLpPoGcCA_Tr2ln2abEvan7pEj_LoC4Q0-z5cudib8,1068
+sieve_referee-1.0.0.dist-info/METADATA,sha256=J-CfaBo8tjpTFnAZFhsvryVWmz6-HBLuiQN-1zevo5o,10513
+sieve_referee-1.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+sieve_referee-1.0.0.dist-info/entry_points.txt,sha256=Yb-iXdSaMta485OGpzlBT_vOrpAx1sdXDrqcvj4en24,57
+sieve_referee-1.0.0.dist-info/top_level.txt,sha256=9p33A4hueB8pP2wmczfziL-mx1vvV8T1QrHZecuKXL4,14
+sieve_referee-1.0.0.dist-info/RECORD,,

sieve_referee-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

sieve_referee-1.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+sieve-referee = sieve_referee.cli:main

sieve_referee-1.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Kai IWASAKI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sieve_referee-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+sieve_referee