readloops 2.3.0__py3-none-any.whl

app/services/corpus.py

@@ -0,0 +1,78 @@
+"""语料库导入：四级/六级词表导入 words 表。"""
+import re
+import time
+from pathlib import Path
+
+from .. import config
+from ..database import get_db
+
+
+def parse_word_line(line: str):
+    """解析词表行：word [phonetic] pos.meaning"""
+    line = line.strip()
+    if not line or len(line) <= 2:
+        return None
+    if re.match(r'^[A-Z]$', line):
+        return None
+    phonetic = ""
+    phonetic_match = re.search(r'\[([^\]]+)\]', line)
+    if phonetic_match:
+        phonetic = phonetic_match.group(1)
+        line = line[:phonetic_match.start()] + line[phonetic_match.end():]
+    parts = line.split(None, 1)
+    if len(parts) < 2:
+        return None
+    word = parts[0].strip()
+    meaning = parts[1].strip() if len(parts) > 1 else ""
+    if not word or not re.match(r'^[a-zA-Z]', word):
+        return None
+    return word.lower(), word, phonetic, meaning
+
+def import_word_list(filepath: str, level: str = "CET4") -> int:
+    """导入词表文件，返回导入数量。"""
+    path = Path(filepath)
+    if not path.exists():
+        return 0
+    now = int(time.time())
+    count = 0
+    with open(path, encoding="utf-8-sig") as f:
+        lines = f.readlines()
+    with get_db() as conn:
+        for line in lines:
+            parsed = parse_word_line(line)
+            if not parsed:
+                continue
+            lemma, text, phonetic, meaning = parsed
+            existing = conn.execute("SELECT id FROM words WHERE lemma=?", (lemma,)).fetchone()
+            if existing:
+                continue
+            conn.execute("""INSERT INTO words
+                (lemma, text, type, meaning, phonetic, level, status, created_at, updated_at)
+                VALUES (?, ?, 'word', ?, ?, ?, 'new', ?, ?)""",
+                (lemma, text, meaning, phonetic, level, now, now))
+            count += 1
+    return count
+
+def import_cet4_words() -> int:
+    """导入四级词表。"""
+    path = config.CORPUS_DIR / "词表" / "CET4_大纲词表.txt"
+    return import_word_list(str(path), "CET4")
+
+def import_cet6_words() -> int:
+    """导入六级词表。"""
+    path = config.CORPUS_DIR / "词表" / "CET6_大纲词表.txt"
+    return import_word_list(str(path), "CET6")
+
+def lookup_local_dict(word: str) -> dict:
+    """从本地 words 表查询单词释义。"""
+    with get_db() as conn:
+        row = conn.execute("SELECT * FROM words WHERE lemma=?", (word.lower(),)).fetchone()
+        if row:
+            return {
+                "word": row["text"],
+                "phonetic": row["phonetic"] or "",
+                "meaning": row["meaning"] or "",
+                "in_vocab": True,
+                "word_id": row["id"],
+            }
+    return {"word": word, "phonetic": "", "meaning": "", "in_vocab": False, "word_id": None}

app/services/dict_import.py

@@ -0,0 +1,236 @@
+"""词典导入：支持 ECDICT（CSV / JSON）与纯文本词表。
+
+设计要点：
+- 流式读取，避免把整份词典读进内存（ECDICT 约 70MB / 77 万词）
+- 先一次性载入已有 lemma 集合，再批量插入，避免逐行查询（O(n²)）
+- 兼容多种来源格式，自动识别
+"""
+import csv
+import json
+import re
+import time
+from pathlib import Path
+
+from ..database import get_db
+
+# ECDICT exchange 字段的类型码映射
+EXCHANGE_TYPES = {
+    "0": "lemma",     # 原型
+    "1": "plural",    # 复数
+    "2": "ing",       # 现在分词
+    "3": "third",     # 第三人称单数
+    "4": "done",      # 过去分词
+    "5": "er",        # 比较级
+    "6": "est",       # 最高级
+    "s": "plural",
+    "i": "ing",
+    "p": "past",
+    "d": "done",
+    "r": "er",
+    "t": "est",
+}
+
+# ECDICT tag 字段 → 项目 level（按优先级匹配）
+LEVEL_TAGS = [
+    ("cet4", "CET4"),
+    ("cet6", "CET6"),
+    ("ky", "KY"),
+    ("gk", "GK"),
+    ("zk", "ZK"),
+]
+
+BATCH_SIZE = 2000
+
+
+def parse_exchange(value):
+    """把 ECDICT 的 exchange 字段解析为 {类型: 词形}。"""
+    if not value or value == "/":
+        return {}
+    result = {}
+    for part in value.split("/"):
+        if ":" not in part:
+            continue
+        code, form = part.split(":", 1)
+        form = form.strip()
+        if form:
+            result[EXCHANGE_TYPES.get(code, code)] = form
+    return result
+
+
+def level_from_tag(tag):
+    """从 ECDICT 的 tag 字段推断词表等级。"""
+    if not tag:
+        return "OTHER"
+    low = tag.lower()
+    for key, level in LEVEL_TAGS:
+        if key in low:
+            return level
+    return "OTHER"
+
+
+def detect_format(path):
+    """识别词表文件格式：ecdict_csv / json / text。"""
+    path = Path(path)
+    if path.suffix.lower() == ".json":
+        return "json"
+    if path.suffix.lower() == ".csv":
+        return "ecdict_csv"
+    # 未知后缀：看首行像不像带表头的 CSV
+    with open(path, encoding="utf-8-sig", errors="replace") as f:
+        head = f.readline()
+    if "," in head and "word" in head.lower():
+        return "ecdict_csv"
+    return "text"
+
+
+def _iter_ecdict_csv(path):
+    """流式产出 ECDICT CSV 中的行。"""
+    with open(path, encoding="utf-8-sig", errors="replace", newline="") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            word = (row.get("word") or "").strip()
+            if not word:
+                continue
+            yield {
+                "lemma": word.lower(),
+                "text": word,
+                "phonetic": (row.get("phonetic") or "").strip(),
+                "meaning": (row.get("translation") or row.get("definition") or "").strip(),
+                "level": level_from_tag(row.get("tag")),
+                "exchange": parse_exchange(row.get("exchange")),
+                "frequency": _to_int(row.get("frq")) or _to_int(row.get("bnc")) or 0,
+            }
+
+
+def _iter_json(path):
+    """产出 JSON 词表行（兼容 list[dict] 与 dict 包裹）。"""
+    data = json.loads(Path(path).read_text(encoding="utf-8-sig"))
+    if isinstance(data, dict):
+        data = data.get("words") or data.get("data") or list(data.values())
+    for row in data:
+        if not isinstance(row, dict):
+            continue
+        word = (row.get("word") or row.get("text") or row.get("lemma") or "").strip()
+        if not word:
+            continue
+        yield {
+            "lemma": word.lower(),
+            "text": word,
+            "phonetic": (row.get("phonetic") or "").strip(),
+            "meaning": (row.get("translation") or row.get("meaning")
+                        or row.get("definition") or "").strip(),
+            "level": row.get("level") or level_from_tag(row.get("tag")),
+            "exchange": (row.get("exchange") if isinstance(row.get("exchange"), dict)
+                         else parse_exchange(row.get("exchange"))),
+            "frequency": _to_int(row.get("frq")) or _to_int(row.get("frequency")) or 0,
+        }
+
+
+def _iter_text(path):
+    """产出纯文本词表行，格式：word [phonetic] pos.meaning"""
+    with open(path, encoding="utf-8-sig", errors="replace") as f:
+        for line in f:
+            line = line.strip()
+            if not line or len(line) <= 2 or re.match(r"^[A-Z]$", line):
+                continue
+            phonetic = ""
+            m = re.search(r"\[([^\]]+)\]", line)
+            if m:
+                phonetic = m.group(1)
+                line = line[:m.start()] + line[m.end():]
+            parts = line.split(None, 1)
+            if len(parts) < 2 or not re.match(r"^[a-zA-Z]", parts[0]):
+                continue
+            word = parts[0].strip()
+            yield {
+                "lemma": word.lower(),
+                "text": word,
+                "phonetic": phonetic,
+                "meaning": parts[1].strip(),
+                "level": "OTHER",
+                "exchange": {},
+                "frequency": 0,
+            }
+
+
+def _to_int(value):
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return 0
+
+
+_ITERATORS = {
+    "ecdict_csv": _iter_ecdict_csv,
+    "json": _iter_json,
+    "text": _iter_text,
+}
+
+
+def import_dictionary(path, level_filter=None, on_progress=None):
+    """导入词典文件。
+
+    path:         文件路径
+    level_filter: 只导入指定等级（如 'CET4'），None 表示全部
+    on_progress:  回调 fn(已导入数)，用于打印进度
+
+    返回 {"format", "inserted", "skipped", "total"}
+    """
+    path = Path(path)
+    if not path.exists():
+        raise FileNotFoundError(f"找不到文件：{path}")
+
+    fmt = detect_format(path)
+    iterator = _ITERATORS[fmt]
+
+    inserted = skipped = 0
+    batch = []
+    now = int(time.time())
+
+    with get_db() as conn:
+        # 一次性载入已有词，避免逐行查询
+        existing = {r[0] for r in conn.execute("SELECT lemma FROM words")}
+
+        for row in iterator(path):
+            lemma = row["lemma"]
+            if not lemma or not re.match(r"^[a-z]", lemma):
+                continue
+            if level_filter and row["level"] != level_filter:
+                continue
+            if lemma in existing:
+                skipped += 1
+                continue
+            existing.add(lemma)
+            batch.append((
+                lemma,
+                row["text"] or lemma,
+                row["meaning"],
+                row["phonetic"],
+                row["level"],
+                json.dumps(row["exchange"], ensure_ascii=False) if row["exchange"] else None,
+                row["frequency"],
+                now,
+                now,
+            ))
+            if len(batch) >= BATCH_SIZE:
+                _flush(conn, batch)
+                inserted += len(batch)
+                batch.clear()
+                if on_progress:
+                    on_progress(inserted)
+
+        if batch:
+            _flush(conn, batch)
+            inserted += len(batch)
+
+    return {"format": fmt, "inserted": inserted, "skipped": skipped}
+
+
+def _flush(conn, batch):
+    conn.executemany(
+        "INSERT OR IGNORE INTO words "
+        "(lemma, text, type, meaning, phonetic, level, exchange, frequency, "
+        " status, created_at, updated_at) "
+        "VALUES (?, ?, 'word', ?, ?, ?, ?, ?, 'new', ?, ?)",
+        batch,
+    )

app/services/similarity.py

@@ -0,0 +1,209 @@
+"""文章相似度计算模块。
+纯 Python 实现 TF-IDF + 余弦相似度，零外部依赖。
+用于：文章去重、同质化检测、真题相似度匹配、高频词提取。
+"""
+import math
+import re
+from collections import Counter, defaultdict
+
+from app import config
+from app.database import get_db
+
+# 英语停用词
+STOPWORDS = set("""
+a an the and or but if then else when while although though because since as
+of in on at to for with by from about into over under after before between
+is are was were be been being have has had do does did will would could should
+may might can shall it its this that these those i you he she we they me him
+her us them my your his our their not no so too very just also only own same
+such any all each both few more most other some than them what which who whom
+whose where why how there here up down out off on again once further
+""".split())
+
+
+def tokenize(text):
+    """分词：转小写，提取单词，过滤停用词和短词。"""
+    words = re.findall(r'[a-zA-Z]+', text.lower())
+    return [w for w in words if w not in STOPWORDS and len(w) > 2]
+
+
+def get_word_freq(text):
+    """获取词频统计。"""
+    return Counter(tokenize(text))
+
+
+def get_top_words(text, n=10):
+    """获取高频词 TOP N。"""
+    freq = get_word_freq(text)
+    return freq.most_common(n)
+
+
+def build_tfidf_vectors(documents):
+    """构建 TF-IDF 向量。
+    documents: list of (doc_id, text)
+    returns: dict of doc_id -> {word: tfidf_weight}
+    """
+    # 计算文档频率（DF）
+    doc_freq = defaultdict(int)
+    doc_tokens = {}
+
+    for doc_id, text in documents:
+        tokens = set(tokenize(text))
+        doc_tokens[doc_id] = tokens
+        for word in tokens:
+            doc_freq[word] += 1
+
+    n_docs = len(documents)
+    vectors = {}
+
+    for doc_id, text in documents:
+        tokens = tokenize(text)
+        tf = Counter(tokens)
+        vector = {}
+        for word, count in tf.items():
+            # TF = 词频 / 总词数
+            tf_weight = count / len(tokens) if tokens else 0
+            # IDF = log(总文档数 / 文档频率)
+            idf_weight = math.log(n_docs / (doc_freq[word] + 1))
+            vector[word] = tf_weight * idf_weight
+        vectors[doc_id] = vector
+
+    return vectors
+
+
+def cosine_similarity(vec1, vec2):
+    """计算两个向量的余弦相似度。"""
+    # 找共同词
+    common = set(vec1.keys()) & set(vec2.keys())
+    if not common:
+        return 0.0
+
+    dot_product = sum(vec1[w] * vec2[w] for w in common)
+    norm1 = math.sqrt(sum(v ** 2 for v in vec1.values()))
+    norm2 = math.sqrt(sum(v ** 2 for v in vec2.values()))
+
+    if norm1 == 0 or norm2 == 0:
+        return 0.0
+
+    return dot_product / (norm1 * norm2)
+
+
+def get_article_similarity(article_text, other_articles, top_n=3):
+    """计算一篇文章与其他文章的相似度，返回 TOP N。
+    other_articles: list of (id, title, text)
+    returns: list of (id, title, similarity_score)
+    """
+    if not other_articles:
+        return []
+
+    # 构建所有文档的 TF-IDF 向量（包含目标文章）
+    all_docs = [('target', article_text)] + [(str(a[0]), a[2]) for a in other_articles]
+    vectors = build_tfidf_vectors(all_docs)
+
+    target_vec = vectors.get('target', {})
+    similarities = []
+
+    for a in other_articles:
+        doc_id = str(a[0])
+        if doc_id in vectors:
+            sim = cosine_similarity(target_vec, vectors[doc_id])
+            similarities.append((a[0], a[1], sim))
+
+    # 按相似度降序排序
+    similarities.sort(key=lambda x: x[2], reverse=True)
+    return similarities[:top_n]
+
+
+def get_exam_similarity(article_text, top_n=3):
+    """计算文章与真题的相似度，返回 TOP N 最相近的真题。"""
+    return _load_exam_from_file(article_text, top_n)
+
+
+def _load_exam_from_file(article_text, top_n=3):
+    """从语料库文件加载真题并计算相似度。
+
+    真题语料受版权保护，不随仓库分发，需用户自备。默认路径：
+    <项目根>/语料库/真题阅读纯文本/all_passages_lazynote.json
+    可用环境变量 READLOOPS_CORPUS_FILE 覆盖。文件不存在时返回空列表。
+    """
+    import json
+    import os
+
+    exam_file = os.getenv('READLOOPS_CORPUS_FILE') or str(
+        config.CORPUS_DIR / '真题阅读纯文本' / 'all_passages_lazynote.json'
+    )
+    if not os.path.exists(exam_file):
+        return []
+
+    with open(exam_file, 'r', encoding='utf-8') as f:
+        passages = json.load(f)
+
+    # 只取仔细阅读（Section C），约350词，和生成文章长度接近
+    exam_articles = []
+    for p in passages:
+        if 'Section C' in p.get('source', ''):
+            title = p.get('title', '') or p.get('source', '')
+            exam_articles.append((p.get('source', ''), title, p.get('text', '')))
+
+    return get_article_similarity(article_text, exam_articles, top_n)
+
+
+def check_duplicate(article_text, recent_articles, threshold=0.4):
+    """检查文章是否与最近文章重复或同质化。
+    返回 (is_duplicate, max_similarity, most_similar_title)
+    """
+    if not recent_articles:
+        return False, 0.0, ''
+
+    other = [(a['id'], a['title'] or '文章', a['content'] or '') for a in recent_articles]
+    similarities = get_article_similarity(article_text, other, top_n=1)
+
+    if similarities:
+        _, title, sim = similarities[0]
+        return sim >= threshold, sim, title
+
+    return False, 0.0, ''
+
+
+def get_article_details(article_id):
+    """获取文章详情：高频词、目标生词、真题相似度。"""
+    with get_db() as conn:
+        article = conn.execute(
+            "SELECT id, title, content, target_words, created_at FROM articles WHERE id=?",
+            (article_id,)
+        ).fetchone()
+
+    if not article:
+        return None
+
+    content = article['content'] or ''
+    target_words = []
+    if article['target_words']:
+        try:
+            import json
+            target_words = json.loads(article['target_words'])
+        except Exception:
+            pass
+
+    # 高频词
+    top_words = get_top_words(content, n=10)
+
+    # 真题相似度
+    exam_sim = get_exam_similarity(content, top_n=3)
+
+    # 词数
+    word_count = len(re.findall(r'[a-zA-Z]+', content))
+
+    return {
+        'id': article['id'],
+        'title': article['title'],
+        'content': content,
+        'word_count': word_count,
+        'created_at': article['created_at'],
+        'target_words': target_words,
+        'top_words': top_words,
+        'exam_similarity': [
+            {'id': s[0], 'title': s[1], 'similarity': round(s[2] * 100, 1)}
+            for s in exam_sim
+        ],
+    }