MLATE 0.1.0__py3-none-any.whl

__init__.py

@@ -0,0 +1 @@
+"""MLATE — Multi-dimensional Literature Analysis and Thematic Exploration"""

__main__.py

@@ -0,0 +1,4 @@
+from cli import main
+
+if __name__ == "__main__":
+    main()

cli.py

@@ -0,0 +1,99 @@
+"""MLATE CLI 入口"""
+import argparse
+from pipeline import MLATEPipeline
+import config
+from logger import logger
+
+
+def main():
+    parser = argparse.ArgumentParser(prog="mlate", description="Multi-dimensional Literature Analysis and Thematic Exploration")
+    g = parser.add_argument_group("全局选项")
+    g.add_argument("--api-base", help="API Base URL")
+    g.add_argument("--model", default=None, help="模型名（默认按配置文件/环境变量/模型推断）")
+    g.add_argument("--workers", type=int, default=8)
+    g.add_argument("--qps", type=float, default=2.0)
+    g.add_argument("--lang", choices=["cn", "en", "both"], help="Output language (cn/en/both)")
+    g.add_argument("--source", choices=["auto", "wos", "scopus", "standard"], default="auto", 
+                   help="Data source type (auto-detect, wos, scopus, standard)")
+
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    # ── config ──
+    pc = sub.add_parser("config", help="管理配置（API Key / Base URL / Model）")
+    pc.add_argument("action", choices=["init", "set", "get", "show"],
+                    help="init=初始化 | set=设置值 key value | get=读取值 key | show=查看全部")
+    pc.add_argument("args", nargs="*", help="set 需要 key value；get 需要 key")
+
+    # ── filter ──
+    pf = sub.add_parser("filter", help="主题筛选：LLM 按用户主题评分排名，筛选 ≥ min-score")
+    pf.add_argument("--input", required=True)
+    pf.add_argument("--output", required=True)
+    pf.add_argument("--topic", required=True, help="筛选主题，如 'ship digital twin propulsion'")
+    pf.add_argument("--min-score", type=float, default=3.0, help="最低评分（1-5），默认 3.0")
+    pf.add_argument("--output-lang", help="LLM 理由与标准的输出语言（默认读取配置 output_lang）")
+
+    # ── explore ──
+    pe = sub.add_parser("explore", help="主题探索：第一阶段（自发发现草案 - 逐篇分析）")
+    pe.add_argument("--input", required=True, help="筛选后的 CSV 输入")
+    pe.add_argument("--output", required=True, help="输出原始草案 JSON 路径")
+    pe.add_argument("--max-papers", type=int)
+    pe.add_argument("--dims", help="初始维度（逗号分隔）")
+    pe.add_argument("--guide", help="研究者引导词")
+    pe.add_argument("--output-lang", help="LLM 发现结果的输出语言（默认读取配置 output_lang）")
+
+    # ── converge ──
+    pcv = sub.add_parser("converge", help="主题探索：第二阶段（智能收敛与总结）")
+    pcv.add_argument("--input", required=True, help="explore 产出的原始草案 JSON")
+    pcv.add_argument("--output", required=True, help="输出最终收敛后的 taxonomy JSON")
+    pcv.add_argument("--output-csv", help="输出带标签的文献 CSV 路径")
+    pcv.add_argument("--limit-cats", type=int, default=10, help="每个维度保留的分类上限")
+    pcv.add_argument("--dims", help="指定收敛的维度（默认全量）")
+    pcv.add_argument("--guide", help="收敛阶段的引导词")
+    pcv.add_argument("--source-csv", help="可选：原始 CSV 路径（用于将 ID 映射回论文标题）")
+    pcv.add_argument("--output-lang", help="LLM 收敛定义的输出语言（默认读取配置 output_lang）")
+
+    # ── translate ──
+    ptr = sub.add_parser("translate", help="自动翻译：对 CSV 列或 JSON 内容进行 LLM 翻译")
+    ptr.add_argument("--input", required=True)
+    ptr.add_argument("--output", required=True)
+    ptr.add_argument("--lang", default="中文", help="目标语言（默认中文）")
+    ptr.add_argument("--cols", help="如果是 CSV，指定需要翻译的列名（逗号分隔）")
+
+    args = parser.parse_args()
+
+    # Set language: Priority CLI > Config > Env (Default: en)
+    cfg = config.load()
+    lang = args.lang or cfg.get("lang")
+    if lang:
+        logger.set_lang(lang)
+    
+    # Resolve output language: CLI > Config > Default "中文"
+    output_lang = getattr(args, "output_lang", None) or cfg.get("output_lang") or "中文"
+
+    # 处理 config 子命令（不需要 pipeline）
+    if args.command == "config":
+        # 重新配置 argparse 以支持 config get <key>
+        config.cmd_config(args)
+        return
+
+    pipe = MLATEPipeline(
+        model=args.model, api_base=args.api_base,
+        max_workers=args.workers, qps=args.qps,
+    )
+
+    if args.command == "filter":
+        pipe.filter(args.input, args.output, topic=args.topic, min_score=args.min_score, source_type=args.source, language=output_lang)
+    elif args.command == "explore":
+        pipe.explore(args.input, args.output, args.max_papers, 
+                     source_type=args.source, researcher_guide=args.guide,
+                     initial_dims=args.dims, language=output_lang)
+    elif args.command == "converge":
+        pipe.converge(args.input, args.output, limit_cats=args.limit_cats,
+                      target_dims=args.dims, researcher_guide=args.guide,
+                      source_csv=args.source_csv, output_csv=args.output_csv, language=output_lang)
+    elif args.command == "translate":
+        pipe.translate(args.input, args.output, target_lang=args.lang, columns=args.cols)
+
+
+if __name__ == "__main__":
+    main()

config.py

@@ -0,0 +1,172 @@
+"""Configuration management: ~/.mlate/config.json + Env var auto-detection
+
+Security Design:
+  - API Keys are NEVER stored on disk.
+  - API Keys must be provided via environment variables (MLATE_API_KEY) or CLI arguments.
+  - Base URL and Model are non-sensitive and can be persisted safely.
+"""
+import os, json, sys, shutil
+from pathlib import Path
+from logger import logger
+
+CONFIG_DIR = Path.home() / ".mlate"
+CONFIG_FILE = CONFIG_DIR / "config.json"
+# Priority for env vars
+ENV_KEY_PRIORITY = ["MLATE_API_KEY", "LLM_API_KEY", "OPENAI_API_KEY", "DEEPSEEK_API_KEY"]
+
+
+def _ensure_dir():
+    CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+    if sys.platform != "win32":
+        CONFIG_DIR.chmod(0o700)
+
+
+def load() -> dict:
+    if CONFIG_FILE.exists():
+        try:
+            with open(CONFIG_FILE) as f:
+                return json.load(f)
+        except (json.JSONDecodeError, OSError):
+            pass
+    return {}
+
+
+def save(cfg: dict):
+    _ensure_dir()
+    tmp = CONFIG_FILE.with_suffix(".tmp")
+    with open(tmp, "w") as f:
+        json.dump(cfg, f, indent=2)
+    if sys.platform != "win32":
+        tmp.chmod(0o600)
+    tmp.rename(CONFIG_FILE)
+
+
+def mask_key(key: str) -> str:
+    if len(key) <= 12:
+        return "****"
+    return key[:8] + "…" + key[-4:]
+
+
+def resolve_api_key() -> str:
+    """Resolution: Env vars only. (Security policy: No CLI or Config file storage)"""
+    for var in ENV_KEY_PRIORITY:
+        v = os.environ.get(var)
+        if v:
+            return v
+    return ""
+
+
+def resolve_base_url(model: str, cli_value: str | None = None) -> str:
+    if cli_value:
+        return cli_value
+    env = os.environ.get("MLATE_API_BASE")
+    if env:
+        return env
+    cfg = load()
+    cfg_base = cfg.get("api_base")
+    if cfg_base:
+        return cfg_base
+    m = model.lower()
+    if any(k in m for k in ["deepseek"]):
+        return "https://api.deepseek.com"
+    if any(k in m for k in ["gpt", "o1", "o3"]):
+        return "https://api.openai.com/v1"
+    return "https://api.deepseek.com"
+
+
+def resolve_model(cli_value: str | None = None) -> str:
+    if cli_value:
+        return cli_value
+    env = os.environ.get("MLATE_MODEL")
+    if env:
+        return env
+    cfg = load()
+    return cfg.get("model", "deepseek-chat")
+
+
+# ── CLI 交互 ──────────────────────────────────────────────────
+def _print_setup_guide():
+    logger.info("")
+    logger.info("=" * 54)
+    logger.info("  API Key not set. Please use environment variables:", "  API Key 未设置，请通过环境变量配置：")
+    logger.info("")
+    logger.info("  Set environment variable (Recommended):", "  设置环境变量（推荐）：")
+    logger.info("     $env:MLATE_API_KEY = \"sk-xxx\"    # Windows PowerShell")
+    logger.info('     export MLATE_API_KEY="sk-xxx"       # Linux/macOS')
+    logger.info("=" * 54)
+
+
+def cmd_config(args):
+    action = args.action
+    rest = args.args
+
+    if action == "init":
+        _ensure_dir()
+        if not CONFIG_FILE.exists():
+            save({})
+            logger.success(f"Created {CONFIG_FILE}", f"已创建 {CONFIG_FILE}")
+        else:
+            logger.info(f"Already exists: {CONFIG_FILE}", f"已存在: {CONFIG_FILE}")
+
+    elif action == "set":
+        if len(rest) < 2:
+            logger.info("Usage: mlate config set <key> <value>", "用法: mlate config set <key> <value>")
+            logger.info("  Keys: api_base, model, lang", "  可用键: api_base, model, lang")
+            return
+        key, value = rest[0], rest[1]
+        if key == "api_key":
+            logger.error("Security policy: api_key cannot be stored in config file.", 
+                         "安全策略：api_key 不支持存储在配置文件中。")
+            logger.info("Please use environment variable MLATE_API_KEY instead.", 
+                         "请改用环境变量 MLATE_API_KEY。")
+            return
+        cfg = load()
+        cfg[key] = value
+        save(cfg)
+        logger.success(f"Successfully set {key}", f"已设置 {key}")
+
+    elif action == "get":
+        if not rest:
+            logger.info("Usage: mlate config get <key>", "用法: mlate config get <key>")
+            logger.info("  Keys: api_base, model, lang", "  可用键: api_base, model, lang")
+            return
+        key = rest[0]
+        if key == "api_key":
+            print(resolve_api_key())
+            return
+        cfg = load()
+        print(cfg.get(key, ""))
+
+    elif action == "show":
+        logger.info(f"Config file: {CONFIG_FILE}", f"配置文件: {CONFIG_FILE}")
+        logger.info(f"Env variable: MLATE_API_KEY", f"环境变量: MLATE_API_KEY")
+        logger.info("")
+        
+        cfg = load()
+        model = resolve_model()
+        base = resolve_base_url(model)
+        key = resolve_api_key()
+        
+        key_source_cn = "环境变量" if key else "无"
+        key_source_en = "Env Var" if key else "None"
+        
+        logger.info("Current effective configuration:", "当前生效配置:")
+        logger.info(f"  model       = {model}")
+        logger.info(f"  api_base    = {base}")
+        logger.info(f"  log_lang    = {cfg.get('lang', 'en')}")
+        logger.info(f"  output_lang = {cfg.get('output_lang', '中文')}")
+        logger.info("")
+        
+        if "api_key" in cfg:
+            del cfg["api_key"] # Clean up legacy key if exists
+            save(cfg)
+            
+        if cfg:
+            logger.info(f"Config file content ({len(cfg)} items):", f"配置文件内容 ({len(cfg)} 项):")
+            for k, v in cfg.items():
+                logger.info(f"  {k} = {v}")
+        else:
+            logger.info("Config file is empty", "配置文件为空")
+
+        if not key:
+            _print_setup_guide()

llm.py

@@ -0,0 +1,79 @@
+"""LLM 调用封装 + RateLimiter
+
+API Key 与 Base URL 解析委托给 config 模块，实现：
+  CLI 参数 > 环境变量 > 配置文件 > 模型名推断
+"""
+import os, time, threading, json
+from openai import OpenAI
+import config
+from logger import logger
+
+__all__ = ["RateLimiter", "LLM"]
+
+
+class RateLimiter:
+    def __init__(self, qps: float = 2.0):
+        self.qps = float(qps)
+        self.min_interval = 1.0 / self.qps if self.qps > 0 else 0
+        self._lock = threading.Lock()
+        self._next_time = 0.0
+
+    def wait(self):
+        if self.min_interval <= 0:
+            return
+        with self._lock:
+            now = time.time()
+            if now < self._next_time:
+                time.sleep(self._next_time - now)
+            self._next_time = max(now, self._next_time) + self.min_interval
+
+
+class LLM:
+    def __init__(self, model: str = None, api_base: str = None):
+        self.model = config.resolve_model(model)
+        self.api_key = config.resolve_api_key()
+        self.api_base = config.resolve_base_url(self.model, api_base)
+
+        if not self.api_key:
+            logger.error("API Key not set. Please configure via environment variables:", "API Key 未设置。请通过环境变量配置：")
+            logger.info("  $env:MLATE_API_KEY = \"sk-xxx\"    # Windows PowerShell")
+            logger.info('  export MLATE_API_KEY="sk-xxx"       # Linux/macOS')
+            raise ValueError("MLATE_API_KEY not set")
+
+        self.client = OpenAI(base_url=self.api_base, api_key=self.api_key)
+
+    def chat(self, messages: list, temperature: float = 0.2, retries: int = 2) -> str | None:
+        """Standard chat completion for plain text"""
+        last_err = None
+        for attempt in range(retries + 1):
+            try:
+                resp = self.client.chat.completions.create(
+                    model=self.model,
+                    messages=messages,
+                    temperature=temperature,
+                )
+                return resp.choices[0].message.content
+            except Exception as e:
+                last_err = e
+                if attempt < retries:
+                    time.sleep(1)
+        logger.error(f"LLM 调用失败: {last_err}", f"LLM call failed: {last_err}")
+        return None
+
+    def chat_json(self, messages: list, temperature: float = 0.2, retries: int = 2) -> dict | None:
+        last_err = None
+        for attempt in range(retries + 1):
+            try:
+                resp = self.client.chat.completions.create(
+                    model=self.model,
+                    messages=messages,
+                    response_format={"type": "json_object"},
+                    temperature=temperature,
+                )
+                return json.loads(resp.choices[0].message.content)
+            except Exception as e:
+                last_err = e
+                if attempt < retries:
+                    time.sleep(1)
+        logger.error(f"LLM 调用失败: {last_err}", f"LLM call failed: {last_err}")
+        return None

loaders.py

@@ -0,0 +1,108 @@
+import pandas as pd
+from pathlib import Path
+from logger import logger
+
+class BaseLoader:
+    """Base class for data loaders."""
+    
+    # Common column mappings for different sources
+    MAPPINGS = {
+        "wos": {
+            "title": ["TI", "Article Title", "Title"],
+            "abstract": ["AB", "Abstract"],
+            "keywords": ["DE", "Author Keywords", "ID", "Keywords Plus", "Keywords"]
+        },
+        "scopus": {
+            "title": ["Title"],
+            "abstract": ["Abstract"],
+            "keywords": ["Author Keywords", "Index Keywords"]
+        },
+        "standard": {
+            "title": ["Title", "题目", "标题"],
+            "abstract": ["Abstract", "摘要"],
+            "keywords": ["Keywords", "Author Keywords", "关键词"]
+        }
+    }
+
+    @staticmethod
+    def load(file_path: str, source_type: str = "auto") -> tuple[pd.DataFrame, dict]:
+        """Load literature data and return (df, mapping)."""
+        path = Path(file_path)
+        ext = path.suffix.lower()
+        
+        # 1. Load the raw data
+        try:
+            if ext == ".csv":
+                df = pd.read_csv(file_path, encoding="utf-8-sig")
+            elif ext in [".xls", ".xlsx"]:
+                df = pd.read_excel(file_path)
+            elif ext in [".txt", ".tsv"]:
+                df = pd.read_csv(file_path, sep="\t", encoding="utf-8-sig")
+            else:
+                raise ValueError(f"Unsupported file format: {ext}")
+        except Exception as e:
+            logger.error(f"Failed to load file {file_path}: {e}")
+            raise
+
+        # 2. Adaptive column mapping (without renaming)
+        return BaseLoader.identify_columns(df, source_type)
+
+    @staticmethod
+    def identify_columns(df: pd.DataFrame, source_type: str = "auto") -> tuple[pd.DataFrame, dict]:
+        """Identify key columns and return (df_cleaned, mapping)."""
+        # Deduplicate columns first
+        df = df.loc[:, ~df.columns.duplicated()]
+        columns = df.columns.tolist()
+        
+        mapping_to_use = {}
+        if source_type != "auto" and source_type in BaseLoader.MAPPINGS:
+            mapping_to_use = BaseLoader.MAPPINGS[source_type]
+        else:
+            best_score = -1
+            best_source = "standard"
+            for src, maps in BaseLoader.MAPPINGS.items():
+                score = sum(1 for target_list in maps.values() 
+                           for col in target_list if col in columns)
+                if score > best_score:
+                    best_score = score
+                    best_source = src
+            mapping_to_use = BaseLoader.MAPPINGS[best_source]
+            logger.info(f"Detected source type: {best_source}")
+
+        # Final mapping of original column names
+        res_mapping = {"title": "Title", "abstract": "Abstract", "keywords": "Keywords"}
+        found = {"title": False, "abstract": False, "keywords": False}
+        
+        # 1. Try predefined mappings
+        for internal_name, candidates in mapping_to_use.items():
+            for cand in candidates:
+                if cand in columns:
+                    res_mapping[internal_name] = cand
+                    found[internal_name] = True
+                    break
+        
+        # 2. Heuristic fallback
+        if not found["title"]:
+            for col in columns:
+                if any(kw in col.lower() for kw in ["title", "ti"]):
+                    res_mapping["title"] = col
+                    found["title"] = True
+                    break
+                    
+        if not found["abstract"]:
+            for col in columns:
+                if any(kw in col.lower() for kw in ["abstract", "ab", "summary"]):
+                    res_mapping["abstract"] = col
+                    found["abstract"] = True
+                    break
+
+        if not found["title"]:
+            logger.warning("Could not find Title column automatically.")
+        
+        # Log the identified mappings
+        logger.section("列名映射确认", "Column Mapping Confirmation")
+        logger.info(f"  题目 (Title)    : {res_mapping['title']}")
+        logger.info(f"  摘要 (Abstract) : {res_mapping['abstract']}")
+        logger.info(f"  关键词 (Keywords): {res_mapping['keywords']}")
+            
+        return df, res_mapping

logger.py

@@ -0,0 +1,70 @@
+import logging
+import sys
+import os
+
+class MLATELogger:
+    """Logger supporting English (default), Chinese, and bilingual output."""
+    
+    def __init__(self):
+        self._logger = logging.getLogger("mlate")
+        self._logger.setLevel(logging.INFO)
+        
+        # Default output to stdout
+        self._handler = logging.StreamHandler(sys.stdout)
+        self._formatter = logging.Formatter("%(message)s")
+        self._handler.setFormatter(self._formatter)
+        self._logger.addHandler(self._handler)
+        
+        # Language settings: 'cn', 'en', 'both'
+        # Priority: Env var MLATE_LANG > Default (en)
+        self.lang = os.environ.get("MLATE_LANG", "en").lower()
+
+    def set_level(self, level):
+        """Set logging level."""
+        self._logger.setLevel(level)
+
+    def set_lang(self, lang: str):
+        """Set output language: 'cn', 'en', 'both'"""
+        if lang in ["cn", "en", "both"]:
+            self.lang = lang
+
+    def _format(self, cn: str, en: str = None) -> str:
+        """Format message based on language settings."""
+        if en is None:
+            return cn
+        
+        if self.lang == "cn":
+            return cn
+        elif self.lang == "en":
+            return en
+        else:
+            # Bilingual mode
+            return f"{cn} | {en}"
+
+    def info(self, cn: str, en: str = None):
+        """Log info message."""
+        self._logger.info(self._format(cn, en))
+
+    def error(self, cn: str, en: str = None):
+        """Log error message."""
+        self._logger.error(self._format(cn, en))
+
+    def warning(self, cn: str, en: str = None):
+        """Log warning message."""
+        self._logger.warning(self._format(cn, en))
+
+    def success(self, cn: str, en: str = None):
+        """Log success message with a checkmark."""
+        self._logger.info(f"✓ {self._format(cn, en)}")
+
+    def section(self, cn: str, en: str = None):
+        """Log section header."""
+        msg = self._format(cn, en)
+        self._logger.info(f"\n── {msg} ──")
+
+    def separator(self):
+        """Log separator line."""
+        self._logger.info("-" * 50)
+
+# 全局单例
+logger = MLATELogger()