memplex 3.2.0__py3-none-any.whl

memnex/config.py

@@ -0,0 +1,366 @@
+"""MemNex configuration system.
+
+Supports loading from YAML files, environment variable overrides (MEMNEX_*),
+and sensible defaults when no configuration is found.
+
+Priority: MEMNEX_* env vars > config.yaml > defaults
+"""
+
+import logging
+import os
+from dataclasses import dataclass, field, fields
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+# ── Sub-configurations ──────────────────────────────────────────
+
+
+@dataclass
+class StorageConfig:
+    """Storage backend configuration."""
+
+    backend: str = "standard"  # lite | standard | enterprise
+    path: str = "~/.memnex"
+
+
+@dataclass
+class EmbeddingConfig:
+    """Embedding model configuration."""
+
+    model: str = "default"  # default | bge-m3 | bge-small | openai
+    dimension: int = 384
+    batch_size: int = 32
+    contextual_retrieval: bool = True
+    hyde_enabled: bool = True
+
+
+@dataclass
+class RerankerConfig:
+    """Reranker scoring configuration."""
+
+    weights: Dict[str, float] = field(default_factory=lambda: {
+        "raw_relevance": 0.25,
+        "semantic_similarity": 0.30,
+        "recency_decay": 0.15,
+        "source_authority": 0.15,
+        "frequency": 0.15,
+    })
+    cross_encoder_enabled: bool = False
+    cross_encoder_model: str = "BAAI/bge-reranker-v2-m3"
+
+
+@dataclass
+class CompactionConfig:
+    """Compaction pipeline configuration."""
+
+    dedup_threshold: float = 0.95
+    chunk_threshold: int = 20000
+    warn_threshold: int = 50000
+    hard_limit: int = 500000
+    field_max_values: int = 20
+    needs_review_ttl_days: int = 30
+    prune_confidence_threshold: float = 0.3
+    prune_max_age_days: int = 180
+    prune_min_access_count: int = 0
+    dedup_use_faiss: bool = True
+
+
+@dataclass
+class GraphConfig:
+    """Graph configuration."""
+
+    semantic_similar_threshold: float = 0.85
+    semantic_similar_max_edges: int = 10
+    semantic_similar_ttl_days: int = 30
+    semantic_similar_sync_on_merge: bool = False
+    community_detection_enabled: bool = True
+    community_min_size: int = 3
+
+
+@dataclass
+class RetrievalConfig:
+    """Retrieval configuration."""
+
+    default_max_tokens: int = 4000
+    skill_max_tokens: int = 2000
+    injection_scan_enabled: bool = True
+
+
+@dataclass
+class LLMConfig:
+    """LLM provider configuration."""
+
+    semantic_extraction: bool = True
+    query_enhancement: bool = True
+    conflict_resolution: bool = True
+    summarization: bool = True
+    reranking: bool = True
+    provider: str = "anthropic"
+    anthropic_api_key: Optional[str] = None
+    local_endpoint: Optional[str] = None
+    local_model: Optional[str] = None
+    fallback_chain: List[str] = field(default_factory=lambda: ["anthropic"])
+    max_input_length: int = 10000
+
+
+@dataclass
+class ObservationConfig:
+    """Observation rate-limiting configuration."""
+
+    max_per_minute: int = 20
+
+
+@dataclass
+class LoggingConfig:
+    """Logging configuration."""
+
+    level: str = "INFO"
+    sanitize_sensitive: bool = True
+
+
+@dataclass
+class EncryptionConfig:
+    """Encryption configuration."""
+
+    enabled: bool = False
+    key_path: str = "~/.memnex/.enc_key"
+
+
+# ── Top-level configuration ─────────────────────────────────────
+
+
+@dataclass
+class MemNexConfig:
+    """Top-level MemNex configuration.
+
+    Collects all sub-configurations into a single object.
+    Use ``load_config()`` to create an instance.
+    """
+
+    storage: StorageConfig = field(default_factory=StorageConfig)
+    embedding: EmbeddingConfig = field(default_factory=EmbeddingConfig)
+    reranker: RerankerConfig = field(default_factory=RerankerConfig)
+    compaction: CompactionConfig = field(default_factory=CompactionConfig)
+    graph: GraphConfig = field(default_factory=GraphConfig)
+    retrieval: RetrievalConfig = field(default_factory=RetrievalConfig)
+    llm: LLMConfig = field(default_factory=LLMConfig)
+    observation: ObservationConfig = field(default_factory=ObservationConfig)
+    logging: LoggingConfig = field(default_factory=LoggingConfig)
+    encryption: EncryptionConfig = field(default_factory=EncryptionConfig)
+
+
+# ── Env-var mapping ─────────────────────────────────────────────
+
+# Maps ``MEMNEX_<SECTION>_<KEY>`` environment variables to
+# ``(MemNexConfig field, sub-config field)`` pairs.
+#
+# Example: ``MEMNEX_STORAGE_BACKEND=enterprise`` sets
+# ``config.storage.backend = "enterprise"``.
+#
+# Keys not listed here are still resolved dynamically in
+# ``_apply_env_overrides()`` using the same naming convention.
+
+_ENV_TYPE_COERCIONS: Dict[str, type] = {
+    # StorageConfig
+    "storage.backend": str,
+    "storage.path": str,
+    # EmbeddingConfig
+    "embedding.model": str,
+    "embedding.dimension": int,
+    "embedding.batch_size": int,
+    "embedding.contextual_retrieval": bool,
+    "embedding.hyde_enabled": bool,
+    # RerankerConfig
+    "reranker.cross_encoder_enabled": bool,
+    "reranker.cross_encoder_model": str,
+    # CompactionConfig
+    "compaction.dedup_threshold": float,
+    "compaction.chunk_threshold": int,
+    "compaction.warn_threshold": int,
+    "compaction.hard_limit": int,
+    "compaction.field_max_values": int,
+    "compaction.needs_review_ttl_days": int,
+    "compaction.prune_confidence_threshold": float,
+    "compaction.prune_max_age_days": int,
+    "compaction.prune_min_access_count": int,
+    "compaction.dedup_use_faiss": bool,
+    # GraphConfig
+    "graph.semantic_similar_threshold": float,
+    "graph.semantic_similar_max_edges": int,
+    "graph.semantic_similar_ttl_days": int,
+    "graph.semantic_similar_sync_on_merge": bool,
+    "graph.community_detection_enabled": bool,
+    "graph.community_min_size": int,
+    # RetrievalConfig
+    "retrieval.default_max_tokens": int,
+    "retrieval.skill_max_tokens": int,
+    "retrieval.injection_scan_enabled": bool,
+    # LLMConfig
+    "llm.semantic_extraction": bool,
+    "llm.query_enhancement": bool,
+    "llm.conflict_resolution": bool,
+    "llm.summarization": bool,
+    "llm.reranking": bool,
+    "llm.provider": str,
+    "llm.anthropic_api_key": str,
+    "llm.local_endpoint": str,
+    "llm.local_model": str,
+    "llm.max_input_length": int,
+    # ObservationConfig
+    "observation.max_per_minute": int,
+    # LoggingConfig
+    "logging.level": str,
+    "logging.sanitize_sensitive": bool,
+    # EncryptionConfig
+    "encryption.enabled": bool,
+    "encryption.key_path": str,
+}
+
+
+def _coerce(value: str, target_type: type) -> Any:
+    """Coerce a string value to the target type."""
+    if target_type is bool:
+        return value.lower() in ("true", "1", "yes", "on")
+    if target_type is int:
+        return int(value)
+    if target_type is float:
+        return float(value)
+    return value
+
+
+def _apply_env_overrides(config: MemNexConfig) -> None:
+    """Apply MEMNEX_* environment variable overrides to *config*.
+
+    Convention: ``MEMNEX_{SECTION}_{KEY}`` (upper-cased).
+
+    For the ``weights`` dict in ``reranker``, individual keys can be set via
+    ``MEMNEX_RERANKER_WEIGHTS_RAW_RELEVANCE=0.3``.
+    """
+    for dotted_path, target_type in _ENV_TYPE_COERCIONS.items():
+        parts = dotted_path.split(".")
+        section_name = parts[0]
+        key_name = parts[1]
+        env_key = f"MEMNEX_{section_name.upper()}_{key_name.upper()}"
+        env_value = os.environ.get(env_key)
+        if env_value is not None:
+            sub_config = getattr(config, section_name)
+            setattr(sub_config, key_name, _coerce(env_value, target_type))
+
+    # Handle reranker.weights sub-dict via MEMNEX_RERANKER_WEIGHTS_<KEY>
+    weights_env_prefix = "MEMNEX_RERANKER_WEIGHTS_"
+    for env_key, env_value in os.environ.items():
+        if env_key.startswith(weights_env_prefix):
+            weight_key = env_key[len(weights_env_prefix):].lower()
+            try:
+                config.reranker.weights[weight_key] = float(env_value)
+            except (ValueError, TypeError):
+                logger.warning("Invalid weight value for %s: %s", env_key, env_value)
+
+    # Handle LLM fallback_chain via MEMNEX_LLM_FALLBACK_CHAIN (comma-separated)
+    fallback_env = os.environ.get("MEMNEX_LLM_FALLBACK_CHAIN")
+    if fallback_env is not None:
+        config.llm.fallback_chain = [s.strip() for s in fallback_env.split(",") if s.strip()]
+
+
+# ── YAML loading helpers ────────────────────────────────────────
+
+
+def _parse_yaml(path: Path) -> Optional[Dict[str, Any]]:
+    """Try to parse a YAML file; return ``None`` if pyyaml is not available."""
+    try:
+        import yaml  # optional dependency
+    except ImportError:
+        logger.debug("PyYAML not installed, skipping config file: %s", path)
+        return None
+
+    if not path.exists():
+        return None
+
+    with open(path, "r", encoding="utf-8") as fh:
+        data = yaml.safe_load(fh)
+
+    return data if isinstance(data, dict) else None
+
+
+def _dict_to_dataclass(cls: type, data: Dict[str, Any]) -> Any:
+    """Recursively convert a plain dict to a dataclass instance.
+
+    Only keys that match known fields are used; unknown keys are silently
+    ignored so that config files with extra sections don't crash.
+    """
+    known_fields = {f.name for f in fields(cls)}
+    kwargs: Dict[str, Any] = {}
+
+    for f in fields(cls):
+        if f.name not in data:
+            continue
+        value = data[f.name]
+        field_type = f.type
+
+        # Resolve string annotations to actual types
+        if isinstance(field_type, str):
+            # field.type is a string like "StorageConfig"
+            # Try to resolve from module globals
+            import sys
+            field_type = sys.modules.get(cls.__module__, None)
+            if field_type is not None:
+                field_type = getattr(field_type, f.type, None)
+
+        # Handle nested dataclasses
+        if isinstance(value, dict) and hasattr(field_type, "__dataclass_fields__"):
+            kwargs[f.name] = _dict_to_dataclass(field_type, value)
+        else:
+            kwargs[f.name] = value
+
+    # Filter out keys not in known_fields to avoid TypeError
+    kwargs = {k: v for k, v in kwargs.items() if k in known_fields}
+    return cls(**kwargs)
+
+
+# ── Public API ──────────────────────────────────────────────────
+
+_DEFAULT_CONFIG_PATH = Path("~/.memnex/config.yaml").expanduser()
+
+
+def load_config(path: Optional[str] = None) -> MemNexConfig:
+    """Load MemNex configuration.
+
+    Resolution order (highest priority first):
+    1. ``MEMNEX_*`` environment variables
+    2. YAML config file at *path* (or ``~/.memnex/config.yaml``)
+    3. Built-in defaults
+
+    Parameters
+    ----------
+    path:
+        Explicit path to a YAML config file.  When ``None`` the default
+        location ``~/.memnex/config.yaml`` is tried.  If the file does
+        not exist, defaults are used.
+
+    Returns
+    -------
+    MemNexConfig
+        Fully resolved configuration object.
+    """
+    config = MemNexConfig()
+
+    # Resolve config file path
+    config_path = Path(path).expanduser() if path else _DEFAULT_CONFIG_PATH
+
+    # Load YAML overlay
+    yaml_data = _parse_yaml(config_path)
+    if yaml_data:
+        config = _dict_to_dataclass(MemNexConfig, yaml_data)
+        logger.debug("Loaded config from %s", config_path)
+    else:
+        logger.debug(
+            "No config file found at %s, using defaults",
+            config_path,
+        )
+
+    # Apply environment variable overrides (highest priority)
+    _apply_env_overrides(config)
+
+    return config

memnex/core/__init__.py

@@ -0,0 +1,13 @@
+"""memnex.core -- pure computation layer.
+
+Primary entry point::
+
+    from memnex.core import CoreEngine
+
+    engine = CoreEngine()
+    extracted = engine.extract(source)
+"""
+
+from memnex.core.engine import CoreEngine
+
+__all__ = ["CoreEngine"]

memnex/core/associator/__init__.py

@@ -0,0 +1,8 @@
+"""Association and linking modules."""
+
+from .term_mapper import TermMapper
+from .ref_linker import RefLinker
+from .entity_aligner import EntityAligner
+from .domain_classifier import DomainClassifier
+
+__all__ = ["TermMapper", "RefLinker", "EntityAligner", "DomainClassifier"]

memnex/core/associator/domain_classifier.py

@@ -0,0 +1,75 @@
+"""DomainClassifier - classifies functions into domain categories."""
+
+from typing import Dict, List, Optional
+from memnex.models.memory import Function
+
+
+class DomainClassifier:
+    """Classifies functions into domain categories based on keyword matching."""
+
+    DOMAIN_KEYWORDS: Dict[str, List[str]] = {
+        "认证模块": ["登录", "登出", "注册", "密码", "验证码", "认证", "OAuth", "login", "logout", "register", "password", "2FA"],
+        "账户模块": ["账户", "账号", "资料", "设置", "偏好", "profile", "account", "settings"],
+        "首页模块": ["首页", "仪表盘", "概览", "指标", "展示", "home", "dashboard", "metrics", "KPI"],
+        "订单模块": ["订单", "下单", "购物车", "order", "cart", "purchase"],
+        "支付模块": ["支付", "付款", "银行卡", "账单", "发票", "payment", "billing", "invoice"],
+        "通知模块": ["通知", "邮件", "短信", "消息", "提醒", "notification", "email", "SMS"],
+        "报表模块": ["报表", "统计", "导出", "分析", "report", "analytics", "export"],
+        "搜索模块": ["搜索", "查询", "过滤", "筛选", "推荐", "search", "filter", "recommendation"],
+        "安全模块": ["安全", "权限", "访问", "角色", "加密", "security", "permission", "access", "role"],
+        "配置模块": ["配置", "设置项", "参数", "开关", "config", "settings", "feature flag"],
+    }
+
+    def classify(self, func: Function) -> str:
+        """
+        Classify a function into a domain category.
+
+        Args:
+            func: Function to classify
+
+        Returns:
+            Domain name (e.g., "认证模块", "支付模块", or "通用")
+        """
+        text_parts = []
+        if func.name:
+            text_parts.append(("name", func.name))
+
+        # Collect text from multi-value fields (List[FieldValue])
+        for fv in func.trigger:
+            text_parts.append(("trigger", fv.desc))
+        for fv in func.condition:
+            text_parts.append(("condition", fv.desc))
+        for fv in func.action:
+            text_parts.append(("action", fv.desc))
+        for fv in func.benefit:
+            text_parts.append(("benefit", fv.desc))
+
+        scores: Dict[str, float] = {}
+        for domain, keywords in self.DOMAIN_KEYWORDS.items():
+            score = 0.0
+            for priority_label, text in text_parts:
+                weight = 2.0 if priority_label == "name" else 1.0
+                text_lower = text.lower()
+                for keyword in keywords:
+                    if keyword.lower() in text_lower:
+                        length_bonus = len(keyword) / 10.0
+                        score += weight + length_bonus
+            scores[domain] = score
+
+        if scores:
+            best_domain = max(scores.items(), key=lambda x: x[1])
+            if best_domain[1] > 0:
+                return best_domain[0]
+
+        return "通用"
+
+    def classify_with_llm_fallback(self, func: Function) -> str:
+        """
+        Classify a function with LLM fallback for ambiguous cases.
+
+        Currently returns "通用" as LLM fallback placeholder.
+        """
+        result = self.classify(func)
+        if result == "通用":
+            return "通用"
+        return result

memnex/core/associator/entity_aligner.py

@@ -0,0 +1,127 @@
+"""Entity alignment using fuzzy matching and semantic similarity."""
+
+import re
+from typing import List, Dict, Any, Optional, Tuple
+from difflib import SequenceMatcher
+
+
+class EntityAligner:
+    """Aligns and merges entities from multiple sources."""
+
+    TERM_EQUIVALENCES = {
+        "login": ["登录", "登入", "认证", "authenticate"],
+        "logout": ["登出", "退出", "signout"],
+        "register": ["注册", "登记", "signup"],
+        "user": ["用户", "user", "users", "member", "会员"],
+        "password": ["密码", "password", "pwd"],
+        "order": ["订单", "order", "订购"],
+        "payment": ["支付", "payment", "pay", "付款"],
+    }
+
+    def __init__(self):
+        """Build reverse mapping from Chinese/alt terms to English canonical forms."""
+        self._chinese_to_english = {}
+        for english, chinese_list in self.TERM_EQUIVALENCES.items():
+            for ch in chinese_list:
+                self._chinese_to_english[ch] = english
+
+    def normalize(self, text: str) -> str:
+        """Normalize text for comparison, translating Chinese terms to English."""
+        normalized = text.lower()
+        normalized = re.sub(r'[^a-z0-9一-鿿]', '', normalized)
+
+        for chinese, english in self._chinese_to_english.items():
+            if chinese in normalized:
+                normalized = normalized.replace(chinese, english)
+
+        return normalized
+
+    def calculate_similarity(self, str1: str, str2: str) -> float:
+        """Calculate similarity between two strings (0-1)."""
+        norm1 = self.normalize(str1)
+        norm2 = self.normalize(str2)
+
+        if norm1 == norm2:
+            return 1.0
+
+        for base, equivalents in self.TERM_EQUIVALENCES.items():
+            if norm1 in equivalents or norm1 == base:
+                if norm2 in equivalents or norm2 == base:
+                    return 0.85
+
+        return SequenceMatcher(None, norm1, norm2).ratio()
+
+    def find_similar(
+        self,
+        target: str,
+        entities: List,
+        threshold: float = 0.6
+    ) -> List[Tuple[Any, float]]:
+        """Find entities similar to target."""
+        results = []
+
+        for entity in entities:
+            entity_name = getattr(entity, 'name', '') or ''
+            score = self.calculate_similarity(target, entity_name)
+            if score >= threshold:
+                results.append((entity, score))
+
+        results.sort(key=lambda x: x[1], reverse=True)
+        return results
+
+    def find_merge_candidates(
+        self,
+        entities: List[Dict],
+        threshold: float = 0.9
+    ) -> List[List[Dict]]:
+        """
+        Find groups of entities that should be merged.
+
+        Uses blocking strategy for O(n) performance instead of O(n^2).
+        """
+        groups = []
+        used = set()
+
+        blocks: Dict[str, List[Dict]] = {}
+        for entity in entities:
+            normalized = self.normalize(entity['name'])
+            first_char = normalized[0] if normalized else '#'
+            if first_char not in blocks:
+                blocks[first_char] = []
+            blocks[first_char].append(entity)
+
+        for block_key, block_entities in blocks.items():
+            for i, entity in enumerate(block_entities):
+                if entity['id'] in used:
+                    continue
+
+                group = [entity]
+                used.add(entity['id'])
+
+                for other in block_entities[i + 1:]:
+                    if other['id'] in used:
+                        continue
+
+                    score = self.calculate_similarity(entity['name'], other['name'])
+                    if score >= threshold:
+                        group.append(other)
+                        used.add(other['id'])
+
+                if len(group) > 1:
+                    groups.append(group)
+
+        return groups
+
+    def suggest_merged_name(self, entities: List[Dict]) -> str:
+        """Suggest a merged name from multiple entities."""
+        if not entities:
+            return ""
+
+        if len(entities) == 1:
+            return entities[0]['name']
+
+        for entity in entities:
+            if re.search(r'[一-鿿]', entity['name']):
+                return entity['name']
+
+        return max(entities, key=lambda x: len(x['name']))['name']