preciso-graphrag 0.3.0rc1__py3-none-any.whl

config.py

@@ -0,0 +1,477 @@
+from __future__ import annotations
+
+import hashlib
+import asyncio
+import importlib
+import logging
+import math
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+# Same logger core/utils.py configures handlers on; config.py must not import
+# core.utils (core.utils imports config).
+logger = logging.getLogger("graphrag_mcp")
+
+# ==========================================================================
+# TOKEN LIMIT DEFAULTS (used by summary.py, merge.py, query.py)
+# Prevents LLM calls from exceeding API token limits
+# ==========================================================================
+DEFAULT_MAX_ENTITY_TOKENS = int(os.getenv("GRAPHRAG_MAX_ENTITY_TOKENS", "4000"))
+DEFAULT_MAX_RELATION_TOKENS = int(os.getenv("GRAPHRAG_MAX_RELATION_TOKENS", "4000"))
+DEFAULT_MAX_TOTAL_TOKENS = int(os.getenv("GRAPHRAG_MAX_TOTAL_TOKENS", "16000"))
+DEFAULT_KG_EVIDENCE_TOP_K = int(os.getenv("GRAPHRAG_KG_EVIDENCE_TOP_K", "8"))
+DEFAULT_KG_EVIDENCE_MIN_SIMILARITY = float(
+    os.getenv("GRAPHRAG_KG_EVIDENCE_MIN_SIMILARITY", "0.35")
+)
+
+# ============================================================================
+# KNOWLEDGE GRAPH DEFAULTS (used by merge.py, query.py)
+# Controls entity/relationship merging and retrieval behavior
+# ============================================================================
+DEFAULT_MAX_FILE_PATHS = 8                     # Max source file paths to store per entity/relation
+DEFAULT_FILE_PATH_MORE_PLACEHOLDER = "more_paths"  # Placeholder when truncating file paths
+DEFAULT_ENTITY_NAME_MAX_LENGTH = 255           # Max entity name length
+DEFAULT_TOP_K = 5                              # Top K results to return in queries
+DEFAULT_CHUNK_TOP_K = 20                       # Top K chunks to retrieve
+GRAPH_FIELD_SEP = "<SEP>"                      # Internal separator for multi-value graph fields
+
+# ============================================================================
+# MERGING & SOURCE TRACKING DEFAULTS (used by merge.py)
+# Controls how entities/relationships are merged when re-ingested
+# ============================================================================
+DEFAULT_SOURCE_IDS_LIMIT_METHOD = "KEEP"       # How to limit stored source references: "KEEP" (keep old) or "FIFO" (newest)
+SOURCE_IDS_LIMIT_METHOD_KEEP = "KEEP"          # Keep older source IDs when limit reached
+SOURCE_IDS_LIMIT_METHOD_FIFO = "FIFO"          # Use FIFO (first-in-first-out) when limit reached
+VALID_SOURCE_IDS_LIMIT_METHODS = {
+    SOURCE_IDS_LIMIT_METHOD_KEEP,
+    SOURCE_IDS_LIMIT_METHOD_FIFO,
+}
+
+# ==========================================================================
+# QUERY DEFAULTS (used by query.py)
+# ==========================================================================
+DEFAULT_HISTORY_TURNS = int(os.getenv("GRAPHRAG_HISTORY_TURNS", "3"))
+
+# ==========================================================================
+# SUMMARY DEFAULTS (used by summary.py, merge.py)
+# Preciso never calls an LLM to compress descriptions — compression is always
+# deferred to the MCP-driving agent (preciso_mcp/tools/pending_summaries_tool.py).
+# ==========================================================================
+DEFAULT_SUMMARY_CONTEXT_TOKENS = int(os.getenv("GRAPHRAG_SUMMARY_CONTEXT_TOKENS", "8000"))
+DEFAULT_SUMMARY_MAX_TOKENS = int(os.getenv("GRAPHRAG_SUMMARY_MAX_TOKENS", "1024"))
+# Two-zone description merge (core/summary.py): descriptions are stored as ONE
+# marker-tagged rolling summary of old mentions + a verbatim tail of the most recent
+# raw_tail_size descriptions. Compression fires only when the tail exceeds this count
+# or the field exceeds summary_context_size — never on a bare description count.
+# GRAPHRAG_FORCE_LLM_SUMMARY_ON_MERGE is honored as a legacy alias for the tail size.
+DEFAULT_RAW_TAIL_SIZE = int(
+    os.getenv(
+        "GRAPHRAG_RAW_TAIL_SIZE",
+        os.getenv("GRAPHRAG_FORCE_LLM_SUMMARY_ON_MERGE", "8"),
+    )
+)
+# Sentinel prefix identifying the single rolling-summary segment inside a SEP-joined
+# description field. Deliberately NOT env-configurable: changing it would orphan
+# markers already persisted in GRAPH_IS_HERE/. ASCII on purpose (tokenizer/grep-safe).
+#
+# CONTRACT — "storage keeps it, every exit strips it":
+# The marker exists ONLY inside GRAPH_IS_HERE/ (graph nodes/edges, incl. the
+# graphml artifact) so that the next merge can find the rolling-summary segment;
+# copying GRAPH_IS_HERE/ to another machine must preserve it. Every surface that
+# leaves storage must pass through core.utils.strip_summary_marker:
+#   - embedding content            core/merge.py (entity_content / rel_content)
+#   - relationship VDB payload     core/merge.py ("description" field)
+#   - LLM query context            core/query.py (_apply_token_truncation)
+#   - user-facing query results    core/utils.py (convert_to_user_format)
+#   - Neo4j export properties      core/export_adapters.py (_node/_edge_properties)
+#   - Qdrant export payloads       core/export_adapters.py (_vector_payload)
+#   - agent-summary tool outputs   preciso_mcp/tools/pending_summaries_tool.py
+#                                   (list_pending_summaries' content_to_summarize
+#                                   is read from raw_tail — never marker-tagged
+#                                   by construction; submit_summary re-embeds via
+#                                   the same entity_content/rel_content path above)
+# Adding a new export/output surface? Strip there too, and extend the guard test
+# in tests/manual/marker_leak_manual.py (and tests/test_marker_leak.py).
+SUMMARY_MARKER = "<<SUM>>"
+
+# ============================================================================
+# EMBEDDING DEFAULTS
+# ============================================================================
+DEFAULT_EMBEDDING_PROVIDER = os.getenv("GRAPHRAG_EMBEDDING_PROVIDER", "ollama")
+DEFAULT_EMBEDDING_MODEL = os.getenv("GRAPHRAG_EMBEDDING_MODEL", "mxbai-embed-large")
+DEFAULT_EMBEDDING_DIM = int(os.getenv("GRAPHRAG_EMBEDDING_DIM", "768"))
+DEFAULT_EMBEDDING_MAX_TOKENS = int(os.getenv("GRAPHRAG_EMBEDDING_MAX_TOKENS", "8192"))
+DEFAULT_OPENAI_EMBEDDING_MODEL = os.getenv("GRAPHRAG_OPENAI_EMBEDDING_MODEL", "text-embedding-3-large")
+DEFAULT_COHERE_EMBEDDING_MODEL = os.getenv("GRAPHRAG_COHERE_EMBEDDING_MODEL", "embed-english-v3.0")
+
+# ============================================================================
+# DOWNSTREAM EXPORT DEFAULTS
+# Optional settings for Neo4j and Qdrant exports.
+# These are read from environment variables so exports stay config-driven.
+# ============================================================================
+DEFAULT_NEO4J_URI = os.getenv("GRAPHRAG_NEO4J_URI", "")
+DEFAULT_NEO4J_USERNAME = os.getenv("GRAPHRAG_NEO4J_USERNAME", "neo4j")
+DEFAULT_NEO4J_PASSWORD = os.getenv("GRAPHRAG_NEO4J_PASSWORD", "")
+DEFAULT_NEO4J_DATABASE = os.getenv("GRAPHRAG_NEO4J_DATABASE", "neo4j")
+
+DEFAULT_QDRANT_URL = os.getenv("GRAPHRAG_QDRANT_URL", "")
+DEFAULT_QDRANT_API_KEY = os.getenv("GRAPHRAG_QDRANT_API_KEY", "")
+DEFAULT_QDRANT_COLLECTION_PREFIX = os.getenv("GRAPHRAG_QDRANT_COLLECTION_PREFIX", "preciso")
+
+DEFAULT_EXPORT_WORKSPACE = os.getenv("GRAPHRAG_EXPORT_WORKSPACE", "default")
+DEFAULT_EXPORT_CLEAR_EXISTING = os.getenv("GRAPHRAG_EXPORT_CLEAR_EXISTING", "false").lower() == "true"
+
+# ============================================================================
+# QUERY HEURISTIC DEFAULTS (used by query.py)
+# Domain heuristics live here (not in core/) so non-financial deployments can
+# override them without touching code.
+# ============================================================================
+# Phrases signalling a two-period comparison query ("hybrid" mode auto-upgrades
+# to "global" when one matches). Override with GRAPHRAG_YOY_SIGNAL_PHRASES as a
+# comma-separated list; items are matched verbatim against the lowercased query,
+# so leading/trailing spaces are significant (e.g. " vs ").
+_DEFAULT_YOY_SIGNAL_PHRASES = [
+    "change from",
+    "changed from",
+    "compare",
+    "comparison",
+    "growth from",
+    "grew from",
+    "increase from",
+    "increased from",
+    "decrease from",
+    "decreased from",
+    "year over year",
+    "yoy",
+    "from fiscal",
+    "from fy",
+    "versus",
+    " vs ",
+    "compared to",
+    "relative to",
+]
+_yoy_phrases_env = os.getenv("GRAPHRAG_YOY_SIGNAL_PHRASES")
+DEFAULT_YOY_SIGNAL_PHRASES = (
+    [phrase.lower() for phrase in _yoy_phrases_env.split(",") if phrase]
+    if _yoy_phrases_env is not None
+    else _DEFAULT_YOY_SIGNAL_PHRASES
+)
+
+# Persona line opening the default rag_response prompt. GRAPHRAG_RAG_PERSONA
+# accepts "neutral" (default), "financial" (the original Preciso wording), or
+# any custom persona sentence used verbatim (avoid `{`/`}` — the assembled
+# prompt goes through str.format).
+_RAG_PERSONAS = {
+    "neutral": "You are a knowledge graph retrieval assistant.",
+    "financial": "You are a financial graph retrieval assistant.",
+}
+_rag_persona_env = os.getenv("GRAPHRAG_RAG_PERSONA", "neutral")
+DEFAULT_RAG_PERSONA = _RAG_PERSONAS.get(_rag_persona_env.strip().lower(), _rag_persona_env)
+
+# ============================================================================
+# LLM PROMPTS (used by query.py)
+# Templates sent to LLM functions for response generation
+# ============================================================================
+PROMPTS = {
+    "rag_response": (
+        # Used by query.py → kg_query()
+        # Final prompt to generate response from knowledge graph context
+        DEFAULT_RAG_PERSONA + "\n"
+        "Use the provided context to answer in {response_type}.\n"
+        "Additional user guidance: {user_prompt}\n\n"
+        "{context_data}"
+    ),
+    "kg_query_context": (
+        # Used by query.py → to format context before sending to LLM
+        "-----Entities-----\n{entities_str}\n\n"
+        "-----Relationships-----\n{relations_str}\n\n"
+        "-----Text Chunks-----\n{text_chunks_str}\n\n"
+        "-----References-----\n{reference_list_str}"
+    ),
+    "fail_response": "No relevant graph context could be constructed for the query.",
+}
+
+
+async def _fallback_embed(texts, **kwargs):
+    """Return deterministic lexical embeddings for offline/test operation.
+
+    This is intentionally small and dependency-free, but unlike the former
+    all-zero placeholder it preserves token overlap so cosine search remains
+    functional when no external embedding provider is configured.
+    """
+    if not isinstance(texts, list):
+        texts = [texts]
+
+    embeddings: list[list[float]] = []
+    for text in texts:
+        vector = [0.0] * 8
+        tokens = re.findall(r"[a-z0-9]+", str(text).lower())
+        for token in tokens:
+            digest = hashlib.blake2b(token.encode("utf-8"), digest_size=1).digest()
+            index = digest[0] % len(vector)
+            vector[index] += 1.0
+        norm = math.sqrt(sum(value * value for value in vector))
+        if norm:
+            vector = [value / norm for value in vector]
+        embeddings.append(vector)
+    return embeddings
+
+
+async def _ollama_embed(texts: list[str], **kwargs) -> list[list[float]]:
+    ollama = importlib.import_module("ollama")
+    model = kwargs.get("model") or kwargs.get("model_name") or DEFAULT_EMBEDDING_MODEL
+    if not isinstance(texts, list):
+        texts = [texts]
+    host = os.getenv("OLLAMA_BASE_URL") or os.getenv("OLLAMA_HOST")
+    if os.getenv("GRAPHRAG_OLLAMA_TASK_PREFIXES") == "v1":
+        if model.startswith("nomic-embed-text"):
+            prefix = "search_query: " if kwargs.get("context") == "query" else "search_document: "
+            texts = [prefix + text for text in texts]
+        elif model.startswith("mxbai-embed-large") and kwargs.get("context") == "query":
+            texts = ["Represent this sentence for searching relevant passages: " + text for text in texts]
+    client = ollama.AsyncClient(**({"host": host} if host else {}))
+    try:
+        if hasattr(client, "embed"):
+            response = await client.embed(model=model, input=texts)
+            return response["embeddings"]
+        # Older clients expose only the single-text endpoint.
+        embeddings: list[list[float]] = []
+        for text in texts:
+            response = await client.embeddings(model=model, prompt=text)
+            embeddings.append(response["embedding"])
+        return embeddings
+    finally:
+        if hasattr(client, "close"):
+            await client.close()
+
+
+async def _openai_embed(texts: list[str], **kwargs) -> list[list[float]]:
+    try:
+        from openai import AsyncOpenAI
+    except Exception as exc:
+        raise ImportError("OpenAI SDK not installed. Run: pip install openai") from exc
+    api_key = os.getenv("GRAPHRAG_OPENAI_API_KEY") or os.getenv("OPENAI_API_KEY")
+    if not api_key:
+        raise ValueError("OpenAI API key missing. Set GRAPHRAG_OPENAI_API_KEY or OPENAI_API_KEY.")
+    model = kwargs.get("model") or kwargs.get("model_name") or DEFAULT_OPENAI_EMBEDDING_MODEL
+    if not isinstance(texts, list):
+        texts = [texts]
+    request = {"model": model, "input": texts}
+    if model.startswith("text-embedding-3"):
+        request["dimensions"] = DEFAULT_EMBEDDING_DIM
+    async with AsyncOpenAI(api_key=api_key) as client:
+        response = await client.embeddings.create(**request)
+    return [item.embedding for item in response.data]
+
+
+async def _cohere_embed(texts: list[str], **kwargs) -> list[list[float]]:
+    try:
+        import cohere
+    except Exception as exc:
+        raise ImportError("Cohere SDK not installed. Run: pip install cohere") from exc
+    api_key = os.getenv("GRAPHRAG_COHERE_API_KEY") or os.getenv("COHERE_API_KEY")
+    if not api_key:
+        raise ValueError("Cohere API key missing. Set GRAPHRAG_COHERE_API_KEY or COHERE_API_KEY.")
+    model = kwargs.get("model") or kwargs.get("model_name") or DEFAULT_COHERE_EMBEDDING_MODEL
+    if not isinstance(texts, list):
+        texts = [texts]
+    input_type = "search_query" if kwargs.get("context") == "query" else "search_document"
+    async with cohere.AsyncClient(api_key) as client:
+        response = await client.embed(texts=texts, model=model, input_type=input_type)
+    embeddings = response.get("embeddings") if isinstance(response, dict) else response.embeddings
+    return list(embeddings or [])
+
+
+def build_default_embedding_func(*, probe: bool = True) -> Any:
+    """Build the EmbeddingFunc for the configured provider.
+
+    For the ollama provider this performs a NETWORK round-trip to probe the
+    model's embedding dimension — call it from server startup (or a script's
+    main path), never at module import.
+    """
+    from core.storage.base import EmbeddingFunc
+
+    provider = DEFAULT_EMBEDDING_PROVIDER.lower()
+    if provider == "fastembed":
+        from core.local_embeddings import build_embedding
+        directory = os.getenv("GRAPHRAG_LOCAL_MODEL_DIR")
+        return build_embedding(Path(directory) if directory else None)
+    if provider == "ollama":
+        # Try to auto-detect embedding dimension from the Ollama model.
+        # If detection fails, fall back to the configured default.
+        embedding = EmbeddingFunc(
+            embedding_dim=DEFAULT_EMBEDDING_DIM,
+            max_token_size=DEFAULT_EMBEDDING_MAX_TOKENS,
+            func=_ollama_embed,
+            model_name=DEFAULT_EMBEDDING_MODEL,
+            provider="ollama",
+            supports_asymmetric=os.getenv("GRAPHRAG_OLLAMA_TASK_PREFIXES") == "v1",
+            model_revision=os.getenv("GRAPHRAG_EMBEDDING_REVISION") or None,
+        )
+        if probe:
+            try:
+                asyncio.get_running_loop()
+            except RuntimeError:
+                asyncio.run(probe_ollama_embedding(embedding))
+            else:
+                # Compatibility for synchronous scripts called inside a loop.
+                # The server uses the async probe directly, without a thread.
+                import concurrent.futures
+                with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
+                    pool.submit(lambda: asyncio.run(probe_ollama_embedding(embedding))).result()
+        return embedding
+    if provider == "openai":
+        return EmbeddingFunc(
+            embedding_dim=DEFAULT_EMBEDDING_DIM,
+            max_token_size=DEFAULT_EMBEDDING_MAX_TOKENS,
+            func=_openai_embed,
+            model_name=DEFAULT_OPENAI_EMBEDDING_MODEL,
+            provider="openai",
+            model_revision=os.getenv("GRAPHRAG_EMBEDDING_REVISION") or None,
+        )
+    if provider == "cohere":
+        return EmbeddingFunc(
+            embedding_dim=DEFAULT_EMBEDDING_DIM,
+            max_token_size=DEFAULT_EMBEDDING_MAX_TOKENS,
+            func=_cohere_embed,
+            model_name=DEFAULT_COHERE_EMBEDDING_MODEL,
+            provider="cohere",
+            supports_asymmetric=True,
+            model_revision=os.getenv("GRAPHRAG_EMBEDDING_REVISION") or None,
+        )
+    if provider == "anthropic":
+        raise ValueError("Anthropic does not provide embeddings. Use openai, cohere, or ollama.")
+    if provider == "fallback":
+        return EmbeddingFunc(
+            embedding_dim=8,
+            max_token_size=DEFAULT_EMBEDDING_MAX_TOKENS,
+            func=_fallback_embed,
+            model_name="fallback",
+            provider="fallback",
+            model_revision="lexical-v1",
+        )
+    raise ValueError(f"Unsupported embedding provider: {DEFAULT_EMBEDDING_PROVIDER}")
+
+
+async def probe_ollama_embedding(embedding: Any) -> None:
+    timeout = float(os.getenv("GRAPHRAG_EMBEDDING_PROBE_TIMEOUT", "15"))
+    if not math.isfinite(timeout) or timeout <= 0:
+        raise ValueError("embedding probe timeout must be finite and greater than zero")
+    try:
+        async with asyncio.timeout(timeout):
+            vectors = await _ollama_embed(["test"], model=embedding.model_name)
+        if not isinstance(vectors, list) or len(vectors) != 1 or not vectors[0]:
+            raise ValueError("Ollama embedding probe returned no vectors")
+        if not all(math.isfinite(float(value)) for value in vectors[0]):
+            raise ValueError("Ollama embedding probe returned non-finite vectors")
+        embedding.embedding_dim = len(vectors[0])
+        embedding.initialization_error = None
+    except Exception as exc:
+        embedding.initialization_error = f"Ollama embedding probe failed for model {embedding.model_name}: {type(exc).__name__}: {exc}"
+        logger.warning("%s; using configured dimension %d", embedding.initialization_error, embedding.embedding_dim)
+
+
+
+def build_global_config(
+    *,
+    working_dir: str | None = None,
+    llm_model_func: Any = None,              # LLM function to call: can be OpenAI, Claude, local model, or None (no LLM)
+    embedding_func: Any = None,              # Embedding function: converts text to vectors for similarity search
+    tokenizer: Any = None,                   # Token counter: counts tokens in text (uses tiktoken or fallback)
+    extra: dict[str, Any] | None = None,     # Additional custom config values
+) -> dict[str, Any]:
+    """
+    Build a global configuration dictionary used throughout the system.
+    
+    This function creates a single config object passed to ALL functions that need settings.
+    Instead of hardcoding values in each file, all functions read from this global_config dict.
+    """
+    base_dir = Path(working_dir or os.getenv("GRAPHRAG_MCP_WORKDIR", "./graphrag_mcp_data"))
+    config: dict[str, Any] = {
+        # ====================================================================
+        # CORE PLUGGABLE FUNCTIONS (can be None or custom implementations)
+        # ====================================================================
+        "working_dir": str(base_dir),                    # Where to store graph/vector DB data
+        "input_dir": os.getenv("GRAPHRAG_INPUT_DIR", "extractions"),
+        "reconciliation_dir": os.getenv("GRAPHRAG_RECONCILIATION_DIR", str(base_dir / "reconciled")),
+        "allow_partial_ingest": os.getenv("GRAPHRAG_ALLOW_PARTIAL_INGEST", "false").lower() == "true",
+        "llm_model_func": llm_model_func,                # LLM to use (defaults to None = no LLM)
+        "embedding_func": embedding_func,                # Embedding model (converts text→vectors)
+        "tokenizer": tokenizer,                          # Token counter (GPT-4o-mini or fallback)
+        
+        # ====================================================================
+        # VECTOR EMBEDDING SETTINGS (used by storage/ and query.py)
+        # Controls how text is converted to vectors for similarity search
+        # ====================================================================
+        "embedding_batch_num": 8,                        # Batch size for embedding operations
+        "vector_db_storage_cls_kwargs": {
+            "cosine_better_than_threshold": 0.6,         # Min similarity threshold for results
+        },
+        
+        # ====================================================================
+        # LLM CACHING SETTINGS (used by utils.py and summary.py)
+        # Caches LLM responses to avoid redundant API calls
+        # ====================================================================
+        "enable_llm_cache": True,                        # Cache query/summarization results
+        "enable_llm_cache_for_entity_extract": True,     # Cache extraction-time LLM calls
+        
+        # ====================================================================
+        # SUMMARY & DESCRIPTION SETTINGS (used by summary.py, merge.py)
+        # Controls how entity/relationship descriptions are merged. Compression
+        # itself is always deferred to the MCP-driving agent — see SUMMARY_MARKER
+        # comment above and preciso_mcp/tools/pending_summaries_tool.py.
+        # ====================================================================
+        "summary_context_size": DEFAULT_SUMMARY_CONTEXT_TOKENS,     # Field size (tokens) that triggers compression
+        "summary_max_tokens": DEFAULT_SUMMARY_MAX_TOKENS,           # Max tokens in an agent-submitted summary
+        "raw_tail_size": DEFAULT_RAW_TAIL_SIZE,                     # Verbatim recent descriptions kept outside the rolling summary
+
+        # ====================================================================
+        # TOKEN LIMITS (used by utils.py, merge.py, query.py)
+        # Prevents text from exceeding API token limits
+        # ====================================================================
+        "embedding_token_limit": int(os.getenv("GRAPHRAG_EMBEDDING_TOKEN_LIMIT", "0")) or None,
+        "max_entity_tokens": DEFAULT_MAX_ENTITY_TOKENS,  # Max tokens per entity description
+        "max_relation_tokens": DEFAULT_MAX_RELATION_TOKENS,  # Max tokens per relation description
+        "max_total_tokens": DEFAULT_MAX_TOTAL_TOKENS,    # Total token budget per query
+        
+        # ====================================================================
+        # KNOWLEDGE GRAPH SETTINGS (used by merge.py, query.py)
+        # Controls entity/relation retrieval and filtering
+        # ====================================================================
+        "kg_evidence_top_k": DEFAULT_KG_EVIDENCE_TOP_K,        # Global cap after all evidence sources are merged
+        "kg_evidence_min_similarity": DEFAULT_KG_EVIDENCE_MIN_SIMILARITY,  # Reject weak evidence
+        "yoy_signal_phrases": DEFAULT_YOY_SIGNAL_PHRASES,      # Comparison-query phrases (query.py mode upgrade)
+        
+        # ====================================================================
+        # SOURCE ID TRACKING (used by merge.py)
+        # Controls how many source references (chunk IDs) to keep per entity/relation
+        # When re-ingesting, older sources may be dropped if limit is reached
+        # ====================================================================
+        "max_source_ids_per_entity": 64,                 # Max source chunks to track per entity
+        "max_source_ids_per_relation": 64,               # Max source chunks to track per relation
+        "source_ids_limit_method": DEFAULT_SOURCE_IDS_LIMIT_METHOD,  # "KEEP" old or "FIFO" newest
+        
+        # ====================================================================
+        # FILE PATH TRACKING (used by merge.py)
+        # Limits how many file paths are stored per entity/relation
+        # ====================================================================
+        "max_file_paths": DEFAULT_MAX_FILE_PATHS,        # Max file paths to store
+        "file_path_more_placeholder": DEFAULT_FILE_PATH_MORE_PLACEHOLDER,  # Placeholder for truncated paths
+        
+        # ====================================================================
+        # GRAPH FILTERING (used by query.py)
+        # Controls graph search and result filtering
+        # ====================================================================
+        "max_graph_nodes": 1000,                         # Max nodes to consider in graph traversal
+        "min_rerank_score": 0.0,                         # Min score to include in results
+        
+        # ====================================================================
+        # PROMPTS (used by query.py)
+        # ====================================================================
+        "system_prompt_template": PROMPTS["rag_response"],       # Default system prompt for queries
+    }
+    if extra:
+        config.update(extra)
+    return config

core/backup.py

@@ -0,0 +1,134 @@
+"""Offline, verified snapshots of the supported local artifact set."""
+
+from contextlib import contextmanager
+from datetime import datetime, timezone
+import fcntl
+import hashlib
+import os
+from pathlib import Path
+import shutil
+import stat
+import tempfile
+
+from core.file_io import atomic_artifact_path
+from core.session_lock import graph_directory_owner
+from core.transactions import ARTIFACTS, JOURNAL
+from core.utils import load_json, write_json
+
+MANIFEST = "backup_manifest.json"
+
+
+def _digest(path):
+    if path.is_symlink() or not path.is_file():
+        raise ValueError(f"Backup artifact must be a regular file: {path.name}")
+    digest = hashlib.sha256()
+    size = 0
+    with path.open("rb") as stream:
+        for block in iter(lambda: stream.read(1024 * 1024), b""):
+            digest.update(block)
+            size += len(block)
+    return {"sha256": digest.hexdigest(), "bytes": size}
+
+
+def verify_backup(source):
+    source = Path(source).resolve()
+    if (source / MANIFEST).is_symlink():
+        raise ValueError("Backup manifest must not be a symlink")
+    manifest = load_json(str(source / MANIFEST))
+    if not manifest or manifest.get("format_version") != 1:
+        raise ValueError("Missing or unsupported backup manifest")
+    files = manifest.get("files")
+    if not isinstance(files, dict) or not files or any(name not in ARTIFACTS for name in files):
+        raise ValueError("Invalid backup artifact list")
+    if {p.name for p in source.iterdir()} != set(files) | {MANIFEST}:
+        raise ValueError("Unexpected or missing backup files")
+    for name, expected in files.items():
+        if not isinstance(expected, dict) or _digest(source / name) != expected:
+            raise ValueError(f"Backup checksum mismatch: {name}")
+    return manifest
+
+
+@contextmanager
+def _fresh_directory(destination):
+    """Publish only a complete directory; never intentionally replace a target."""
+    destination = Path(destination).absolute()
+    if destination.exists() or destination.is_symlink():
+        raise FileExistsError(f"Destination already exists: {destination}")
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    staging = Path(tempfile.mkdtemp(prefix=f".{destination.name}-", dir=destination.parent))
+    try:
+        yield staging
+        if destination.exists() or destination.is_symlink():
+            raise FileExistsError(f"Destination appeared during copy: {destination}")
+        os.rename(staging, destination)
+        descriptor = os.open(destination.parent, os.O_RDONLY)
+        try:
+            os.fsync(descriptor)
+        finally:
+            os.close(descriptor)
+    finally:
+        if staging.exists():
+            shutil.rmtree(staging)
+
+
+def _copy(source, destination):
+    # Reject links and special files before reading. O_NOFOLLOW also protects
+    # the final source path component if it changes between inspection/open.
+    if source.is_symlink() or not source.is_file():
+        raise ValueError(f"Artifact must be a regular file: {source.name}")
+    descriptor = os.open(source, os.O_RDONLY | os.O_NOFOLLOW)
+    with os.fdopen(descriptor, "rb") as stream:
+        if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode):
+            raise ValueError(f"Artifact must be a regular file: {source.name}")
+        with atomic_artifact_path(destination) as temporary:
+            with temporary.open("wb") as output:
+                shutil.copyfileobj(stream, output, length=1024 * 1024)
+
+
+def backup_graph(source, destination):
+    source = Path(source).resolve()
+    if not source.is_dir():
+        raise ValueError("Source graph directory does not exist")
+    target = Path(destination).absolute()
+    if target.resolve().is_relative_to(source):
+        raise ValueError("Backup destination must be outside the graph directory")
+    # Stop the server first. These nonblocking locks also reject standalone
+    # mutations that use the session lock without owning a server runtime.
+    with graph_directory_owner(str(source)):
+        descriptor = os.open(source / ".preciso-ingestion.lock", os.O_CREAT | os.O_RDWR, 0o600)
+        try:
+            try:
+                fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            except BlockingIOError as exc:
+                raise RuntimeError("Graph mutation is active; stop writers before backup") from exc
+            if (source / JOURNAL).exists() or (source / JOURNAL).is_symlink():
+                raise RuntimeError("Unfinished transaction; recover the graph before backup")
+            files = [name for name in ARTIFACTS if (source / name).exists() or (source / name).is_symlink()]
+            if not files:
+                raise ValueError("No persisted graph artifacts to back up")
+            with _fresh_directory(target) as staging:
+                for name in files:
+                    _copy(source / name, staging / name)
+                manifest = {
+                    "format_version": 1,
+                    "created_at": datetime.now(timezone.utc).isoformat(),
+                    "files": {name: _digest(staging / name) for name in files},
+                }
+                write_json(manifest, str(staging / MANIFEST))
+                verify_backup(staging)
+            return manifest
+        finally:
+            os.close(descriptor)
+
+
+def restore_graph(source, destination):
+    source = Path(source).resolve()
+    manifest = verify_backup(source)
+    if Path(destination).resolve().is_relative_to(source):
+        raise ValueError("Restore destination must be outside the backup directory")
+    with _fresh_directory(destination) as staging:
+        for name, expected in manifest["files"].items():
+            _copy(source / name, staging / name)
+            if _digest(staging / name) != expected:
+                raise ValueError(f"Backup changed during restore: {name}")
+    return manifest