chunksilo 2.0.0__py3-none-any.whl

chunksilo/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: Apache-2.0
+"""ChunkSilo - Local RAG-based semantic document search."""
+
+__version__ = "2.0.0"

chunksilo/__main__.py

@@ -0,0 +1,3 @@
+from chunksilo.cli import main
+
+main()

chunksilo/cfgload.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""
+Shared configuration loading for ChunkSilo.
+
+Loads configuration from config.yaml, searching in standard locations.
+"""
+import logging
+import os
+import yaml
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def _find_config() -> Path:
+    """Find config.yaml using a priority-based search.
+
+    Search order:
+    1. CHUNKSILO_CONFIG environment variable
+    2. ./config.yaml (current working directory)
+    3. ~/.config/chunksilo/config.yaml (XDG standard)
+    """
+    env_path = os.environ.get("CHUNKSILO_CONFIG")
+    if env_path:
+        return Path(env_path)
+
+    cwd_path = Path.cwd() / "config.yaml"
+    if cwd_path.exists():
+        return cwd_path
+
+    xdg_path = Path.home() / ".config" / "chunksilo" / "config.yaml"
+    if xdg_path.exists():
+        return xdg_path
+
+    # Return cwd path as default (will fall through to defaults if not found)
+    return cwd_path
+
+
+CONFIG_PATH = _find_config()
+
+_DEFAULTS: dict[str, Any] = {
+    "indexing": {
+        "directories": ["./data"],
+        "defaults": {
+            "include": ["**/*.pdf", "**/*.md", "**/*.txt", "**/*.docx", "**/*.doc"],
+            "exclude": [],
+            "recursive": True,
+        },
+        "chunk_size": 1600,
+        "chunk_overlap": 200,
+    },
+    "retrieval": {
+        "embed_model_name": "BAAI/bge-small-en-v1.5",
+        "embed_top_k": 20,
+        "rerank_model_name": "ms-marco-MiniLM-L-12-v2",
+        "rerank_top_k": 5,
+        "rerank_candidates": 100,
+        "score_threshold": 0.1,
+        "recency_boost": 0.3,
+        "recency_half_life_days": 365,
+        "bm25_similarity_top_k": 10,
+        "offline": False,
+    },
+    "confluence": {
+        "url": "",
+        "username": "",
+        "api_token": "",
+        "timeout": 10.0,
+        "max_results": 30,
+    },
+    "ssl": {
+        "ca_bundle_path": "",
+    },
+    "storage": {
+        "storage_dir": "./storage",
+        "model_cache_dir": "./models",
+    },
+}
+
+_config_cache: dict[str, Any] | None = None
+
+
+def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
+    """Deep merge override into base, returning a new dict."""
+    result = base.copy()
+    for key, value in override.items():
+        if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+            result[key] = _deep_merge(result[key], value)
+        else:
+            result[key] = value
+    return result
+
+
+def load_config(config_path: Path | None = None) -> dict[str, Any]:
+    """Load configuration from YAML file with defaults.
+
+    Args:
+        config_path: Optional path to config file. If None, uses default CONFIG_PATH.
+                    Results are cached only when using the default path.
+
+    Returns:
+        Configuration dictionary with defaults merged in.
+    """
+    global _config_cache
+
+    # Return cached config if available (only for default path)
+    if _config_cache is not None and config_path is None:
+        return _config_cache
+
+    path = config_path or CONFIG_PATH
+
+    if not path.exists():
+        logger.info("Config file not found at %s; using built-in defaults", path)
+        return _DEFAULTS.copy()
+
+    logger.info("Using config: %s", path)
+
+    with open(path, "r", encoding="utf-8") as f:
+        user_config = yaml.safe_load(f) or {}
+
+    result = _deep_merge(_DEFAULTS, user_config)
+
+    # Cache result only for default path
+    if config_path is None:
+        _config_cache = result
+
+    return result
+
+
+def get(key: str, default: Any = None) -> Any:
+    """Get a config value by dot-notation key.
+
+    Args:
+        key: Dot-separated key path (e.g., 'retrieval.embed_top_k')
+        default: Value to return if key not found
+
+    Returns:
+        Configuration value or default.
+
+    Example:
+        >>> get('retrieval.embed_top_k')
+        20
+        >>> get('storage.storage_dir')
+        './storage'
+    """
+    config = load_config()
+    keys = key.split(".")
+    value: Any = config
+    for k in keys:
+        if isinstance(value, dict) and k in value:
+            value = value[k]
+        else:
+            return default
+    return value
+
+
+def reload_config() -> dict[str, Any]:
+    """Force reload configuration from disk, clearing the cache."""
+    global _config_cache
+    _config_cache = None
+    return load_config()

chunksilo/cli.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""
+CLI entry point for the chunksilo command.
+
+Usage:
+    chunksilo "query text" [--date-from YYYY-MM-DD] [--date-to YYYY-MM-DD] [--config PATH] [--json]
+    chunksilo --build-index [--config PATH]
+    chunksilo --download-models [--config PATH]
+"""
+import argparse
+import json
+import logging
+import os
+import sys
+from pathlib import Path
+
+
+def main():
+    """Entry point for the `chunksilo` command."""
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+    parser = argparse.ArgumentParser(
+        prog="chunksilo",
+        description="Search indexed documents using ChunkSilo",
+        epilog=(
+            "config file search order (first found wins):\n"
+            "  1. --config PATH argument\n"
+            "  2. CHUNKSILO_CONFIG environment variable\n"
+            "  3. ./config.yaml\n"
+            "  4. ~/.config/chunksilo/config.yaml\n"
+            "  If none found, built-in defaults are used."
+        ),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("query", nargs="?", default=None, help="Search query text")
+    parser.add_argument("--date-from", help="Start date filter (YYYY-MM-DD, inclusive)")
+    parser.add_argument("--date-to", help="End date filter (YYYY-MM-DD, inclusive)")
+    parser.add_argument("--config", help="Path to config.yaml (overrides auto-discovery)")
+    parser.add_argument("--json", action="store_true", help="Output as JSON")
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        help="Show diagnostic messages (model loading, search stats)")
+    parser.add_argument("--build-index", action="store_true",
+                        help="Build or update the search index, then exit")
+    parser.add_argument("--download-models", action="store_true",
+                        help="Download required ML models, then exit")
+
+    args = parser.parse_args()
+
+    log_level = logging.INFO if args.verbose or args.build_index or args.download_models else logging.WARNING
+    logging.basicConfig(level=log_level, format="%(message)s", stream=sys.stderr)
+
+    config_path = Path(args.config) if args.config else None
+
+    if args.build_index or args.download_models:
+        from .index import build_index
+
+        build_index(
+            download_only=args.download_models,
+            config_path=config_path,
+        )
+        return
+
+    if not args.query:
+        parser.error("query is required (or use --build-index / --download-models)")
+
+    from .search import run_search
+
+    result = run_search(
+        query=args.query,
+        date_from=args.date_from,
+        date_to=args.date_to,
+        config_path=config_path,
+    )
+
+    if args.json:
+        print(json.dumps(result, indent=2))
+        return
+
+    # Check for errors
+    if result.get("error"):
+        print(f"Error: {result['error']}", file=sys.stderr)
+        sys.exit(1)
+
+    # Human-readable output
+    matched_files = result.get("matched_files", [])
+    chunks = result.get("chunks", [])
+
+    if matched_files:
+        print(f"\nMatched files ({len(matched_files)}):")
+        for f in matched_files:
+            print(f"  {f.get('uri', 'unknown')}  (score: {f.get('score', 0):.4f})")
+
+    if not chunks:
+        print("\nNo results found.")
+        return
+
+    print(f"\nResults ({len(chunks)}):\n")
+
+    for i, chunk in enumerate(chunks, 1):
+        loc = chunk.get("location", {})
+        uri = loc.get("uri") or "unknown"
+        heading = " > ".join(loc.get("heading_path") or [])
+        score = chunk.get("score", 0)
+
+        print(f"[{i}] {uri}")
+        if heading:
+            print(f"    Heading: {heading}")
+        if loc.get("page"):
+            print(f"    Page: {loc['page']}")
+        if loc.get("line"):
+            print(f"    Line: {loc['line']}")
+        print(f"    Score: {score:.3f}")
+
+        text = chunk.get("text", "")
+        preview = text[:200].replace("\n", " ")
+        if len(text) > 200:
+            preview += "..."
+        print(f"    {preview}")
+        print()
+
+    retrieval_time = result.get("retrieval_time", "")
+    if retrieval_time:
+        print(f"({retrieval_time})")

chunksilo/confluence_html_formatter.py

@@ -0,0 +1,96 @@
+# SPDX-License-Identifier: Apache-2.0
+"""
+Custom HTML formatter for Confluence content.
+
+TEMPORARY FIX: Strips syntax highlighting <span> tags that cause issues
+with markdownify. Remove this file when the upstream issue is fixed.
+
+To remove this workaround:
+1. Delete this file
+2. Remove the import and patch_confluence_reader() call from chunksilo.py
+"""
+
+from bs4 import BeautifulSoup
+
+
+def clean_confluence_html(html: str) -> str:
+    """
+    Pre-process Confluence HTML to remove problematic syntax highlighting spans.
+
+    Confluence wraps code in many <span> tags for syntax highlighting that can
+    cause markdownify to insert unwanted newlines between characters.
+
+    Args:
+        html: Raw HTML string from Confluence
+
+    Returns:
+        Cleaned HTML with syntax highlighting spans unwrapped
+    """
+    if not html:
+        return html
+
+    soup = BeautifulSoup(html, "html.parser")
+
+    # Find and unwrap syntax highlighting spans
+    # These are typically <span> tags with class attributes for highlighting
+    # or inline style attributes for colors
+    for span in soup.find_all("span"):
+        # Check if this looks like a syntax highlighting span
+        span_class = span.get("class", [])
+        span_style = span.get("style", "")
+
+        # Common patterns: spans with color styles, or code-related classes
+        is_syntax_span = (
+            "color" in span_style
+            or "background" in span_style
+            or any("code" in c for c in span_class if isinstance(c, str))
+        )
+
+        if is_syntax_span:
+            span.unwrap()  # Replace span with its contents
+
+    return str(soup)
+
+
+class CleanHtmlTextParser:
+    """
+    Drop-in replacement for llama_index's HtmlTextParser that cleans
+    syntax highlighting spans before conversion.
+    """
+
+    def __init__(self):
+        try:
+            from markdownify import markdownify  # noqa: F401
+        except ImportError:
+            raise ImportError(
+                "`markdownify` package not found, please run `pip install markdownify`"
+            )
+
+    def convert(self, html: str) -> str:
+        from markdownify import markdownify
+
+        if not html:
+            return ""
+
+        # Clean the HTML first
+        cleaned_html = clean_confluence_html(html)
+
+        return markdownify(
+            cleaned_html,
+            heading_style="ATX",
+            bullets="*",
+            strip=["script", "style"],
+        )
+
+
+def patch_confluence_reader():
+    """
+    Monkey-patch the ConfluenceReader to use our clean HTML parser.
+    Call this before creating ConfluenceReader instances.
+    """
+    try:
+        import llama_index.readers.confluence.html_parser as html_parser_module
+
+        html_parser_module.HtmlTextParser = CleanHtmlTextParser
+    except ImportError:
+        pass  # ConfluenceReader not installed