gnosisllm-knowledge 0.3.0__py3-none-any.whl → 0.4.3__py3-none-any.whl

gnosisllm_knowledge/cli/commands/load.py

@@ -2,6 +2,10 @@
 
 Fetches content, chunks it for optimal embedding, and indexes
 into OpenSearch with automatic embedding generation via ingest pipeline.
+
+Note:
+    This library is tenant-agnostic. Multi-tenancy is achieved through index
+    isolation - each tenant should use a separate index (e.g., "knowledge-{account_id}").
 """
 
 from __future__ import annotations
@@ -26,6 +30,14 @@ from gnosisllm_knowledge.chunking.sentence import SentenceChunker
 from gnosisllm_knowledge.cli.display.service import RichDisplayService
 from gnosisllm_knowledge.cli.utils.config import CliConfig
 from gnosisllm_knowledge.core.domain.document import Document, DocumentStatus
+from gnosisllm_knowledge.core.events.emitter import EventEmitter
+from gnosisllm_knowledge.core.events.types import (
+    DiscoveryCompletedEvent,
+    DiscoveryFailedEvent,
+    DiscoveryProgressEvent,
+    DiscoveryStartedEvent,
+    EventType,
+)
 from gnosisllm_knowledge.fetchers.config import NeoreaderConfig
 from gnosisllm_knowledge.fetchers.neoreader import NeoreaderContentFetcher
 from gnosisllm_knowledge.loaders.factory import LoaderFactory
@@ -39,7 +51,6 @@ async def load_command(
     source: str,
     source_type: str | None = None,
     index_name: str = "knowledge",
-    account_id: str | None = None,
     collection_id: str | None = None,
     source_id: str | None = None,
     batch_size: int = 100,
@@ -47,15 +58,22 @@ async def load_command(
     force: bool = False,
     dry_run: bool = False,
     verbose: bool = False,
+    discovery: bool = False,
+    max_depth: int = 3,
+    max_pages: int = 100,
+    same_domain: bool = True,
 ) -> None:
     """Execute the load command.
 
+    Note:
+        Multi-tenancy is achieved through index isolation. Use tenant-specific
+        index names instead (e.g., --index knowledge-tenant-123).
+
     Args:
         display: Display service for output.
         source: URL or sitemap to load content from.
-        source_type: Source type (website, sitemap) or auto-detect.
-        index_name: Target index name.
-        account_id: Multi-tenant account ID.
+        source_type: Source type (website, sitemap, discovery) or auto-detect.
+        index_name: Target index name (use tenant-specific name for isolation).
         collection_id: Collection grouping ID.
         source_id: Source identifier (defaults to URL).
         batch_size: Documents per indexing batch.
@@ -63,6 +81,10 @@ async def load_command(
         force: Delete existing source documents first.
         dry_run: Preview without indexing.
         verbose: Show per-document progress.
+        discovery: Use discovery loader (website crawling) instead of single URL.
+        max_depth: Maximum crawl depth for discovery (default: 3).
+        max_pages: Maximum pages to discover (default: 100).
+        same_domain: Only crawl URLs on the same domain (default: True).
     """
     # Load configuration
     cli_config = CliConfig.from_env()
@@ -70,10 +92,18 @@ async def load_command(
     # Auto-detect source type
     detected_type = source_type
     if not detected_type:
-        if "sitemap" in source.lower() or source.endswith(".xml"):
+        if discovery:
+            detected_type = "discovery"
+        elif "sitemap" in source.lower() or source.endswith(".xml"):
             detected_type = "sitemap"
         else:
             detected_type = "website"
+    elif discovery and detected_type != "discovery":
+        # --discovery flag overrides explicit type for website URLs
+        display.warning(
+            f"Using discovery loader (--discovery flag overrides --type {detected_type})"
+        )
+        detected_type = "discovery"
 
     # Default source_id to URL
     final_source_id = source_id or source
@@ -85,18 +115,28 @@ async def load_command(
     )
 
     # Show configuration
+    is_auto_detected = not source_type and not discovery
+    type_suffix = " (auto-detected)" if is_auto_detected else ""
     config_rows = [
         ("Source", source[:50] + "..." if len(source) > 50 else source),
-        ("Type", f"{detected_type} {'(auto-detected)' if not source_type else ''}"),
+        ("Type", f"{detected_type}{type_suffix}"),
         ("Target Index", index_name),
         ("Batch Size", str(batch_size)),
-        ("Max URLs", str(max_urls) if detected_type == "sitemap" else "N/A"),
+    ]
+
+    # Add type-specific configuration
+    if detected_type == "sitemap":
+        config_rows.append(("Max URLs", str(max_urls)))
+    elif detected_type == "discovery":
+        config_rows.append(("Max Depth", str(max_depth)))
+        config_rows.append(("Max Pages", str(max_pages)))
+        config_rows.append(("Same Domain", "Yes" if same_domain else "No"))
+
+    config_rows.extend([
         ("Neoreader", cli_config.neoreader_host),
         ("OpenSearch", f"{cli_config.opensearch_host}:{cli_config.opensearch_port}"),
-    ]
+    ])
 
-    if account_id:
-        config_rows.append(("Account ID", account_id))
     if collection_id:
         config_rows.append(("Collection ID", collection_id))
     if force:
@@ -119,9 +159,16 @@ async def load_command(
         display.warning(f"Cannot connect to Neoreader at {cli_config.neoreader_host}")
         display.info("Continuing with fallback HTTP fetcher...")
 
+    # Create event emitter for discovery progress tracking
+    event_emitter = EventEmitter()
+
     # Create loader
     chunker = SentenceChunker()
-    loader_factory = LoaderFactory(fetcher=fetcher, chunker=chunker)
+    loader_factory = LoaderFactory(
+        fetcher=fetcher,
+        chunker=chunker,
+        event_emitter=event_emitter,
+    )
 
     try:
         loader = loader_factory.create(detected_type)
@@ -151,9 +198,48 @@ async def load_command(
         )
         sys.exit(1)
 
-    # Load documents
+    # Build loader options for discovery
+    loader_options: dict = {}
+    if detected_type == "discovery":
+        loader_options = {
+            "max_depth": max_depth,
+            "max_pages": max_pages,
+            "same_domain": same_domain,
+        }
+
+    # Load documents with discovery progress display
     documents: list[Document] = []
     url_count = 0
+    discovery_state: dict = {"started": False, "completed": False, "job_id": None}
+
+    # Register discovery event handlers for Rich display
+    def _on_discovery_started(event: DiscoveryStartedEvent) -> None:
+        discovery_state["started"] = True
+        discovery_state["job_id"] = event.job_id
+
+    def _on_discovery_progress(event: DiscoveryProgressEvent) -> None:
+        # Update will be handled in the progress context
+        discovery_state["percent"] = event.percent
+        discovery_state["pages_crawled"] = event.pages_crawled
+        discovery_state["urls_discovered"] = event.urls_discovered
+        discovery_state["current_depth"] = event.current_depth
+        discovery_state["message"] = event.message
+
+    def _on_discovery_completed(event: DiscoveryCompletedEvent) -> None:
+        discovery_state["completed"] = True
+        discovery_state["urls_count"] = event.urls_count
+        discovery_state["duration_seconds"] = event.duration_seconds
+
+    def _on_discovery_failed(event: DiscoveryFailedEvent) -> None:
+        discovery_state["failed"] = True
+        discovery_state["error"] = event.error
+
+    # Register discovery event handlers
+    if detected_type == "discovery":
+        event_emitter.add_handler(EventType.DISCOVERY_STARTED, _on_discovery_started)
+        event_emitter.add_handler(EventType.DISCOVERY_PROGRESS, _on_discovery_progress)
+        event_emitter.add_handler(EventType.DISCOVERY_COMPLETED, _on_discovery_completed)
+        event_emitter.add_handler(EventType.DISCOVERY_FAILED, _on_discovery_failed)
 
     with Progress(
         SpinnerColumn(),
@@ -164,9 +250,32 @@ async def load_command(
         TimeElapsedColumn(),
         console=display.console,
     ) as progress:
-        load_task = progress.add_task("Loading content...", total=None)
+        # Add task for discovery phase (if applicable)
+        if detected_type == "discovery":
+            discovery_task = progress.add_task(
+                "Discovering URLs...",
+                total=max_pages,
+            )
+
+        load_task = progress.add_task("Loading content...", total=None, visible=False)
+
+        async for doc in loader.load_streaming(source, **loader_options):
+            # Update discovery progress if available
+            if detected_type == "discovery":
+                if discovery_state.get("started") and not discovery_state.get("completed"):
+                    pages = discovery_state.get("pages_crawled", 0)
+                    urls = discovery_state.get("urls_discovered", 0)
+                    depth = discovery_state.get("current_depth", 0)
+                    progress.update(
+                        discovery_task,
+                        completed=pages,
+                        description=f"Discovering... (depth {depth}, {urls} URLs found)",
+                    )
+                elif discovery_state.get("completed"):
+                    # Hide discovery task and show load task
+                    progress.update(discovery_task, visible=False)
+                    progress.update(load_task, visible=True)
 
-        async for doc in loader.load_streaming(source):
             documents.append(doc)
             url_count += 1
             progress.update(load_task, advance=1, description=f"Loading... ({url_count} docs)")
@@ -176,6 +285,13 @@ async def load_command(
 
         progress.update(load_task, completed=url_count)
 
+    # Show discovery summary if applicable
+    if detected_type == "discovery" and discovery_state.get("completed"):
+        display.success(
+            f"Discovered {discovery_state.get('urls_count', 0)} URLs "
+            f"in {discovery_state.get('duration_seconds', 0):.1f}s"
+        )
+
     display.success(f"Loaded {len(documents)} documents")
 
     if not documents:
@@ -223,7 +339,6 @@ async def load_command(
                     url=doc.url,
                     title=doc.title,
                     source=final_source_id,
-                    account_id=account_id,
                     collection_id=collection_id,
                     source_id=final_source_id,
                     metadata=doc.metadata,
@@ -238,7 +353,6 @@ async def load_command(
                         url=doc.url,
                         title=doc.title,
                         source=final_source_id,
-                        account_id=account_id,
                         collection_id=collection_id,
                         source_id=final_source_id,
                         chunk_index=i,
@@ -267,8 +381,11 @@ async def load_command(
     )
 
     try:
-        # Create indexer config
+        # Create indexer config from environment, preserving k-NN and other settings
+        # This ensures proper vector mappings are used when creating indices
+        base_config = OpenSearchConfig.from_env()
         opensearch_config = OpenSearchConfig(
+            # CLI/CliConfig overrides
             host=cli_config.opensearch_host,
             port=cli_config.opensearch_port,
             username=cli_config.opensearch_username,
@@ -276,7 +393,21 @@ async def load_command(
             use_ssl=cli_config.opensearch_use_ssl,
             verify_certs=cli_config.opensearch_verify_certs,
             model_id=cli_config.opensearch_model_id,
-            ingest_pipeline_name=cli_config.opensearch_pipeline_name,
+            ingest_pipeline_name=cli_config.opensearch_ingest_pipeline_name,
+            # Preserve env-based k-NN settings for proper index mappings
+            embedding_model=base_config.embedding_model,
+            embedding_dimension=base_config.embedding_dimension,
+            embedding_field=base_config.embedding_field,
+            knn_engine=base_config.knn_engine,
+            knn_space_type=base_config.knn_space_type,
+            knn_algo_param_ef_search=base_config.knn_algo_param_ef_search,
+            knn_algo_param_ef_construction=base_config.knn_algo_param_ef_construction,
+            knn_algo_param_m=base_config.knn_algo_param_m,
+            # Preserve other settings
+            index_prefix=base_config.index_prefix,
+            number_of_shards=base_config.number_of_shards,
+            number_of_replicas=base_config.number_of_replicas,
+            search_pipeline_name=base_config.search_pipeline_name,
         )
 
         indexer = OpenSearchIndexer(client, opensearch_config)
@@ -314,6 +445,7 @@ async def load_command(
 
         indexed_count = 0
         failed_count = 0
+        all_errors: list[dict] = []
 
         with Progress(
             SpinnerColumn(),
@@ -333,6 +465,8 @@ async def load_command(
                 result = await indexer.bulk_index(batch, index_name, batch_size=batch_size)
                 indexed_count += result.indexed_count
                 failed_count += result.failed_count
+                if result.errors:
+                    all_errors.extend(result.errors)
 
                 progress.update(index_task, advance=len(batch))
 
@@ -354,12 +488,28 @@ async def load_command(
                 style="success",
             )
         else:
+            # Build error details section
+            error_details = ""
+            if all_errors:
+                error_details = "\n\n[bold red]Error Details:[/bold red]\n"
+                for i, err in enumerate(all_errors[:5], 1):  # Show first 5 errors
+                    if isinstance(err, dict):
+                        error_type = err.get("error", {}).get("type", "unknown") if isinstance(err.get("error"), dict) else str(err.get("error", "unknown"))
+                        error_reason = err.get("error", {}).get("reason", "No reason provided") if isinstance(err.get("error"), dict) else str(err.get("error", "No details"))
+                        doc_id = err.get("_id", "unknown")
+                        error_details += f"  {i}. [dim]Doc {doc_id}:[/dim] {error_type} - {error_reason}\n"
+                    else:
+                        error_details += f"  {i}. {err}\n"
+                if len(all_errors) > 5:
+                    error_details += f"  ... and {len(all_errors) - 5} more errors\n"
+
             display.panel(
                 f"Documents Loaded:     [cyan]{len(documents)}[/cyan]\n"
                 f"Chunks Created:       [cyan]{len(chunked_documents)}[/cyan]\n"
                 f"Documents Indexed:    [green]{indexed_count}[/green]\n"
                 f"Documents Failed:     [red]{failed_count}[/red]\n"
-                f"Index:                [cyan]{index_name}[/cyan]",
+                f"Index:                [cyan]{index_name}[/cyan]"
+                f"{error_details}",
                 title="Loading Complete (with errors)",
                 style="warning",
             )

gnosisllm_knowledge/cli/commands/memory.py

@@ -608,6 +608,14 @@ async def session_list_command(
         limit: Maximum sessions to return.
         json_output: Output as JSON.
     """
+    # Show warning about OpenSearch sessions bug
+    if not json_output:
+        display.warning(
+            "[yellow]⚠ Known Issue:[/yellow] Sessions have a bug in OpenSearch 3.4.0. "
+            "The sessions index is not auto-created. See docs/memory.md for details."
+        )
+        display.newline()
+
     cli_config = CliConfig.from_env()
     memory = Memory.from_config(_create_memory_config(cli_config))
 
@@ -638,6 +646,8 @@ async def session_list_command(
                 for s in sessions
             ],
             "total": len(sessions),
+            "warning": "Sessions have a known bug in OpenSearch 3.4.0. "
+            "The sessions index is not auto-created. See docs/memory.md for details.",
         }
         print(json.dumps(output, indent=2))
         return

gnosisllm_knowledge/cli/commands/search.py

@@ -5,6 +5,10 @@ Supports multiple search modes:
 - keyword: Traditional BM25 text matching
 - hybrid: Combined semantic + keyword (default, best results)
 - agentic: AI-powered search with reasoning and answer generation
+
+Note:
+    This library is tenant-agnostic. Multi-tenancy is achieved through index
+    isolation - each tenant should use a separate index (e.g., "knowledge-{account_id}").
 """
 
 from __future__ import annotations
@@ -44,7 +48,6 @@ async def search_command(
     index_name: str = "knowledge",
     limit: int = 5,
     offset: int = 0,
-    account_id: str | None = None,
     collection_ids: str | None = None,
     source_ids: str | None = None,
     min_score: float = 0.0,
@@ -55,14 +58,17 @@ async def search_command(
 ) -> None:
     """Execute the search command.
 
+    Note:
+        Multi-tenancy is achieved through index isolation. Use tenant-specific
+        index names instead (e.g., --index knowledge-tenant-123).
+
     Args:
         display: Display service for output.
         query: Search query text.
         mode: Search mode (semantic, keyword, hybrid, agentic).
-        index_name: Index to search.
+        index_name: Index to search (use tenant-specific name for isolation).
         limit: Maximum results to return.
         offset: Pagination offset.
-        account_id: Filter by account ID.
         collection_ids: Filter by collection IDs (comma-separated).
         source_ids: Filter by source IDs (comma-separated).
         min_score: Minimum score threshold.
@@ -86,7 +92,6 @@ async def search_command(
             query=query or "",
             index_name=index_name,
             agent_type="flow",  # Default to flow for single queries
-            account_id=account_id,
             collection_ids=collection_ids,
             source_ids=source_ids,
             limit=limit,
@@ -117,7 +122,6 @@ async def search_command(
             index_name=index_name,
             mode=mode,
             limit=limit,
-            account_id=account_id,
             collection_ids=collection_ids,
             source_ids=source_ids,
             min_score=min_score,
@@ -146,7 +150,6 @@ async def search_command(
         index_name=index_name,
         limit=limit,
         offset=offset,
-        account_id=account_id,
         collection_ids=collection_ids,
         source_ids=source_ids,
         min_score=min_score,
@@ -167,7 +170,6 @@ async def _execute_search(
     index_name: str,
     limit: int,
     offset: int,
-    account_id: str | None,
     collection_ids: str | None,
     source_ids: str | None,
     min_score: float,
@@ -214,7 +216,6 @@ async def _execute_search(
             mode=_get_search_mode(mode),
             limit=limit,
             offset=offset,
-            account_id=account_id,
             collection_ids=collection_list,
             source_ids=source_list,
             min_score=min_score,
@@ -315,7 +316,6 @@ async def _interactive_search(
     index_name: str,
     mode: str,
     limit: int,
-    account_id: str | None,
     collection_ids: str | None,
     source_ids: str | None,
     min_score: float,
@@ -396,7 +396,6 @@ async def _interactive_search(
                     mode=_get_search_mode(mode),
                     limit=limit,
                     offset=0,
-                    account_id=account_id,
                     collection_ids=collection_list,
                     source_ids=source_list,
                     min_score=min_score,

gnosisllm_knowledge/cli/commands/setup.py

@@ -90,8 +90,11 @@ async def setup_command(
 
     display.newline()
 
-    # Create OpenSearch config
+    # Create OpenSearch config from environment, then override with CLI args
+    # This ensures all env vars (including pipeline names) are respected
+    base_config = OpenSearchConfig.from_env()
     opensearch_config = OpenSearchConfig(
+        # CLI overrides (if provided)
         host=final_host,
         port=final_port,
         username=final_username,
@@ -101,6 +104,27 @@ async def setup_command(
         openai_api_key=cli_config.openai_api_key,
         embedding_model=cli_config.openai_embedding_model,
         embedding_dimension=cli_config.openai_embedding_dimension,
+        # Preserve env-based config for pipelines and other settings
+        ingest_pipeline_name=base_config.ingest_pipeline_name,
+        search_pipeline_name=base_config.search_pipeline_name,
+        index_prefix=base_config.index_prefix,
+        model_id=base_config.model_id,
+        model_group_id=base_config.model_group_id,
+        embedding_field=base_config.embedding_field,
+        # k-NN settings
+        knn_engine=base_config.knn_engine,
+        knn_space_type=base_config.knn_space_type,
+        knn_algo_param_ef_search=base_config.knn_algo_param_ef_search,
+        knn_algo_param_ef_construction=base_config.knn_algo_param_ef_construction,
+        knn_algo_param_m=base_config.knn_algo_param_m,
+        # Index settings
+        number_of_shards=base_config.number_of_shards,
+        number_of_replicas=base_config.number_of_replicas,
+        refresh_interval=base_config.refresh_interval,
+        # Agentic settings
+        agentic_llm_model=base_config.agentic_llm_model,
+        agentic_max_iterations=base_config.agentic_max_iterations,
+        agentic_timeout_seconds=base_config.agentic_timeout_seconds,
     )
 
     # Create OpenSearch client

gnosisllm_knowledge/cli/utils/config.py

@@ -27,7 +27,7 @@ class CliConfig:
     opensearch_verify_certs: bool = False
     opensearch_model_id: str | None = None
     opensearch_index_name: str = "knowledge"
-    opensearch_pipeline_name: str = "gnosisllm-ingest-pipeline"
+    opensearch_ingest_pipeline_name: str = "gnosisllm-ingest-pipeline"
     opensearch_search_pipeline_name: str = "gnosisllm-search-pipeline"
 
     # OpenAI
@@ -78,11 +78,11 @@ class CliConfig:
             == "true",
             opensearch_model_id=os.getenv("OPENSEARCH_MODEL_ID"),
             opensearch_index_name=os.getenv("OPENSEARCH_INDEX_NAME", "knowledge"),
-            opensearch_pipeline_name=os.getenv(
-                "OPENSEARCH_PIPELINE_NAME", "gnosisllm-ingest-pipeline"
+            opensearch_ingest_pipeline_name=os.getenv(
+                "OPENSEARCH_INGEST_PIPELINE", "gnosisllm-ingest-pipeline"
             ),
             opensearch_search_pipeline_name=os.getenv(
-                "OPENSEARCH_SEARCH_PIPELINE_NAME", "gnosisllm-search-pipeline"
+                "OPENSEARCH_SEARCH_PIPELINE", "gnosisllm-search-pipeline"
             ),
             openai_api_key=os.getenv("OPENAI_API_KEY"),
             openai_embedding_model=os.getenv("OPENAI_EMBEDDING_MODEL", "text-embedding-ada-002"),

gnosisllm_knowledge/core/domain/__init__.py

@@ -1,5 +1,12 @@
 """Domain models - Value objects and entities."""
 
+from gnosisllm_knowledge.core.domain.discovery import (
+    DiscoveredURL,
+    DiscoveryConfig,
+    DiscoveryJobStatus,
+    DiscoveryProgress,
+    DiscoveryStats,
+)
 from gnosisllm_knowledge.core.domain.document import Document, DocumentStatus, TextChunk
 from gnosisllm_knowledge.core.domain.memory import (
     ContainerConfig,
@@ -41,6 +48,12 @@ from gnosisllm_knowledge.core.domain.search import (
 from gnosisllm_knowledge.core.domain.source import SourceConfig
 
 __all__ = [
+    # Discovery
+    "DiscoveredURL",
+    "DiscoveryConfig",
+    "DiscoveryJobStatus",
+    "DiscoveryProgress",
+    "DiscoveryStats",
     # Document
     "Document",
     "DocumentStatus",