gnosisllm-knowledge 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

gnosisllm_knowledge/cli/commands/load.py

@@ -2,6 +2,10 @@
 
 Fetches content, chunks it for optimal embedding, and indexes
 into OpenSearch with automatic embedding generation via ingest pipeline.
+
+Note:
+    This library is tenant-agnostic. Multi-tenancy is achieved through index
+    isolation - each tenant should use a separate index (e.g., "knowledge-{account_id}").
 """
 
 from __future__ import annotations
@@ -26,6 +30,14 @@ from gnosisllm_knowledge.chunking.sentence import SentenceChunker
 from gnosisllm_knowledge.cli.display.service import RichDisplayService
 from gnosisllm_knowledge.cli.utils.config import CliConfig
 from gnosisllm_knowledge.core.domain.document import Document, DocumentStatus
+from gnosisllm_knowledge.core.events.emitter import EventEmitter
+from gnosisllm_knowledge.core.events.types import (
+    DiscoveryCompletedEvent,
+    DiscoveryFailedEvent,
+    DiscoveryProgressEvent,
+    DiscoveryStartedEvent,
+    EventType,
+)
 from gnosisllm_knowledge.fetchers.config import NeoreaderConfig
 from gnosisllm_knowledge.fetchers.neoreader import NeoreaderContentFetcher
 from gnosisllm_knowledge.loaders.factory import LoaderFactory
@@ -39,7 +51,6 @@ async def load_command(
     source: str,
     source_type: str | None = None,
     index_name: str = "knowledge",
-    account_id: str | None = None,
     collection_id: str | None = None,
     source_id: str | None = None,
     batch_size: int = 100,
@@ -47,15 +58,22 @@ async def load_command(
     force: bool = False,
     dry_run: bool = False,
     verbose: bool = False,
+    discovery: bool = False,
+    max_depth: int = 3,
+    max_pages: int = 100,
+    same_domain: bool = True,
 ) -> None:
     """Execute the load command.
 
+    Note:
+        Multi-tenancy is achieved through index isolation. Use tenant-specific
+        index names instead (e.g., --index knowledge-tenant-123).
+
     Args:
         display: Display service for output.
         source: URL or sitemap to load content from.
-        source_type: Source type (website, sitemap) or auto-detect.
-        index_name: Target index name.
-        account_id: Multi-tenant account ID.
+        source_type: Source type (website, sitemap, discovery) or auto-detect.
+        index_name: Target index name (use tenant-specific name for isolation).
         collection_id: Collection grouping ID.
         source_id: Source identifier (defaults to URL).
         batch_size: Documents per indexing batch.
@@ -63,6 +81,10 @@ async def load_command(
         force: Delete existing source documents first.
         dry_run: Preview without indexing.
         verbose: Show per-document progress.
+        discovery: Use discovery loader (website crawling) instead of single URL.
+        max_depth: Maximum crawl depth for discovery (default: 3).
+        max_pages: Maximum pages to discover (default: 100).
+        same_domain: Only crawl URLs on the same domain (default: True).
     """
     # Load configuration
     cli_config = CliConfig.from_env()
@@ -70,10 +92,18 @@ async def load_command(
     # Auto-detect source type
     detected_type = source_type
     if not detected_type:
-        if "sitemap" in source.lower() or source.endswith(".xml"):
+        if discovery:
+            detected_type = "discovery"
+        elif "sitemap" in source.lower() or source.endswith(".xml"):
             detected_type = "sitemap"
         else:
             detected_type = "website"
+    elif discovery and detected_type != "discovery":
+        # --discovery flag overrides explicit type for website URLs
+        display.warning(
+            f"Using discovery loader (--discovery flag overrides --type {detected_type})"
+        )
+        detected_type = "discovery"
 
     # Default source_id to URL
     final_source_id = source_id or source
@@ -85,18 +115,28 @@ async def load_command(
     )
 
     # Show configuration
+    is_auto_detected = not source_type and not discovery
+    type_suffix = " (auto-detected)" if is_auto_detected else ""
     config_rows = [
         ("Source", source[:50] + "..." if len(source) > 50 else source),
-        ("Type", f"{detected_type} {'(auto-detected)' if not source_type else ''}"),
+        ("Type", f"{detected_type}{type_suffix}"),
         ("Target Index", index_name),
         ("Batch Size", str(batch_size)),
-        ("Max URLs", str(max_urls) if detected_type == "sitemap" else "N/A"),
+    ]
+
+    # Add type-specific configuration
+    if detected_type == "sitemap":
+        config_rows.append(("Max URLs", str(max_urls)))
+    elif detected_type == "discovery":
+        config_rows.append(("Max Depth", str(max_depth)))
+        config_rows.append(("Max Pages", str(max_pages)))
+        config_rows.append(("Same Domain", "Yes" if same_domain else "No"))
+
+    config_rows.extend([
         ("Neoreader", cli_config.neoreader_host),
         ("OpenSearch", f"{cli_config.opensearch_host}:{cli_config.opensearch_port}"),
-    ]
+    ])
 
-    if account_id:
-        config_rows.append(("Account ID", account_id))
     if collection_id:
         config_rows.append(("Collection ID", collection_id))
     if force:
@@ -119,9 +159,16 @@ async def load_command(
         display.warning(f"Cannot connect to Neoreader at {cli_config.neoreader_host}")
         display.info("Continuing with fallback HTTP fetcher...")
 
+    # Create event emitter for discovery progress tracking
+    event_emitter = EventEmitter()
+
     # Create loader
     chunker = SentenceChunker()
-    loader_factory = LoaderFactory(fetcher=fetcher, chunker=chunker)
+    loader_factory = LoaderFactory(
+        fetcher=fetcher,
+        chunker=chunker,
+        event_emitter=event_emitter,
+    )
 
     try:
         loader = loader_factory.create(detected_type)
@@ -151,9 +198,48 @@ async def load_command(
         )
         sys.exit(1)
 
-    # Load documents
+    # Build loader options for discovery
+    loader_options: dict = {}
+    if detected_type == "discovery":
+        loader_options = {
+            "max_depth": max_depth,
+            "max_pages": max_pages,
+            "same_domain": same_domain,
+        }
+
+    # Load documents with discovery progress display
     documents: list[Document] = []
     url_count = 0
+    discovery_state: dict = {"started": False, "completed": False, "job_id": None}
+
+    # Register discovery event handlers for Rich display
+    def _on_discovery_started(event: DiscoveryStartedEvent) -> None:
+        discovery_state["started"] = True
+        discovery_state["job_id"] = event.job_id
+
+    def _on_discovery_progress(event: DiscoveryProgressEvent) -> None:
+        # Update will be handled in the progress context
+        discovery_state["percent"] = event.percent
+        discovery_state["pages_crawled"] = event.pages_crawled
+        discovery_state["urls_discovered"] = event.urls_discovered
+        discovery_state["current_depth"] = event.current_depth
+        discovery_state["message"] = event.message
+
+    def _on_discovery_completed(event: DiscoveryCompletedEvent) -> None:
+        discovery_state["completed"] = True
+        discovery_state["urls_count"] = event.urls_count
+        discovery_state["duration_seconds"] = event.duration_seconds
+
+    def _on_discovery_failed(event: DiscoveryFailedEvent) -> None:
+        discovery_state["failed"] = True
+        discovery_state["error"] = event.error
+
+    # Register discovery event handlers
+    if detected_type == "discovery":
+        event_emitter.add_handler(EventType.DISCOVERY_STARTED, _on_discovery_started)
+        event_emitter.add_handler(EventType.DISCOVERY_PROGRESS, _on_discovery_progress)
+        event_emitter.add_handler(EventType.DISCOVERY_COMPLETED, _on_discovery_completed)
+        event_emitter.add_handler(EventType.DISCOVERY_FAILED, _on_discovery_failed)
 
     with Progress(
         SpinnerColumn(),
@@ -164,9 +250,32 @@ async def load_command(
         TimeElapsedColumn(),
         console=display.console,
     ) as progress:
-        load_task = progress.add_task("Loading content...", total=None)
+        # Add task for discovery phase (if applicable)
+        if detected_type == "discovery":
+            discovery_task = progress.add_task(
+                "Discovering URLs...",
+                total=max_pages,
+            )
+
+        load_task = progress.add_task("Loading content...", total=None, visible=False)
+
+        async for doc in loader.load_streaming(source, **loader_options):
+            # Update discovery progress if available
+            if detected_type == "discovery":
+                if discovery_state.get("started") and not discovery_state.get("completed"):
+                    pages = discovery_state.get("pages_crawled", 0)
+                    urls = discovery_state.get("urls_discovered", 0)
+                    depth = discovery_state.get("current_depth", 0)
+                    progress.update(
+                        discovery_task,
+                        completed=pages,
+                        description=f"Discovering... (depth {depth}, {urls} URLs found)",
+                    )
+                elif discovery_state.get("completed"):
+                    # Hide discovery task and show load task
+                    progress.update(discovery_task, visible=False)
+                    progress.update(load_task, visible=True)
 
-        async for doc in loader.load_streaming(source):
             documents.append(doc)
             url_count += 1
             progress.update(load_task, advance=1, description=f"Loading... ({url_count} docs)")
@@ -176,6 +285,13 @@ async def load_command(
 
         progress.update(load_task, completed=url_count)
 
+    # Show discovery summary if applicable
+    if detected_type == "discovery" and discovery_state.get("completed"):
+        display.success(
+            f"Discovered {discovery_state.get('urls_count', 0)} URLs "
+            f"in {discovery_state.get('duration_seconds', 0):.1f}s"
+        )
+
     display.success(f"Loaded {len(documents)} documents")
 
     if not documents:
@@ -223,7 +339,6 @@ async def load_command(
                     url=doc.url,
                     title=doc.title,
                     source=final_source_id,
-                    account_id=account_id,
                     collection_id=collection_id,
                     source_id=final_source_id,
                     metadata=doc.metadata,
@@ -238,7 +353,6 @@ async def load_command(
                         url=doc.url,
                         title=doc.title,
                         source=final_source_id,
-                        account_id=account_id,
                         collection_id=collection_id,
                         source_id=final_source_id,
                         chunk_index=i,
@@ -267,8 +381,11 @@ async def load_command(
     )
 
     try:
-        # Create indexer config
+        # Create indexer config from environment, preserving k-NN and other settings
+        # This ensures proper vector mappings are used when creating indices
+        base_config = OpenSearchConfig.from_env()
         opensearch_config = OpenSearchConfig(
+            # CLI/CliConfig overrides
             host=cli_config.opensearch_host,
             port=cli_config.opensearch_port,
             username=cli_config.opensearch_username,
@@ -276,7 +393,21 @@ async def load_command(
             use_ssl=cli_config.opensearch_use_ssl,
             verify_certs=cli_config.opensearch_verify_certs,
             model_id=cli_config.opensearch_model_id,
-            ingest_pipeline_name=cli_config.opensearch_pipeline_name,
+            ingest_pipeline_name=cli_config.opensearch_ingest_pipeline_name,
+            # Preserve env-based k-NN settings for proper index mappings
+            embedding_model=base_config.embedding_model,
+            embedding_dimension=base_config.embedding_dimension,
+            embedding_field=base_config.embedding_field,
+            knn_engine=base_config.knn_engine,
+            knn_space_type=base_config.knn_space_type,
+            knn_algo_param_ef_search=base_config.knn_algo_param_ef_search,
+            knn_algo_param_ef_construction=base_config.knn_algo_param_ef_construction,
+            knn_algo_param_m=base_config.knn_algo_param_m,
+            # Preserve other settings
+            index_prefix=base_config.index_prefix,
+            number_of_shards=base_config.number_of_shards,
+            number_of_replicas=base_config.number_of_replicas,
+            search_pipeline_name=base_config.search_pipeline_name,
         )
 
         indexer = OpenSearchIndexer(client, opensearch_config)
@@ -314,6 +445,7 @@ async def load_command(
 
         indexed_count = 0
         failed_count = 0
+        all_errors: list[dict] = []
 
         with Progress(
             SpinnerColumn(),
@@ -333,6 +465,8 @@ async def load_command(
                 result = await indexer.bulk_index(batch, index_name, batch_size=batch_size)
                 indexed_count += result.indexed_count
                 failed_count += result.failed_count
+                if result.errors:
+                    all_errors.extend(result.errors)
 
                 progress.update(index_task, advance=len(batch))
 
@@ -354,12 +488,28 @@ async def load_command(
                 style="success",
             )
         else:
+            # Build error details section
+            error_details = ""
+            if all_errors:
+                error_details = "\n\n[bold red]Error Details:[/bold red]\n"
+                for i, err in enumerate(all_errors[:5], 1):  # Show first 5 errors
+                    if isinstance(err, dict):
+                        error_type = err.get("error", {}).get("type", "unknown") if isinstance(err.get("error"), dict) else str(err.get("error", "unknown"))
+                        error_reason = err.get("error", {}).get("reason", "No reason provided") if isinstance(err.get("error"), dict) else str(err.get("error", "No details"))
+                        doc_id = err.get("_id", "unknown")
+                        error_details += f"  {i}. [dim]Doc {doc_id}:[/dim] {error_type} - {error_reason}\n"
+                    else:
+                        error_details += f"  {i}. {err}\n"
+                if len(all_errors) > 5:
+                    error_details += f"  ... and {len(all_errors) - 5} more errors\n"
+
             display.panel(
                 f"Documents Loaded:     [cyan]{len(documents)}[/cyan]\n"
                 f"Chunks Created:       [cyan]{len(chunked_documents)}[/cyan]\n"
                 f"Documents Indexed:    [green]{indexed_count}[/green]\n"
                 f"Documents Failed:     [red]{failed_count}[/red]\n"
-                f"Index:                [cyan]{index_name}[/cyan]",
+                f"Index:                [cyan]{index_name}[/cyan]"
+                f"{error_details}",
                 title="Loading Complete (with errors)",
                 style="warning",
             )