remdb 0.3.172__py3-none-any.whl → 0.3.223__py3-none-any.whl

rem/cli/README.md

@@ -434,6 +434,68 @@ Ensure you're using the correct model format:
 - OpenAI: `openai:gpt-4o-mini`, `openai:gpt-4o`
 - Anthropic: `anthropic:claude-sonnet-4-5-20250929`
 
+## Data Visibility: PUBLIC vs PRIVATE
+
+**IMPORTANT: All ingested data is PUBLIC by default.** This is the correct behavior
+for shared knowledge bases (ontologies, procedures, reference data).
+
+### Why PUBLIC by Default?
+
+Most data in REM should be searchable by all users:
+- Clinical ontologies (disorders, symptoms, drugs)
+- Procedures and protocols (SCID-5, PHQ-9, etc.)
+- Reference documentation
+- Shared domain knowledge
+
+The `rem_lookup()` function searches for data where `user_id IS NULL`, which means
+public data. If you set `user_id` on data, it becomes invisible to other users.
+
+### Ingesting Public Data (Default)
+
+```bash
+# Standard ingestion - data is PUBLIC
+rem process ingest ontology/procedures/ --table ontologies
+
+# From S3 - also PUBLIC
+rem process ingest s3://bucket/docs/reference.pdf
+```
+
+### Ingesting Private Data (Rare)
+
+Private data requires explicit `--make-private` flag:
+
+```bash
+# Private user data - requires --make-private and --user-id
+rem process ingest personal-notes.md --make-private --user-id user-123
+```
+
+**When to use private data:**
+- User-uploaded personal documents
+- Session-specific content
+- User notes and annotations
+
+**NEVER use private data for:**
+- Ontologies and reference material
+- Clinical procedures and protocols
+- Shared knowledge bases
+- Anything that should be searchable by agents
+
+### Common Mistake
+
+If agents can't find data via `search_rem`, the most common cause is that the data
+was ingested with a `user_id` set. Check with:
+
+```sql
+SELECT name, user_id FROM ontologies WHERE name = 'phq-9-procedure';
+-- user_id should be NULL for public data
+```
+
+Fix by setting user_id to NULL:
+```sql
+UPDATE ontologies SET user_id = NULL WHERE user_id IS NOT NULL;
+UPDATE kv_store SET user_id = NULL WHERE entity_type = 'ontologies' AND user_id IS NOT NULL;
+```
+
 ## Next Steps
 
 1. **Implement Schema Registry**

rem/cli/commands/ask.py

@@ -75,7 +75,7 @@ async def run_agent_streaming(
     """
     Run agent in streaming mode using agent.iter() with usage limits.
 
-    Design Pattern (from carrier):
+    Design Pattern:
     - Use agent.iter() for complete execution with tool call visibility
     - run_stream() stops after first output, missing tool calls
     - Stream tool call markers: [Calling: tool_name]

rem/cli/commands/db.py

@@ -333,64 +333,120 @@ def rebuild_cache(connection: str | None):
 
 @click.command()
 @click.argument("file_path", type=click.Path(exists=True, path_type=Path))
+@click.option("--table", "-t", default=None, help="Target table name (required for non-YAML formats)")
 @click.option("--user-id", default=None, help="User ID to scope data privately (default: public/shared)")
 @click.option("--dry-run", is_flag=True, help="Show what would be loaded without loading")
-def load(file_path: Path, user_id: str | None, dry_run: bool):
+def load(file_path: Path, table: str | None, user_id: str | None, dry_run: bool):
     """
-    Load data from YAML file into database.
+    Load data from file into database.
 
-    File format:
-        - table: resources
-          key_field: name
-          rows:
-            - name: Example
-              content: Test data...
+    Supports YAML with embedded metadata, or any tabular format via Polars
+    (jsonl, parquet, csv, json, arrow, etc.). For non-YAML formats, use --table.
 
     Examples:
-        rem db load rem/tests/data/graph_seed.yaml
-        rem db load data.yaml --user-id my-user  # Private to user
-        rem db load data.yaml --dry-run
+        rem db load data.yaml                        # YAML with metadata
+        rem db load data.jsonl -t resources          # Any Polars-supported format
     """
-    asyncio.run(_load_async(file_path, user_id, dry_run))
+    asyncio.run(_load_async(file_path, table, user_id, dry_run))
 
 
-async def _load_async(file_path: Path, user_id: str | None, dry_run: bool):
+def _load_dataframe_from_file(file_path: Path) -> "pl.DataFrame":
+    """Load any Polars-supported file format into a DataFrame."""
+    import polars as pl
+
+    suffix = file_path.suffix.lower()
+
+    if suffix in {".jsonl", ".ndjson"}:
+        return pl.read_ndjson(file_path)
+    elif suffix in {".parquet", ".pq"}:
+        return pl.read_parquet(file_path)
+    elif suffix == ".csv":
+        return pl.read_csv(file_path)
+    elif suffix == ".json":
+        return pl.read_json(file_path)
+    elif suffix in {".ipc", ".arrow"}:
+        return pl.read_ipc(file_path)
+    else:
+        raise ValueError(f"Unsupported file format: {suffix}. Use any Polars-supported format.")
+
+
+async def _load_async(file_path: Path, table: str | None, user_id: str | None, dry_run: bool):
     """Async implementation of load command."""
+    import polars as pl
     import yaml
     from ...models.core.inline_edge import InlineEdge
-    from ...models.entities import Resource, Moment, User, Message, SharedSession, Schema
+    from ...models.entities import SharedSession
     from ...services.postgres import get_postgres_service
+    from ...utils.model_helpers import get_table_name
+    from ... import get_model_registry
 
     logger.info(f"Loading data from: {file_path}")
     scope_msg = f"user: {user_id}" if user_id else "public"
     logger.info(f"Scope: {scope_msg}")
 
-    # Load YAML file
-    with open(file_path) as f:
-        data = yaml.safe_load(f)
-
-    if not isinstance(data, list):
-        logger.error("YAML must be a list of table definitions")
-        raise click.Abort()
-
-    if dry_run:
-        logger.info("DRY RUN - Would load:")
-        logger.info(yaml.dump(data, default_flow_style=False))
-        return
+    suffix = file_path.suffix.lower()
+    is_yaml = suffix in {".yaml", ".yml"}
 
-    # Map table names to model classes
-    # CoreModel subclasses use Repository.upsert()
+    # Build MODEL_MAP dynamically from registry
+    registry = get_model_registry()
+    registry.register_core_models()
     MODEL_MAP = {
-        "users": User,
-        "moments": Moment,
-        "resources": Resource,
-        "messages": Message,
-        "schemas": Schema,
+        get_table_name(model): model
+        for model in registry.get_model_classes().values()
     }
 
     # Non-CoreModel tables that need direct SQL insertion
     DIRECT_INSERT_TABLES = {"shared_sessions"}
 
+    # Parse file based on format
+    if is_yaml:
+        # YAML with embedded metadata
+        with open(file_path) as f:
+            data = yaml.safe_load(f)
+
+        if not isinstance(data, list):
+            logger.error("YAML must be a list of table definitions")
+            raise click.Abort()
+
+        if dry_run:
+            logger.info("DRY RUN - Would load:")
+            logger.info(yaml.dump(data, default_flow_style=False))
+            return
+
+        table_defs = data
+    else:
+        # Polars-supported format - require --table
+        if not table:
+            logger.error(f"For {suffix} files, --table is required. Example: rem db load {file_path.name} -t resources")
+            raise click.Abort()
+
+        try:
+            df = _load_dataframe_from_file(file_path)
+        except Exception as e:
+            logger.error(f"Failed to load file: {e}")
+            raise click.Abort()
+
+        rows = df.to_dicts()
+
+        if dry_run:
+            logger.info(f"DRY RUN - Would load {len(rows)} rows to table '{table}':")
+            logger.info(f"Columns: {list(df.columns)}")
+
+            # Validate first row against model if table is known
+            if table in MODEL_MAP and rows:
+                from ...utils.model_helpers import validate_data_for_model
+                result = validate_data_for_model(MODEL_MAP[table], rows[0])
+                if result.extra_fields:
+                    logger.warning(f"Unknown fields (ignored): {result.extra_fields}")
+                if result.valid:
+                    logger.success(f"Sample row validates OK. Required: {result.required_fields or '(none)'}")
+                else:
+                    result.log_errors("Sample row")
+            return
+
+        # Wrap as single table definition
+        table_defs = [{"table": table, "rows": rows}]
+
     # Connect to database
     pg = get_postgres_service()
     if not pg:
@@ -399,23 +455,23 @@ async def _load_async(file_path: Path, user_id: str | None, dry_run: bool):
 
     await pg.connect()
 
+    # Start embedding worker for generating embeddings
+    if pg.embedding_worker:
+        await pg.embedding_worker.start()
+
     try:
         total_loaded = 0
 
-        for table_def in data:
+        for table_def in table_defs:
             table_name = table_def["table"]
-            key_field = table_def.get("key_field", "id")
             rows = table_def.get("rows", [])
 
             # Handle direct insert tables (non-CoreModel)
             if table_name in DIRECT_INSERT_TABLES:
                 for row_data in rows:
-                    # Add tenant_id if not present
-                    if "tenant_id" not in row_data:
-                        row_data["tenant_id"] = "default"
+                    # tenant_id is optional - NULL means public/shared
 
                     if table_name == "shared_sessions":
-                        # Insert shared_session directly
                         await pg.fetch(
                             """INSERT INTO shared_sessions
                                (session_id, owner_user_id, shared_with_user_id, tenant_id)
@@ -424,7 +480,7 @@ async def _load_async(file_path: Path, user_id: str | None, dry_run: bool):
                             row_data["session_id"],
                             row_data["owner_user_id"],
                             row_data["shared_with_user_id"],
-                            row_data["tenant_id"],
+                            row_data.get("tenant_id"),  # Optional - NULL means public
                         )
                         total_loaded += 1
                         logger.success(f"Loaded shared_session: {row_data['owner_user_id']} -> {row_data['shared_with_user_id']}")
@@ -434,16 +490,11 @@ async def _load_async(file_path: Path, user_id: str | None, dry_run: bool):
                 logger.warning(f"Unknown table: {table_name}, skipping")
                 continue
 
-            model_class = MODEL_MAP[table_name]  # Type is inferred from MODEL_MAP
+            model_class = MODEL_MAP[table_name]
 
-            for row_data in rows:
-                # Add user_id and tenant_id only if explicitly provided
-                # Default is public (None) - data is shared/visible to all
-                # Pass --user-id to scope data privately to a specific user
-                if "user_id" not in row_data and user_id is not None:
-                    row_data["user_id"] = user_id
-                if "tenant_id" not in row_data and user_id is not None:
-                    row_data["tenant_id"] = row_data.get("user_id", user_id)
+            for row_idx, row_data in enumerate(rows):
+                # tenant_id and user_id are optional - NULL means public/shared data
+                # Data files can explicitly set tenant_id/user_id if needed
 
                 # Convert graph_edges to InlineEdge format if present
                 if "graph_edges" in row_data:
@@ -452,30 +503,40 @@ async def _load_async(file_path: Path, user_id: str | None, dry_run: bool):
                         for edge in row_data["graph_edges"]
                     ]
 
-                # Convert any ISO timestamp strings with Z suffix to naive datetime
-                # This handles fields like starts_timestamp, ends_timestamp, etc.
+                # Convert ISO timestamp strings
                 from ...utils.date_utils import parse_iso
                 for key, value in list(row_data.items()):
                     if isinstance(value, str) and (key.endswith("_timestamp") or key.endswith("_at")):
                         try:
                             row_data[key] = parse_iso(value)
                         except (ValueError, TypeError):
-                            pass  # Not a valid datetime string, leave as-is
+                            pass
 
-                # Create model instance and upsert via repository
                 from ...services.postgres.repository import Repository
+                from ...utils.model_helpers import validate_data_for_model
 
-                instance = model_class(**row_data)
-                repo = Repository(model_class, table_name, pg)  # Type inferred from MODEL_MAP
-                await repo.upsert(instance)  # type: ignore[arg-type]
+                result = validate_data_for_model(model_class, row_data)
+                if not result.valid:
+                    result.log_errors(f"Row {row_idx + 1} ({table_name})")
+                    raise click.Abort()
+
+                repo = Repository(model_class, table_name, pg)
+                await repo.upsert(result.instance)  # type: ignore[arg-type]
                 total_loaded += 1
 
-                # Log based on model type
-                name = getattr(instance, 'name', getattr(instance, 'id', '?'))
+                name = getattr(result.instance, 'name', getattr(result.instance, 'id', '?'))
                 logger.success(f"Loaded {table_name[:-1]}: {name}")
 
         logger.success(f"Data loaded successfully! Total rows: {total_loaded}")
 
+        # Wait for embeddings to complete
+        if pg.embedding_worker and pg.embedding_worker.running:
+            queue_size = pg.embedding_worker.task_queue.qsize()
+            if queue_size > 0:
+                logger.info(f"Waiting for {queue_size} embeddings to complete...")
+            await pg.embedding_worker.stop()
+            logger.success("Embeddings generated successfully")
+
     finally:
         await pg.disconnect()
 
@@ -580,7 +641,7 @@ async def _diff_async(
 
         if not result.has_changes:
             click.secho("✓ No schema drift detected", fg="green")
-            click.echo("  Database matches Pydantic models")
+            click.echo("  Database matches source (tables, functions, triggers, views)")
             if result.filtered_count > 0:
                 click.echo()
                 click.secho(f"  ({result.filtered_count} destructive change(s) hidden by '{strategy}' strategy)", fg="yellow")
@@ -592,17 +653,34 @@ async def _diff_async(
         if result.filtered_count > 0:
             click.secho(f"   ({result.filtered_count} destructive change(s) hidden by '{strategy}' strategy)", fg="yellow")
         click.echo()
-        click.echo("Changes:")
-        for line in result.summary:
-            if line.startswith("+"):
-                click.secho(f"  {line}", fg="green")
-            elif line.startswith("-"):
-                click.secho(f"  {line}", fg="red")
-            elif line.startswith("~"):
-                click.secho(f"  {line}", fg="yellow")
-            else:
-                click.echo(f"  {line}")
-        click.echo()
+
+        # Table/column changes (Alembic)
+        if result.summary:
+            click.echo("Table Changes:")
+            for line in result.summary:
+                if line.startswith("+"):
+                    click.secho(f"  {line}", fg="green")
+                elif line.startswith("-"):
+                    click.secho(f"  {line}", fg="red")
+                elif line.startswith("~"):
+                    click.secho(f"  {line}", fg="yellow")
+                else:
+                    click.echo(f"  {line}")
+            click.echo()
+
+        # Programmable object changes (functions, triggers, views)
+        if result.programmable_summary:
+            click.echo("Programmable Objects (functions/triggers/views):")
+            for line in result.programmable_summary:
+                if line.startswith("+"):
+                    click.secho(f"  {line}", fg="green")
+                elif line.startswith("-"):
+                    click.secho(f"  {line}", fg="red")
+                elif line.startswith("~"):
+                    click.secho(f"  {line}", fg="yellow")
+                else:
+                    click.echo(f"  {line}")
+            click.echo()
 
         # Generate migration if requested
         if generate:

rem/cli/commands/process.py

@@ -11,39 +11,102 @@ from rem.services.content import ContentService
 
 
 @click.command(name="ingest")
-@click.argument("file_path", type=click.Path(exists=True))
-@click.option("--user-id", default=None, help="User ID to scope file privately (default: public/shared)")
+@click.argument("path", type=click.Path(exists=True))
+@click.option("--table", "-t", default=None, help="Target table (e.g., ontologies, resources). Auto-detected for schemas.")
+@click.option("--make-private", is_flag=True, help="Make data private to a specific user. RARELY NEEDED - most data should be public/shared.")
+@click.option("--user-id", default=None, help="User ID for private data. REQUIRES --make-private flag.")
 @click.option("--category", help="Optional file category")
 @click.option("--tags", help="Optional comma-separated tags")
+@click.option("--pattern", "-p", default="**/*.md", help="Glob pattern for directory ingestion (default: **/*.md)")
+@click.option("--dry-run", is_flag=True, help="Show what would be ingested without making changes")
 def process_ingest(
-    file_path: str,
+    path: str,
+    table: str | None,
+    make_private: bool,
     user_id: str | None,
     category: str | None,
     tags: str | None,
+    pattern: str,
+    dry_run: bool,
 ):
     """
-    Ingest a file into REM (storage + parsing + embedding).
+    Ingest files into REM (storage + parsing + embedding).
 
-    This command performs the full ingestion pipeline:
-    1. Reads the file from the local path.
-    2. Stores it in the configured storage (local/S3).
-    3. Parses the content.
-    4. Chunks and embeds the content into Resources.
-    5. Creates a File entity record.
+    Supports both single files and directories. For directories, recursively
+    processes files matching the pattern (default: **/*.md).
+
+    **IMPORTANT: Data is PUBLIC by default.** This is the correct behavior for
+    shared knowledge bases (ontologies, procedures, reference data). Private
+    user-scoped data is rarely needed and requires explicit --make-private flag.
+
+    Target table is auto-detected for schemas (agent.yaml → schemas table).
+    Use --table to explicitly set the target (e.g., ontologies for clinical knowledge).
 
     Examples:
         rem process ingest sample.pdf
         rem process ingest contract.docx --category legal --tags contract,2023
         rem process ingest agent.yaml  # Auto-detects kind=agent, saves to schemas table
+
+        # Directory ingestion into ontologies table (PUBLIC - no user-id needed)
+        rem process ingest ontology/procedures/scid-5/ --table ontologies
+        rem process ingest ontology/ --table ontologies --pattern "**/*.md"
+
+        # Preview what would be ingested
+        rem process ingest ontology/ --table ontologies --dry-run
+
+        # RARE: Private user-scoped data (requires --make-private)
+        rem process ingest private-notes.md --make-private --user-id user-123
     """
     import asyncio
+
+    # Validate: user_id requires --make-private flag
+    if user_id and not make_private:
+        raise click.UsageError(
+            "Setting --user-id requires the --make-private flag.\n\n"
+            "Data should be PUBLIC by default (no user-id). Private user-scoped data\n"
+            "is rarely needed - only use --make-private for truly personal content.\n\n"
+            "Example: rem process ingest file.md --make-private --user-id user-123"
+        )
+
+    # If --make-private is set, user_id is required
+    if make_private and not user_id:
+        raise click.UsageError(
+            "--make-private requires --user-id to specify which user owns the data.\n\n"
+            "Example: rem process ingest file.md --make-private --user-id user-123"
+        )
+
+    # Clear user_id if not making private (ensure None for public data)
+    effective_user_id = user_id if make_private else None
+    from pathlib import Path
     from ...services.content import ContentService
 
     async def _ingest():
-        # Initialize ContentService with repositories for proper resource saving
         from rem.services.postgres import get_postgres_service
         from rem.services.postgres.repository import Repository
-        from rem.models.entities import File, Resource
+        from rem.models.entities import File, Resource, Ontology
+
+        input_path = Path(path)
+        tag_list = tags.split(",") if tags else None
+
+        # Collect files to process
+        if input_path.is_dir():
+            files_to_process = list(input_path.glob(pattern))
+            if not files_to_process:
+                logger.error(f"No files matching '{pattern}' found in {input_path}")
+                sys.exit(1)
+            logger.info(f"Found {len(files_to_process)} files matching '{pattern}'")
+        else:
+            files_to_process = [input_path]
+
+        # Dry run: just show what would be processed
+        if dry_run:
+            logger.info("DRY RUN - Would ingest:")
+            for f in files_to_process[:20]:
+                entity_key = f.stem  # filename without extension
+                logger.info(f"  {f} → {table or 'auto-detect'} (key: {entity_key})")
+            if len(files_to_process) > 20:
+                logger.info(f"  ... and {len(files_to_process) - 20} more files")
+            return
 
         db = get_postgres_service()
         if not db:
@@ -51,53 +114,118 @@ def process_ingest(
         await db.connect()
 
         try:
-            file_repo = Repository(File, "files", db=db)
-            resource_repo = Repository(Resource, "resources", db=db)
-            service = ContentService(file_repo=file_repo, resource_repo=resource_repo)
-
-            tag_list = tags.split(",") if tags else None
-
-            scope_msg = f"user: {user_id}" if user_id else "public"
-            logger.info(f"Ingesting file: {file_path} ({scope_msg})")
-            result = await service.ingest_file(
-                file_uri=file_path,
-                user_id=user_id,
-                category=category,
-                tags=tag_list,
-                is_local_server=True, # CLI is local
-            )
-
-            # Handle schema ingestion (agents/evaluators)
-            if result.get("schema_name"):
-                logger.success(f"Schema ingested: {result['schema_name']} (kind={result.get('kind', 'agent')})")
-                logger.info(f"Version: {result.get('version', '1.0.0')}")
-            # Handle file ingestion
-            elif result.get("processing_status") == "completed":
-                logger.success(f"File ingested: {result['file_name']}")
-                logger.info(f"File ID: {result['file_id']}")
-                logger.info(f"Resources created: {result['resources_created']}")
+            # Direct table ingestion (ontologies, etc.)
+            if table:
+                await _ingest_to_table(
+                    db=db,
+                    files=files_to_process,
+                    table_name=table,
+                    user_id=effective_user_id,
+                    category=category,
+                    tag_list=tag_list,
+                )
             else:
-                logger.error(f"Ingestion failed: {result.get('message', 'Unknown error')}")
-                sys.exit(1)
+                # Standard file ingestion via ContentService
+                file_repo = Repository(File, "files", db=db)
+                resource_repo = Repository(Resource, "resources", db=db)
+                service = ContentService(file_repo=file_repo, resource_repo=resource_repo)
+
+                for file_path in files_to_process:
+                    scope_msg = f"user: {effective_user_id}" if effective_user_id else "public"
+                    logger.info(f"Ingesting: {file_path} ({scope_msg})")
+
+                    result = await service.ingest_file(
+                        file_uri=str(file_path),
+                        user_id=effective_user_id,
+                        category=category,
+                        tags=tag_list,
+                        is_local_server=True,
+                    )
+
+                    # Handle schema ingestion (agents/evaluators)
+                    if result.get("schema_name"):
+                        logger.success(f"Schema: {result['schema_name']} (kind={result.get('kind', 'agent')})")
+                    elif result.get("processing_status") == "completed":
+                        logger.success(f"File: {result['file_name']} ({result['resources_created']} resources)")
+                    else:
+                        logger.error(f"Failed: {result.get('message', 'Unknown error')}")
 
         except Exception as e:
             logger.error(f"Error during ingestion: {e}")
             sys.exit(1)
         finally:
-            # Wait for global embedding worker to finish queued tasks
+            # Wait for embedding worker to finish
             from rem.services.embeddings.worker import get_global_embedding_worker
             try:
                 worker = get_global_embedding_worker()
                 if worker and worker.running and not worker.task_queue.empty():
-                    logger.info(f"Waiting for {worker.task_queue.qsize()} embedding tasks to complete...")
-                    # Worker.stop() waits for queue to drain (see worker.py line ~148)
+                    logger.info(f"Waiting for {worker.task_queue.qsize()} embedding tasks...")
                     await worker.stop()
             except RuntimeError:
-                # Worker doesn't exist yet - no tasks queued
                 pass
 
             await db.disconnect()
 
+    async def _ingest_to_table(db, files, table_name, user_id, category, tag_list):
+        """Direct ingestion of files to a specific table (ontologies, etc.)."""
+        from rem.services.postgres.repository import Repository
+        from rem import get_model_registry
+        from rem.utils.model_helpers import get_table_name
+
+        # Get model class for table
+        registry = get_model_registry()
+        registry.register_core_models()
+        model_class = None
+        for model in registry.get_model_classes().values():
+            if get_table_name(model) == table_name:
+                model_class = model
+                break
+
+        if not model_class:
+            logger.error(f"Unknown table: {table_name}")
+            sys.exit(1)
+
+        repo = Repository(model_class, table_name, db=db)
+        processed = 0
+        failed = 0
+
+        for file_path in files:
+            try:
+                # Read file content
+                content = file_path.read_text(encoding="utf-8")
+                entity_key = file_path.stem  # filename without extension
+
+                # Build entity based on table
+                entity_data = {
+                    "name": entity_key,
+                    "content": content,
+                    "tags": tag_list or [],
+                }
+
+                # Add optional fields
+                if category:
+                    entity_data["category"] = category
+
+                # Scoping: user_id for private data, "public" for shared
+                # tenant_id="public" is the default for shared knowledge bases
+                entity_data["tenant_id"] = user_id or "public"
+                entity_data["user_id"] = user_id  # None = public/shared
+
+                # For ontologies, add URI
+                if table_name == "ontologies":
+                    entity_data["uri"] = f"file://{file_path.absolute()}"
+
+                entity = model_class(**entity_data)
+                await repo.upsert(entity, embeddable_fields=["content"], generate_embeddings=True)
+                processed += 1
+                logger.success(f"  ✓ {entity_key}")
+
+            except Exception as e:
+                failed += 1
+                logger.error(f"  ✗ {file_path.name}: {e}")
+
+        logger.info(f"Completed: {processed} succeeded, {failed} failed")
+
     asyncio.run(_ingest())
 
 def register_commands(group: click.Group):