strapi-kit 0.0.6__py3-none-any.whl → 0.1.0__py3-none-any.whl

strapi_kit/export/importer.py

@@ -111,6 +111,12 @@ class StrapiImporter:
             # Step 1.5: Load schemas from export metadata
             self._load_schemas_from_export(export_data)
 
+            # Step 1.6: Validate relations if requested
+            if options.validate_relations:
+                if options.progress_callback:
+                    options.progress_callback(10, 100, "Validating relations")
+                self._validate_relations(export_data, result)
+
             # Step 2: Filter content types if specified
             content_types_to_import = self._get_content_types_to_import(export_data, options)
 
@@ -189,6 +195,54 @@ class StrapiImporter:
         if export_data.get_entity_count() == 0:
             result.add_warning("No entities to import")
 
+    def _validate_relations(self, export_data: ExportData, result: ImportResult) -> None:
+        """Validate that all relation targets exist in export data.
+
+        This pre-import validation ensures all referenced entities are present
+        in the export, warning about any missing targets.
+
+        Args:
+            export_data: Export data to validate
+            result: Result object to add warnings to
+        """
+        # Build set of available IDs per content type (both int and str for v5)
+        available_ids: dict[str, set[int | str]] = {}
+        for ct, entities in export_data.entities.items():
+            ids: set[int | str] = set()
+            for e in entities:
+                ids.add(e.id)
+                # Include document_id for v5 string-based relations
+                if e.document_id:
+                    ids.add(e.document_id)
+            available_ids[ct] = ids
+
+        # Check all relations
+        for ct, entities in export_data.entities.items():
+            # Get schema for this content type
+            schema = export_data.metadata.schemas.get(ct)
+            if not schema:
+                continue
+
+            for entity in entities:
+                for field_name, target_ids in entity.relations.items():
+                    target_ct = schema.get_field_target(field_name)
+                    if not target_ct:
+                        continue
+
+                    if target_ct not in available_ids:
+                        result.add_warning(
+                            f"{ct}#{entity.id}.{field_name} -> target type '{target_ct}' "
+                            "not in export"
+                        )
+                        continue
+
+                    missing = set(target_ids) - available_ids.get(target_ct, set())
+                    if missing:
+                        result.add_warning(
+                            f"{ct}#{entity.id}.{field_name} -> missing IDs in {target_ct}: "
+                            f"{sorted(missing)}"
+                        )
+
     def _get_content_types_to_import(
         self, export_data: ExportData, options: ImportOptions
     ) -> list[str]:
@@ -231,83 +285,144 @@ class StrapiImporter:
             options: Import options
             result: Result object to update
         """
+        total_entities = sum(len(export_data.entities.get(ct, [])) for ct in content_types)
+        processed = 0
+
         for content_type in content_types:
             entities = export_data.entities.get(content_type, [])
 
             # Get endpoint from schema (prefers plural_name) or fallback to UID
             endpoint = self._get_endpoint(content_type)
 
-            for entity in entities:
-                try:
-                    # Update media references if we have mappings
-                    entity_data = entity.data
-                    if media_id_mapping:
-                        entity_data = MediaHandler.update_media_references(
-                            entity.data, media_id_mapping
-                        )
-
-                    if options.dry_run:
-                        # Just validate, don't actually create
-                        result.entities_imported += 1
-                        continue
-
-                    # Check for existing entity if document_id is available (for conflict handling)
-                    existing_id: int | None = None
-                    if entity.document_id:
-                        existing_id = self._check_entity_exists(endpoint, entity.document_id)
+            # Process entities in batches for progress reporting
+            for batch_start in range(0, len(entities), options.batch_size):
+                batch = entities[batch_start : batch_start + options.batch_size]
+
+                for entity in batch:
+                    try:
+                        # Update media references if we have mappings
+                        entity_data = entity.data
+                        if media_id_mapping:
+                            entity_data = MediaHandler.update_media_references(
+                                entity.data, media_id_mapping
+                            )
 
-                    if existing_id is not None:
-                        # Entity already exists - handle according to conflict resolution
-                        if options.conflict_resolution == ConflictResolution.SKIP:
-                            result.entities_skipped += 1
-                            # Still track the ID mapping for relations
-                            if content_type not in result.id_mapping:
-                                result.id_mapping[content_type] = {}
-                            result.id_mapping[content_type][entity.id] = existing_id
+                        if options.dry_run:
+                            # Just validate, don't actually create
+                            result.entities_imported += 1
                             continue
 
-                        elif options.conflict_resolution == ConflictResolution.FAIL:
-                            raise ImportExportError(
-                                f"Entity already exists: {content_type} with documentId "
-                                f"{entity.document_id}. Use conflict_resolution=SKIP or UPDATE."
-                            )
+                        # Check for existing entity if document_id is available (for conflict handling)
+                        existing_id: int | None = None
+                        if entity.document_id:
+                            existing_id = self._check_entity_exists(endpoint, entity.document_id)
 
-                        elif options.conflict_resolution == ConflictResolution.UPDATE:
-                            # Update existing entity
-                            response = self.client.update(
-                                f"{endpoint}/{entity.document_id}",
-                                entity_data,
-                            )
-                            if response.data:
+                        if existing_id is not None:
+                            # Entity already exists - handle according to conflict resolution
+                            if options.conflict_resolution == ConflictResolution.SKIP:
+                                result.entities_skipped += 1
+                                # Still track the ID mapping for relations
                                 if content_type not in result.id_mapping:
                                     result.id_mapping[content_type] = {}
-                                result.id_mapping[content_type][entity.id] = response.data.id
-                                result.entities_updated += 1
-                            continue
-
-                    # Create new entity
-                    response = self.client.create(endpoint, entity_data)
+                                result.id_mapping[content_type][entity.id] = existing_id
+                                # Track document_id mappings for v5
+                                if entity.document_id:
+                                    if content_type not in result.doc_id_mapping:
+                                        result.doc_id_mapping[content_type] = {}
+                                    result.doc_id_mapping[content_type][entity.id] = (
+                                        entity.document_id
+                                    )
+                                    # Track reverse mapping for v5 string relation resolution
+                                    if content_type not in result.doc_id_to_new_id:
+                                        result.doc_id_to_new_id[content_type] = {}
+                                    result.doc_id_to_new_id[content_type][entity.document_id] = (
+                                        existing_id
+                                    )
+                                continue
+
+                            elif options.conflict_resolution == ConflictResolution.FAIL:
+                                raise ImportExportError(
+                                    f"Entity already exists: {content_type} with documentId "
+                                    f"{entity.document_id}. Use conflict_resolution=SKIP or UPDATE."
+                                )
+
+                            elif options.conflict_resolution == ConflictResolution.UPDATE:
+                                # Update existing entity
+                                response = self.client.update(
+                                    f"{endpoint}/{entity.document_id}",
+                                    entity_data,
+                                )
+                                if response.data:
+                                    if content_type not in result.id_mapping:
+                                        result.id_mapping[content_type] = {}
+                                    result.id_mapping[content_type][entity.id] = response.data.id
+                                    # Track document_id mappings for v5
+                                    if entity.document_id:
+                                        if content_type not in result.doc_id_mapping:
+                                            result.doc_id_mapping[content_type] = {}
+                                        result.doc_id_mapping[content_type][entity.id] = (
+                                            entity.document_id
+                                        )
+                                        # Track reverse mapping for v5 string relation resolution
+                                        if content_type not in result.doc_id_to_new_id:
+                                            result.doc_id_to_new_id[content_type] = {}
+                                        result.doc_id_to_new_id[content_type][
+                                            entity.document_id
+                                        ] = response.data.id
+                                    result.entities_updated += 1
+                                continue
+
+                        # Create new entity
+                        response = self.client.create(endpoint, entity_data)
+
+                        if response.data:
+                            # Track ID mapping for relation resolution
+                            if content_type not in result.id_mapping:
+                                result.id_mapping[content_type] = {}
+                            result.id_mapping[content_type][entity.id] = response.data.id
+
+                            # Track document_id mappings for v5
+                            if response.data.document_id:
+                                if content_type not in result.doc_id_mapping:
+                                    result.doc_id_mapping[content_type] = {}
+                                result.doc_id_mapping[content_type][entity.id] = (
+                                    response.data.document_id
+                                )
+
+                            # Track reverse mapping for v5 string relation resolution
+                            if entity.document_id:
+                                if content_type not in result.doc_id_to_new_id:
+                                    result.doc_id_to_new_id[content_type] = {}
+                                result.doc_id_to_new_id[content_type][entity.document_id] = (
+                                    response.data.id
+                                )
+
+                            result.entities_imported += 1
+
+                    except ValidationError as e:
+                        result.add_error(
+                            f"Validation error importing {content_type} #{entity.id}: {e}"
+                        )
+                        result.entities_failed += 1
 
-                    if response.data:
-                        # Track ID mapping for relation resolution
-                        if content_type not in result.id_mapping:
-                            result.id_mapping[content_type] = {}
+                    except ImportExportError:
+                        # Re-raise ImportExportError (e.g., from FAIL conflict resolution)
+                        raise
 
-                        result.id_mapping[content_type][entity.id] = response.data.id
-                        result.entities_imported += 1
+                    except StrapiError as e:
+                        # Catch Strapi-specific errors (API errors, network issues, etc.)
+                        result.add_error(f"Failed to import {content_type} #{entity.id}: {e}")
+                        result.entities_failed += 1
 
-                except ValidationError as e:
-                    result.add_error(f"Validation error importing {content_type} #{entity.id}: {e}")
-                    result.entities_failed += 1
+                    finally:
+                        processed += 1
 
-                except ImportExportError:
-                    # Re-raise ImportExportError (e.g., from FAIL conflict resolution)
-                    raise
-
-                except StrapiError as e:
-                    # Catch Strapi-specific errors (API errors, network issues, etc.)
-                    result.add_error(f"Failed to import {content_type} #{entity.id}: {e}")
-                    result.entities_failed += 1
+                # Batch progress callback
+                if options.progress_callback and total_entities > 0:
+                    progress = 40 + int((processed / total_entities) * 20)
+                    options.progress_callback(
+                        progress, 100, f"Importing entities ({processed}/{total_entities})"
+                    )
 
     def _check_entity_exists(self, endpoint: str, document_id: str) -> int | None:
         """Check if an entity exists by document ID.
@@ -370,6 +485,14 @@ class StrapiImporter:
 
                 new_id = result.id_mapping[content_type][old_id]
 
+                # Get document_id for v5 endpoint (falls back to numeric ID for v4)
+                entity_endpoint_id: str | int = new_id
+                if (
+                    content_type in result.doc_id_mapping
+                    and old_id in result.doc_id_mapping[content_type]
+                ):
+                    entity_endpoint_id = result.doc_id_mapping[content_type][old_id]
+
                 # Get schema for this content type
                 try:
                     schema = self._schema_cache.get_schema(content_type)
@@ -383,7 +506,10 @@ class StrapiImporter:
 
                     # FIXED: Resolve relations using schema
                     resolved_relations = self._resolve_relations_with_schema(
-                        entity.relations, schema, result.id_mapping
+                        entity.relations,
+                        schema,
+                        result.id_mapping,
+                        result.doc_id_to_new_id,
                     )
 
                     if not resolved_relations:
@@ -393,10 +519,10 @@ class StrapiImporter:
                     relation_payload = RelationResolver.build_relation_payload(resolved_relations)
 
                     if relation_payload:
-                        # Update entity with relations
+                        # Update entity with relations (use document_id for v5)
                         # Note: update() already wraps data in {"data": ...}
                         self.client.update(
-                            f"{endpoint}/{new_id}",
+                            f"{endpoint}/{entity_endpoint_id}",
                             relation_payload,
                         )
 
@@ -446,6 +572,16 @@ class StrapiImporter:
                     result.media_imported += 1
                     continue
 
+                # Check for existing media with same hash (if not overwriting)
+                if not options.overwrite_media:
+                    existing_id = self._find_media_by_hash(exported_media.hash)
+                    if existing_id is not None:
+                        # Use existing media - no need to upload
+                        media_id_mapping[exported_media.id] = existing_id
+                        result.media_skipped += 1
+                        logger.debug(f"Media {exported_media.name} already exists (hash match)")
+                        continue
+
                 # Find local file with path traversal protection
                 file_path = (media_path / exported_media.local_path).resolve()
 
@@ -491,21 +627,46 @@ class StrapiImporter:
 
         logger.info(f"Loaded {self._schema_cache.cache_size} schemas from export")
 
+    def _find_media_by_hash(self, file_hash: str) -> int | None:
+        """Find existing media file by hash.
+
+        Args:
+            file_hash: File hash to search for
+
+        Returns:
+            Media ID if found, None otherwise
+        """
+        try:
+            from strapi_kit.models import FilterBuilder, StrapiQuery
+
+            query = StrapiQuery().filter(FilterBuilder().eq("hash", file_hash))
+            response = self.client.list_media(query)
+
+            if response.data:
+                return response.data[0].id
+        except Exception:  # noqa: BLE001, S110 - Intentionally ignore lookup failures
+            pass
+        return None
+
     def _resolve_relations_with_schema(
         self,
         relations: dict[str, list[int | str]],
         schema: ContentTypeSchema,
         id_mapping: dict[str, dict[int, int]],
+        doc_id_to_new_id: dict[str, dict[str, int]] | None = None,
     ) -> dict[str, list[int]]:
         """Resolve relation IDs using schema information.
 
         Uses content type schemas to determine relation targets, enabling
-        proper ID mapping during import.
+        proper ID mapping during import. Handles both numeric IDs and
+        string documentIds (v5 format).
 
         Args:
             relations: Raw relations from export (field -> [old_ids])
             schema: Schema for the content type
             id_mapping: Full ID mapping (content_type -> {old_id: new_id})
+            doc_id_to_new_id: Optional document_id mapping for v5 string IDs
+                (content_type -> {old_document_id: new_id})
 
         Returns:
             Resolved relations with new IDs
@@ -529,12 +690,18 @@ class StrapiImporter:
                 continue
 
             target_mapping = id_mapping[target_content_type]
+            target_doc_mapping = (
+                doc_id_to_new_id.get(target_content_type, {}) if doc_id_to_new_id else {}
+            )
 
-            # Resolve old IDs to new IDs
+            # Resolve old IDs to new IDs (supports both int and str IDs)
             new_ids = []
             for old_id in old_ids:
                 if isinstance(old_id, int) and old_id in target_mapping:
                     new_ids.append(target_mapping[old_id])
+                elif isinstance(old_id, str) and old_id in target_doc_mapping:
+                    # V5 string documentId - look up in doc_id mapping
+                    new_ids.append(target_doc_mapping[old_id])
                 else:
                     logger.warning(
                         f"Could not resolve {target_content_type} ID {old_id} "
@@ -617,3 +784,289 @@ class StrapiImporter:
             API endpoint (e.g., "articles")
         """
         return StrapiImporter._uid_to_endpoint_fallback(uid)
+
+    def import_from_jsonl(
+        self,
+        jsonl_path: str | Path,
+        options: ImportOptions | None = None,
+        media_dir: Path | str | None = None,
+    ) -> ImportResult:
+        """Import data from JSONL file with two-pass streaming.
+
+        This method uses two-pass streaming for true O(1) memory usage:
+        - Pass 1: Create entities, store only ID mappings (old_id -> new_id)
+        - Pass 2: Re-read file to resolve relations using ID mappings
+
+        Memory profile: O(entity_count x 2 ints) for ID mappings only,
+        not O(entities) for full entity objects.
+
+        Args:
+            jsonl_path: Path to input JSONL file
+            options: Import options (uses defaults if None)
+            media_dir: Directory containing media files from export
+
+        Returns:
+            ImportResult with statistics and any errors
+
+        Raises:
+            ImportExportError: If import fails critically
+
+        Example:
+            >>> result = importer.import_from_jsonl(
+            ...     "export.jsonl",
+            ...     media_dir="media/"
+            ... )
+            >>> if result.success:
+            ...     print(f"Imported {result.entities_imported} entities")
+        """
+        from strapi_kit.export.jsonl_reader import JSONLImportReader
+
+        if options is None:
+            options = ImportOptions()
+
+        result = ImportResult(success=False, dry_run=options.dry_run)
+        jsonl_path = Path(jsonl_path)
+
+        try:
+            # ============================================================
+            # Pass 1: Read metadata, import media, create entities
+            # Store only ID mappings (O(entity_count x 2 ints))
+            # ============================================================
+            with JSONLImportReader(jsonl_path) as reader:
+                # Step 1: Read metadata
+                if options.progress_callback:
+                    options.progress_callback(0, 100, "Reading metadata")
+
+                metadata = reader.read_metadata()
+
+                # Load schemas from metadata
+                for ct, schema in metadata.schemas.items():
+                    self._schema_cache.cache_schema(ct, schema)
+
+                # Step 2: Import media first (if requested)
+                # Use separate reader to avoid consuming entity stream (Issue #30)
+                media_id_mapping: dict[int, int] = {}
+                if options.import_media and media_dir:
+                    if options.progress_callback:
+                        options.progress_callback(10, 100, "Importing media files")
+
+                    # Read media manifest with separate reader to preserve entity stream
+                    with JSONLImportReader(jsonl_path) as media_reader:
+                        media_reader.read_metadata()  # Skip metadata
+                        media_files = media_reader.read_media_manifest()
+
+                    if media_files:
+                        media_dir_path = Path(media_dir)
+                        for media in media_files:
+                            try:
+                                if options.dry_run:
+                                    result.media_imported += 1
+                                    continue
+
+                                # Check for existing media (overwrite_media option)
+                                if (
+                                    hasattr(options, "overwrite_media")
+                                    and not options.overwrite_media
+                                ):
+                                    # Try to find by hash
+                                    existing = self._find_media_by_hash(media.hash)
+                                    if existing is not None:
+                                        media_id_mapping[media.id] = existing
+                                        result.media_skipped += 1
+                                        continue
+
+                                # Upload media file with path traversal protection
+                                local_path = (media_dir_path / media.local_path).resolve()
+
+                                # Security: Ensure resolved path stays within media_dir_path
+                                if not local_path.is_relative_to(media_dir_path.resolve()):
+                                    result.add_error(
+                                        f"Security: Invalid media path {media.local_path} - "
+                                        "path traversal detected"
+                                    )
+                                    result.media_skipped += 1
+                                    continue
+
+                                if local_path.exists():
+                                    uploaded = MediaHandler.upload_media_file(
+                                        self.client, local_path, media
+                                    )
+                                    media_id_mapping[media.id] = uploaded.id
+                                    result.media_imported += 1
+                                else:
+                                    result.add_warning(f"Media file not found: {local_path}")
+                                    result.media_skipped += 1
+                            except StrapiError as e:
+                                result.add_error(f"Failed to import media {media.id}: {e}")
+                                result.media_skipped += 1
+
+                # Step 3: Create entities - streaming with ID mapping only
+                if options.progress_callback:
+                    options.progress_callback(30, 100, "Creating entities (pass 1)")
+
+                # Store only ID mappings: old_id -> new_id (O(entity_count x 2 ints))
+                id_mappings: dict[str, dict[int, int]] = {}
+                # Store document_id mappings for v5 endpoint updates
+                doc_id_mappings: dict[str, dict[int, str]] = {}
+                # Store reverse document_id mapping for v5 string relation resolution
+                doc_id_to_new_id_mappings: dict[str, dict[str, int]] = {}
+
+                for entity in reader.iter_entities():
+                    # Filter by content types if specified
+                    if options.content_types and entity.content_type not in options.content_types:
+                        continue
+
+                    content_type = entity.content_type
+                    if content_type not in id_mappings:
+                        id_mappings[content_type] = {}
+                        doc_id_mappings[content_type] = {}
+                        doc_id_to_new_id_mappings[content_type] = {}
+
+                    try:
+                        # Update media references
+                        entity_data = entity.data
+                        if media_id_mapping:
+                            entity_data = MediaHandler.update_media_references(
+                                entity.data, media_id_mapping
+                            )
+
+                        if options.dry_run:
+                            result.entities_imported += 1
+                            continue
+
+                        # Get endpoint
+                        endpoint = self._get_endpoint(content_type)
+
+                        # Check for existing entity
+                        existing_id = None
+                        if entity.document_id:
+                            existing_id = self._check_entity_exists(endpoint, entity.document_id)
+
+                        if existing_id is not None:
+                            if options.conflict_resolution == ConflictResolution.SKIP:
+                                id_mappings[content_type][entity.id] = existing_id
+                                # Track document_id mappings for v5
+                                if entity.document_id:
+                                    doc_id_mappings[content_type][entity.id] = entity.document_id
+                                    doc_id_to_new_id_mappings[content_type][entity.document_id] = (
+                                        existing_id
+                                    )
+                                result.entities_skipped += 1
+                                continue
+                            elif options.conflict_resolution == ConflictResolution.FAIL:
+                                result.add_error(
+                                    f"Entity already exists: {content_type} {entity.document_id}"
+                                )
+                                result.entities_failed += 1
+                                continue
+                            elif options.conflict_resolution == ConflictResolution.UPDATE:
+                                # Update existing entity (use document_id for v5 endpoint)
+                                self.client.update(
+                                    f"{endpoint}/{entity.document_id}", data=entity_data
+                                )
+                                id_mappings[content_type][entity.id] = existing_id
+                                # Track document_id mappings for v5
+                                if entity.document_id:
+                                    doc_id_mappings[content_type][entity.id] = entity.document_id
+                                    doc_id_to_new_id_mappings[content_type][entity.document_id] = (
+                                        existing_id
+                                    )
+                                result.entities_updated += 1
+                                continue
+
+                        # Create new entity
+                        response = self.client.create(endpoint, data=entity_data)
+                        if response.data:
+                            id_mappings[content_type][entity.id] = response.data.id
+                            # Track document_id for v5 endpoint updates
+                            if response.data.document_id:
+                                doc_id_mappings[content_type][entity.id] = response.data.document_id
+                            # Track reverse mapping for v5 string relation resolution
+                            if entity.document_id:
+                                doc_id_to_new_id_mappings[content_type][entity.document_id] = (
+                                    response.data.id
+                                )
+                        result.entities_imported += 1
+
+                    except StrapiError as e:
+                        result.add_error(f"Failed to import {content_type} {entity.id}: {e}")
+                        result.entities_failed += 1
+
+            # ============================================================
+            # Pass 2: Re-read file to resolve relations using ID mappings
+            # True O(1) memory - entities processed one at a time
+            # ============================================================
+            if not options.skip_relations and not options.dry_run:
+                if options.progress_callback:
+                    options.progress_callback(70, 100, "Resolving relations (pass 2)")
+
+                with JSONLImportReader(jsonl_path) as reader2:
+                    # Skip metadata (already loaded)
+                    reader2.read_metadata()
+
+                    for entity in reader2.iter_entities():
+                        # Filter by content types if specified
+                        if (
+                            options.content_types
+                            and entity.content_type not in options.content_types
+                        ):
+                            continue
+
+                        # Skip entities without relations
+                        if not entity.relations:
+                            continue
+
+                        content_type = entity.content_type
+                        endpoint = self._get_endpoint(content_type)
+
+                        # Get new ID for this entity
+                        new_id = id_mappings.get(content_type, {}).get(entity.id)
+                        if new_id is None:
+                            continue
+
+                        # Get document_id for v5 endpoint (falls back to numeric ID for v4)
+                        entity_endpoint_id: str | int = new_id
+                        if (
+                            content_type in doc_id_mappings
+                            and entity.id in doc_id_mappings[content_type]
+                        ):
+                            entity_endpoint_id = doc_id_mappings[content_type][entity.id]
+
+                        # Get schema from cache
+                        try:
+                            schema = self._schema_cache.get_schema(content_type)
+                        except Exception:  # noqa: BLE001, S112 - Skip content types without schema
+                            continue
+
+                        try:
+                            # Resolve relations using ID mappings
+                            resolved = self._resolve_relations_with_schema(
+                                entity.relations,
+                                schema,
+                                id_mappings,
+                                doc_id_to_new_id_mappings,
+                            )
+
+                            if resolved:
+                                payload = RelationResolver.build_relation_payload(resolved)
+                                self.client.update(f"{endpoint}/{entity_endpoint_id}", data=payload)
+                                result.relations_imported += 1
+                        except StrapiError as e:
+                            result.add_warning(
+                                f"Failed to import relations for {content_type} {entity.id}: {e}"
+                            )
+
+            if options.progress_callback:
+                options.progress_callback(100, 100, "Import complete")
+
+            # Copy local mappings to result for caller access
+            result.id_mapping = id_mappings
+            result.doc_id_mapping = doc_id_mappings
+            result.doc_id_to_new_id = doc_id_to_new_id_mappings
+
+            result.success = result.entities_failed == 0
+            return result
+
+        except Exception as e:
+            result.add_error(f"JSONL import failed: {e}")
+            raise ImportExportError(f"JSONL import failed: {e}") from e