morphik 1.2.3__tar.gz → 1.2.4__tar.gz

{morphik-1.2.3 → morphik-1.2.4}/.gitignore

@@ -58,6 +58,3 @@ multi_vector_embeddings_*.json
 # Rust build artifacts
 morphik_rust/target/
 morphik_rust/Cargo.lock
-DOCS_BY_FACILITIES/
-eval_freshworks/
-iep_test/

{morphik-1.2.3 → morphik-1.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: morphik
-Version: 1.2.3
+Version: 1.2.4
 Summary: Morphik Python Client
 Author-email: Morphik <founders@morphik.ai>
 Requires-Python: >=3.8
@@ -91,6 +91,12 @@ response = db.query(
 )
 
 print(response.completion)
+
+# Migrate this app's documents into another Morphik deployment.
+# Run this from a machine that can reach both source and target, such as
+# inside a customer's VPN for on-prem targets.
+result = db.migrate(target_uri="morphik://owner_id:token@onprem.example.com", target_is_local=True)
+print(result.created_count, result.skipped_count, result.failed_count)
 ```
 
 ### Nested Folders & Folder Depth
@@ -106,11 +112,6 @@ renamed = moved.rename("specs-v2")
 # Scope queries to a path and include descendants with folder_depth=-1
 chunks = folder.retrieve_chunks(query="design notes", folder_depth=-1)
 docs = db.list_documents(folder_name="/projects/alpha", folder_depth=-1)
-
-# List only the fields you need. The server reads and returns just those columns, so
-# the full document text is never downloaded — fast for large corpora.
-for doc in db.list_documents(fields=["metadata"]).documents:
-    print(doc.external_id, doc.metadata)
 ```
 
 `Folder.full_path` is exposed on folder objects, and `Document.folder_path` mirrors server responses for tracing scope.

{morphik-1.2.3 → morphik-1.2.4}/README.md

@@ -78,6 +78,12 @@ response = db.query(
 )
 
 print(response.completion)
+
+# Migrate this app's documents into another Morphik deployment.
+# Run this from a machine that can reach both source and target, such as
+# inside a customer's VPN for on-prem targets.
+result = db.migrate(target_uri="morphik://owner_id:token@onprem.example.com", target_is_local=True)
+print(result.created_count, result.skipped_count, result.failed_count)
 ```
 
 ### Nested Folders & Folder Depth
@@ -93,11 +99,6 @@ renamed = moved.rename("specs-v2")
 # Scope queries to a path and include descendants with folder_depth=-1
 chunks = folder.retrieve_chunks(query="design notes", folder_depth=-1)
 docs = db.list_documents(folder_name="/projects/alpha", folder_depth=-1)
-
-# List only the fields you need. The server reads and returns just those columns, so
-# the full document text is never downloaded — fast for large corpora.
-for doc in db.list_documents(fields=["metadata"]).documents:
-    print(doc.external_id, doc.metadata)
 ```
 
 `Folder.full_path` is exposed on folder objects, and `Document.folder_path` mirrors server responses for tracing scope.

{morphik-1.2.3 → morphik-1.2.4}/morphik/__init__.py

@@ -3,7 +3,7 @@ Morphik Python SDK for document ingestion and querying.
 """
 
 from .async_ import AsyncMorphik
-from .models import Document, DocumentQueryResponse, Summary
+from .models import Document, DocumentQueryResponse, MigrationDocumentResult, MigrationResult, Summary
 from .sync import Morphik
 
 __all__ = [
@@ -12,6 +12,8 @@ __all__ = [
     "Document",
     "Summary",
     "DocumentQueryResponse",
+    "MigrationDocumentResult",
+    "MigrationResult",
 ]
 
-__version__ = "1.2.3"
+__version__ = "1.2.4"

{morphik-1.2.3 → morphik-1.2.4}/morphik/_internal.py

@@ -428,7 +428,6 @@ class _MorphikClientLogic:
         completed_only: bool,
         sort_by: Optional[str],
         sort_direction: str,
-        fields: Optional[List[str]] = None,
     ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
         """Prepare request for list_docs endpoint"""
         params = {}
@@ -451,15 +450,6 @@ class _MorphikClientLogic:
             "sort_by": sort_by,
             "sort_direction": sort_direction,
         }
-        if fields:
-            # Always include the fields required to reconstruct a Document client-side, so
-            # projected responses still parse into Document objects. When any metadata field
-            # is requested, also pull metadata_types so typed values (datetime/date/decimal)
-            # are reconstructed instead of returned as raw strings.
-            projected = ["external_id", "content_type", *fields]
-            if any(field.split(".", 1)[0] == "metadata" for field in fields):
-                projected.append("metadata_types")
-            data["fields"] = list(dict.fromkeys(projected))
         return params, data
 
     def _prepare_batch_get_documents_request(

{morphik-1.2.3 → morphik-1.2.4}/morphik/_scoped_ops.py

@@ -277,7 +277,6 @@ class _ScopedOperationsMixin:
         completed_only: bool,
         sort_by: Optional[str],
         sort_direction: str,
-        fields: Optional[List[str]] = None,
     ):
         params, data = self._logic._prepare_list_documents_request(
             skip,
@@ -292,7 +291,6 @@ class _ScopedOperationsMixin:
             completed_only,
             sort_by,
             sort_direction,
-            fields,
         )
 
         return self._execute_scoped_operation(

{morphik-1.2.3 → morphik-1.2.4}/morphik/_shared.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import json
 from pathlib import Path
-from typing import Any, Dict, Iterable, List, Optional, Union
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
 from urllib.parse import quote
 
 from pydantic import BaseModel
@@ -10,6 +10,17 @@ from pydantic import BaseModel
 MAX_LIMIT = 500
 MIN_LOG_HOURS = 0.1
 MAX_LOG_HOURS = 168.0
+MIGRATION_SOURCE_METADATA_KEY = "_morphik_migration"
+MIGRATION_RESERVED_METADATA_FIELDS = {
+    "app_id",
+    "end_user_id",
+    "external_id",
+    "filename",
+    "folder_id",
+    "folder_name",
+    "folder_path",
+    "owner_id",
+}
 
 
 def merge_folders(
@@ -197,3 +208,45 @@ def normalize_additional_folders(
     if additional_folders:
         return list(additional_folders) + folder_list
     return folder_list
+
+
+def build_migration_metadata(
+    document: Any,
+    *,
+    include_source_metadata: bool = True,
+) -> Tuple[Dict[str, Any], Dict[str, str]]:
+    """Prepare document metadata for migration ingestion.
+
+    The target API owns fields such as external_id, folder_name, and app_id, so
+    those values must travel through dedicated migration parameters instead of
+    user metadata.
+    """
+    metadata = dict(getattr(document, "metadata", None) or {})
+    metadata_types = dict(getattr(document, "metadata_types", None) or {})
+
+    for field in MIGRATION_RESERVED_METADATA_FIELDS:
+        metadata.pop(field, None)
+        metadata_types.pop(field, None)
+
+    if include_source_metadata:
+        system_metadata = getattr(document, "system_metadata", None) or {}
+        source_info = {
+            "source_document_id": getattr(document, "external_id", None),
+            "source_app_id": getattr(document, "app_id", None),
+            "source_filename": getattr(document, "filename", None),
+            "source_created_at": system_metadata.get("created_at") if isinstance(system_metadata, dict) else None,
+            "source_updated_at": system_metadata.get("updated_at") if isinstance(system_metadata, dict) else None,
+        }
+        source_info = {key: value for key, value in source_info.items() if value is not None}
+
+        existing_source_info = metadata.get(MIGRATION_SOURCE_METADATA_KEY)
+        if isinstance(existing_source_info, dict):
+            existing_source_info = dict(existing_source_info)
+            for key, value in source_info.items():
+                existing_source_info.setdefault(key, value)
+            metadata[MIGRATION_SOURCE_METADATA_KEY] = existing_source_info
+        else:
+            metadata[MIGRATION_SOURCE_METADATA_KEY] = source_info
+        metadata_types.setdefault(MIGRATION_SOURCE_METADATA_KEY, "object")
+
+    return metadata, metadata_types

{morphik-1.2.3 → morphik-1.2.4}/morphik/async_.py

@@ -1,3 +1,4 @@
+import inspect
 import json
 import logging
 import warnings
@@ -19,6 +20,7 @@ from ._shared import (
     build_folder_rename_path,
     build_list_apps_params,
     build_logs_params,
+    build_migration_metadata,
     build_rename_app_params,
     build_requeue_payload,
     build_rotate_app_params,
@@ -42,6 +44,8 @@ from .models import (
     IngestTextRequest,
     ListDocsResponse,
     LogResponse,
+    MigrationDocumentResult,
+    MigrationResult,
     QueryPromptOverrides,
     RequeueIngestionJob,
     RequeueIngestionResponse,
@@ -267,15 +271,8 @@ class _AsyncScopedClientOps:
         completed_only: bool = False,
         sort_by: Optional[str] = "updated_at",
         sort_direction: str = "desc",
-        fields: Optional[List[str]] = None,
     ) -> ListDocsResponse:
-        """List documents within this scope (async).
-
-        Args:
-            fields: Optional list of fields to return for each document (e.g. ["metadata"]).
-                Only those fields are read and returned, so the full document text is never
-                downloaded. external_id and content_type are always included.
-        """
+        """List documents within this scope (async)."""
         effective_folder = self._merge_folders(additional_folders)
         return await self._client._scoped_list_documents(
             skip=skip,
@@ -290,7 +287,6 @@ class _AsyncScopedClientOps:
             completed_only=completed_only,
             sort_by=sort_by,
             sort_direction=sort_direction,
-            fields=fields,
         )
 
     async def batch_get_documents(
@@ -1239,7 +1235,6 @@ class AsyncMorphik(_ScopedOperationsMixin):
         completed_only: bool = False,
         sort_by: Optional[str] = "updated_at",
         sort_direction: str = "desc",
-        fields: Optional[List[str]] = None,
     ) -> ListDocsResponse:
         """
         List accessible documents.
@@ -1256,9 +1251,6 @@ class AsyncMorphik(_ScopedOperationsMixin):
             completed_only: Only return completed documents
             sort_by: Field to sort by (created_at, updated_at, filename, external_id)
             sort_direction: Sort direction (asc, desc)
-            fields: Optional list of fields to return for each document (e.g. ["metadata"]).
-                Only those fields are read and returned, so the full document text is never
-                downloaded. external_id and content_type are always included.
         Returns:
             ListDocsResponse: Response with documents and metadata
 
@@ -1276,7 +1268,234 @@ class AsyncMorphik(_ScopedOperationsMixin):
             completed_only=completed_only,
             sort_by=sort_by,
             sort_direction=sort_direction,
-            fields=fields,
+        )
+
+    async def migrate(
+        self,
+        target_uri: str,
+        *,
+        filters: Optional[Dict[str, Any]] = None,
+        folder_name: Optional[Union[str, List[str]]] = None,
+        folder_depth: Optional[int] = None,
+        end_user_id: Optional[str] = None,
+        skip: int = 0,
+        limit: Optional[int] = None,
+        batch_size: int = 100,
+        completed_only: bool = True,
+        use_colpali: bool = True,
+        preserve_folders: bool = True,
+        preserve_end_user_id: bool = True,
+        preserve_summaries: bool = True,
+        include_source_metadata: bool = True,
+        on_conflict: Literal["skip", "fail"] = "skip",
+        continue_on_error: bool = True,
+        target_timeout: Optional[int] = None,
+        target_is_local: bool = False,
+        progress_callback: Optional[Callable[[MigrationDocumentResult], Any]] = None,
+    ) -> MigrationResult:
+        """Migrate documents from this client into another Morphik URI.
+
+        Set target_is_local=True for local/on-prem targets that should use HTTP
+        or skip TLS verification.
+        """
+        if batch_size <= 0:
+            raise ValueError("batch_size must be greater than 0")
+        if limit is not None and limit < 0:
+            raise ValueError("limit must be greater than or equal to 0")
+
+        target = AsyncMorphik(target_uri, timeout=target_timeout or self._logic._timeout, is_local=target_is_local)
+        results: List[MigrationDocumentResult] = []
+        total_source_count: Optional[int] = None
+        current_skip = max(skip, 0)
+        remaining = limit
+        page_size = min(batch_size, 500)
+
+        try:
+            while remaining is None or remaining > 0:
+                current_limit = page_size if remaining is None else min(page_size, remaining)
+                page = await self._scoped_list_documents(
+                    skip=current_skip,
+                    limit=current_limit,
+                    filters=filters,
+                    folder_name=folder_name,
+                    folder_depth=folder_depth,
+                    end_user_id=end_user_id,
+                    include_total_count=total_source_count is None,
+                    include_status_counts=False,
+                    include_folder_counts=False,
+                    completed_only=completed_only,
+                    sort_by="updated_at",
+                    sort_direction="desc",
+                )
+                if total_source_count is None:
+                    total_source_count = page.total_count
+                if not page.documents:
+                    break
+
+                for source_document in page.documents:
+                    try:
+                        result = await self._migrate_single_document(
+                            target=target,
+                            source_document=source_document,
+                            use_colpali=use_colpali,
+                            preserve_folders=preserve_folders,
+                            preserve_end_user_id=preserve_end_user_id,
+                            preserve_summaries=preserve_summaries,
+                            include_source_metadata=include_source_metadata,
+                            on_conflict=on_conflict,
+                        )
+                    except Exception as exc:  # noqa: BLE001
+                        result = MigrationDocumentResult(
+                            source_document_id=source_document.external_id,
+                            filename=source_document.filename,
+                            status="failed",
+                            error=str(exc),
+                        )
+                        if not continue_on_error:
+                            results.append(result)
+                            await self._emit_migration_progress(progress_callback, result)
+                            raise
+
+                    results.append(result)
+                    await self._emit_migration_progress(progress_callback, result)
+
+                if remaining is not None:
+                    remaining -= len(page.documents)
+                if not page.has_more:
+                    break
+                next_skip = page.next_skip if page.next_skip is not None else current_skip + page.returned_count
+                if next_skip <= current_skip:
+                    break
+                current_skip = next_skip
+        finally:
+            await target.close()
+
+        return self._build_migration_result(results, total_source_count)
+
+    async def _migrate_single_document(
+        self,
+        *,
+        target: "AsyncMorphik",
+        source_document: Document,
+        use_colpali: bool,
+        preserve_folders: bool,
+        preserve_end_user_id: bool,
+        preserve_summaries: bool,
+        include_source_metadata: bool,
+        on_conflict: Literal["skip", "fail"],
+    ) -> MigrationDocumentResult:
+        metadata, metadata_types = build_migration_metadata(
+            source_document,
+            include_source_metadata=include_source_metadata,
+        )
+        file_bytes = await self.get_document_file(source_document.external_id)
+        folder = (source_document.folder_path or source_document.folder_name) if preserve_folders else None
+        end_user = source_document.end_user_id if preserve_end_user_id else None
+
+        status, target_document = await target._ingest_migrated_document(
+            source_document_id=source_document.external_id,
+            file_content=file_bytes,
+            filename=source_document.filename or source_document.external_id,
+            content_type=source_document.content_type,
+            metadata=metadata,
+            metadata_types=metadata_types,
+            folder_name=folder,
+            end_user_id=end_user,
+            use_colpali=use_colpali,
+            on_conflict=on_conflict,
+        )
+
+        if preserve_summaries and status == "created":
+            await self._copy_document_summary(source_document.external_id, target, target_document.external_id)
+
+        return MigrationDocumentResult(
+            source_document_id=source_document.external_id,
+            target_document_id=target_document.external_id,
+            filename=source_document.filename,
+            status=status,
+        )
+
+    async def _ingest_migrated_document(
+        self,
+        *,
+        source_document_id: str,
+        file_content: bytes,
+        filename: str,
+        content_type: Optional[str],
+        metadata: Dict[str, Any],
+        metadata_types: Optional[Dict[str, str]],
+        folder_name: Optional[str],
+        end_user_id: Optional[str],
+        use_colpali: bool,
+        on_conflict: Literal["skip", "fail"],
+    ) -> tuple[str, Document]:
+        serialized_metadata, inferred_types = self._logic._serialize_metadata_map(metadata)
+        metadata_type_payload = {**inferred_types, **(metadata_types or {})}
+        metadata_type_payload = {
+            key: value for key, value in metadata_type_payload.items() if key in serialized_metadata
+        }
+
+        form_data: Dict[str, Any] = {
+            "source_document_id": source_document_id,
+            "metadata": json.dumps(serialized_metadata),
+            "metadata_types": json.dumps(metadata_type_payload),
+            "use_colpali": str(use_colpali).lower(),
+            "on_conflict": on_conflict,
+        }
+        if folder_name:
+            form_data["folder_name"] = folder_name
+        if end_user_id:
+            form_data["end_user_id"] = end_user_id
+
+        file_obj = BytesIO(file_content)
+        files = {"file": (filename, file_obj, content_type or "application/octet-stream")}
+        response = await self._request("POST", "migrate/document", data=form_data, files=files)
+        document = self._logic._parse_document_response(response["document"])
+        document._client = self
+        return response.get("status", "created"), document
+
+    async def _copy_document_summary(
+        self,
+        source_document_id: str,
+        target: "AsyncMorphik",
+        target_document_id: str,
+    ) -> None:
+        try:
+            summary = await self.get_document_summary(source_document_id)
+        except httpx.HTTPStatusError as exc:
+            if exc.response.status_code == 404:
+                return
+            raise
+        await target.upsert_document_summary(
+            document_id=target_document_id,
+            content=summary.content,
+            versioning=False,
+            overwrite_latest=True,
+        )
+
+    @staticmethod
+    async def _emit_migration_progress(
+        progress_callback: Optional[Callable[[MigrationDocumentResult], Any]],
+        result: MigrationDocumentResult,
+    ) -> None:
+        if progress_callback is None:
+            return
+        callback_result = progress_callback(result)
+        if inspect.isawaitable(callback_result):
+            await callback_result
+
+    @staticmethod
+    def _build_migration_result(
+        results: List[MigrationDocumentResult],
+        total_source_count: Optional[int],
+    ) -> MigrationResult:
+        return MigrationResult(
+            documents=results,
+            total_source_count=total_source_count,
+            attempted_count=len(results),
+            created_count=sum(1 for item in results if item.status == "created"),
+            skipped_count=sum(1 for item in results if item.status == "skipped"),
+            failed_count=sum(1 for item in results if item.status == "failed"),
         )
 
     async def get_document(self, document_id: str) -> Document:

{morphik-1.2.3 → morphik-1.2.4}/morphik/models.py

@@ -415,6 +415,27 @@ class ListDocsResponse(BaseModel):
     folder_counts: Optional[List[FolderCount]] = Field(None, description="Document counts by folder")
 
 
+class MigrationDocumentResult(BaseModel):
+    """Per-document result from a migration run."""
+
+    source_document_id: str = Field(..., description="Document ID in the source Morphik app")
+    target_document_id: Optional[str] = Field(None, description="Document ID in the target Morphik app")
+    filename: Optional[str] = Field(None, description="Migrated filename")
+    status: Literal["created", "skipped", "failed"] = Field(..., description="Migration outcome")
+    error: Optional[str] = Field(None, description="Error message when status is failed")
+
+
+class MigrationResult(BaseModel):
+    """Summary returned by client.migrate."""
+
+    documents: List[MigrationDocumentResult] = Field(default_factory=list)
+    total_source_count: Optional[int] = Field(None, description="Total matching source documents, when available")
+    attempted_count: int = Field(..., description="Number of source documents attempted")
+    created_count: int = Field(..., description="Number of target documents created")
+    skipped_count: int = Field(..., description="Number of target documents skipped because they already existed")
+    failed_count: int = Field(..., description="Number of documents that failed migration")
+
+
 class IngestTextRequest(BaseModel):
     """Request model for ingesting text content"""
 

{morphik-1.2.3 → morphik-1.2.4}/morphik/sync.py

@@ -19,6 +19,7 @@ from ._shared import (
     build_folder_rename_path,
     build_list_apps_params,
     build_logs_params,
+    build_migration_metadata,
     build_rename_app_params,
     build_requeue_payload,
     build_rotate_app_params,
@@ -42,6 +43,8 @@ from .models import (
     IngestTextRequest,
     ListDocsResponse,
     LogResponse,
+    MigrationDocumentResult,
+    MigrationResult,
     QueryPromptOverrides,
     RequeueIngestionJob,
     RequeueIngestionResponse,
@@ -283,16 +286,9 @@ class _ScopedClientOps:
         completed_only: bool = False,
         sort_by: Optional[str] = "updated_at",
         sort_direction: str = "desc",
-        fields: Optional[List[str]] = None,
     ) -> ListDocsResponse:
         """
         List documents within this scope.
-
-        Args:
-            fields: Optional list of fields to return for each document (e.g.
-                ["metadata"]). Only those fields are read and returned, so the full
-                document text is never downloaded. external_id and content_type are
-                always included.
         """
         effective_folder = self._merge_folders(additional_folders)
         return self._client._scoped_list_documents(
@@ -308,7 +304,6 @@ class _ScopedClientOps:
             completed_only=completed_only,
             sort_by=sort_by,
             sort_direction=sort_direction,
-            fields=fields,
         )
 
     def batch_get_documents(
@@ -1278,7 +1273,6 @@ class Morphik(_ScopedOperationsMixin):
         completed_only: bool = False,
         sort_by: Optional[str] = "updated_at",
         sort_direction: str = "desc",
-        fields: Optional[List[str]] = None,
     ) -> ListDocsResponse:
         """
         List accessible documents.
@@ -1295,9 +1289,6 @@ class Morphik(_ScopedOperationsMixin):
             completed_only: Only return completed documents
             sort_by: Field to sort by (created_at, updated_at, filename, external_id)
             sort_direction: Sort direction (asc, desc)
-            fields: Optional list of fields to return for each document (e.g. ["metadata"]).
-                Only those fields are read and returned, so the full document text is never
-                downloaded. external_id and content_type are always included.
         Returns:
             ListDocsResponse: Response with documents and metadata
 
@@ -1315,7 +1306,222 @@ class Morphik(_ScopedOperationsMixin):
             completed_only=completed_only,
             sort_by=sort_by,
             sort_direction=sort_direction,
-            fields=fields,
+        )
+
+    def migrate(
+        self,
+        target_uri: str,
+        *,
+        filters: Optional[Dict[str, Any]] = None,
+        folder_name: Optional[Union[str, List[str]]] = None,
+        folder_depth: Optional[int] = None,
+        end_user_id: Optional[str] = None,
+        skip: int = 0,
+        limit: Optional[int] = None,
+        batch_size: int = 100,
+        completed_only: bool = True,
+        use_colpali: bool = True,
+        preserve_folders: bool = True,
+        preserve_end_user_id: bool = True,
+        preserve_summaries: bool = True,
+        include_source_metadata: bool = True,
+        on_conflict: Literal["skip", "fail"] = "skip",
+        continue_on_error: bool = True,
+        target_timeout: Optional[int] = None,
+        target_is_local: bool = False,
+        progress_callback: Optional[Callable[[MigrationDocumentResult], None]] = None,
+    ) -> MigrationResult:
+        """Migrate documents from this client into another Morphik URI.
+
+        The caller should run this from a network location that can reach both
+        the source URI and the target URI, for example from inside a customer's
+        VPN when the target deployment is on-prem. Set target_is_local=True for
+        local/on-prem targets that should use HTTP or skip TLS verification.
+        """
+        if batch_size <= 0:
+            raise ValueError("batch_size must be greater than 0")
+        if limit is not None and limit < 0:
+            raise ValueError("limit must be greater than or equal to 0")
+
+        target = Morphik(target_uri, timeout=target_timeout or self._logic._timeout, is_local=target_is_local)
+        results: List[MigrationDocumentResult] = []
+        total_source_count: Optional[int] = None
+        current_skip = max(skip, 0)
+        remaining = limit
+        page_size = min(batch_size, 500)
+
+        try:
+            while remaining is None or remaining > 0:
+                current_limit = page_size if remaining is None else min(page_size, remaining)
+                page = self._scoped_list_documents(
+                    skip=current_skip,
+                    limit=current_limit,
+                    filters=filters,
+                    folder_name=folder_name,
+                    folder_depth=folder_depth,
+                    end_user_id=end_user_id,
+                    include_total_count=total_source_count is None,
+                    include_status_counts=False,
+                    include_folder_counts=False,
+                    completed_only=completed_only,
+                    sort_by="updated_at",
+                    sort_direction="desc",
+                )
+                if total_source_count is None:
+                    total_source_count = page.total_count
+                if not page.documents:
+                    break
+
+                for source_document in page.documents:
+                    try:
+                        result = self._migrate_single_document(
+                            target=target,
+                            source_document=source_document,
+                            use_colpali=use_colpali,
+                            preserve_folders=preserve_folders,
+                            preserve_end_user_id=preserve_end_user_id,
+                            preserve_summaries=preserve_summaries,
+                            include_source_metadata=include_source_metadata,
+                            on_conflict=on_conflict,
+                        )
+                    except Exception as exc:  # noqa: BLE001
+                        result = MigrationDocumentResult(
+                            source_document_id=source_document.external_id,
+                            filename=source_document.filename,
+                            status="failed",
+                            error=str(exc),
+                        )
+                        if not continue_on_error:
+                            results.append(result)
+                            if progress_callback:
+                                progress_callback(result)
+                            raise
+
+                    results.append(result)
+                    if progress_callback:
+                        progress_callback(result)
+
+                if remaining is not None:
+                    remaining -= len(page.documents)
+                if not page.has_more:
+                    break
+                next_skip = page.next_skip if page.next_skip is not None else current_skip + page.returned_count
+                if next_skip <= current_skip:
+                    break
+                current_skip = next_skip
+        finally:
+            target.close()
+
+        return self._build_migration_result(results, total_source_count)
+
+    def _migrate_single_document(
+        self,
+        *,
+        target: "Morphik",
+        source_document: Document,
+        use_colpali: bool,
+        preserve_folders: bool,
+        preserve_end_user_id: bool,
+        preserve_summaries: bool,
+        include_source_metadata: bool,
+        on_conflict: Literal["skip", "fail"],
+    ) -> MigrationDocumentResult:
+        metadata, metadata_types = build_migration_metadata(
+            source_document,
+            include_source_metadata=include_source_metadata,
+        )
+        file_bytes = self.get_document_file(source_document.external_id)
+        folder = (source_document.folder_path or source_document.folder_name) if preserve_folders else None
+        end_user = source_document.end_user_id if preserve_end_user_id else None
+
+        status, target_document = target._ingest_migrated_document(
+            source_document_id=source_document.external_id,
+            file_content=file_bytes,
+            filename=source_document.filename or source_document.external_id,
+            content_type=source_document.content_type,
+            metadata=metadata,
+            metadata_types=metadata_types,
+            folder_name=folder,
+            end_user_id=end_user,
+            use_colpali=use_colpali,
+            on_conflict=on_conflict,
+        )
+
+        if preserve_summaries and status == "created":
+            self._copy_document_summary(source_document.external_id, target, target_document.external_id)
+
+        return MigrationDocumentResult(
+            source_document_id=source_document.external_id,
+            target_document_id=target_document.external_id,
+            filename=source_document.filename,
+            status=status,
+        )
+
+    def _ingest_migrated_document(
+        self,
+        *,
+        source_document_id: str,
+        file_content: bytes,
+        filename: str,
+        content_type: Optional[str],
+        metadata: Dict[str, Any],
+        metadata_types: Optional[Dict[str, str]],
+        folder_name: Optional[str],
+        end_user_id: Optional[str],
+        use_colpali: bool,
+        on_conflict: Literal["skip", "fail"],
+    ) -> tuple[str, Document]:
+        serialized_metadata, inferred_types = self._logic._serialize_metadata_map(metadata)
+        metadata_type_payload = {**inferred_types, **(metadata_types or {})}
+        metadata_type_payload = {
+            key: value for key, value in metadata_type_payload.items() if key in serialized_metadata
+        }
+
+        form_data: Dict[str, Any] = {
+            "source_document_id": source_document_id,
+            "metadata": json.dumps(serialized_metadata),
+            "metadata_types": json.dumps(metadata_type_payload),
+            "use_colpali": str(use_colpali).lower(),
+            "on_conflict": on_conflict,
+        }
+        if folder_name:
+            form_data["folder_name"] = folder_name
+        if end_user_id:
+            form_data["end_user_id"] = end_user_id
+
+        file_obj = BytesIO(file_content)
+        files = {"file": (filename, file_obj, content_type or "application/octet-stream")}
+        response = self._request("POST", "migrate/document", data=form_data, files=files)
+        document = self._logic._parse_document_response(response["document"])
+        document._client = self
+        return response.get("status", "created"), document
+
+    def _copy_document_summary(self, source_document_id: str, target: "Morphik", target_document_id: str) -> None:
+        try:
+            summary = self.get_document_summary(source_document_id)
+        except httpx.HTTPStatusError as exc:
+            if exc.response.status_code == 404:
+                return
+            raise
+        target.upsert_document_summary(
+            document_id=target_document_id,
+            content=summary.content,
+            versioning=False,
+            overwrite_latest=True,
+        )
+
+    @staticmethod
+    def _build_migration_result(
+        results: List[MigrationDocumentResult],
+        total_source_count: Optional[int],
+    ) -> MigrationResult:
+        return MigrationResult(
+            documents=results,
+            total_source_count=total_source_count,
+            attempted_count=len(results),
+            created_count=sum(1 for item in results if item.status == "created"),
+            skipped_count=sum(1 for item in results if item.status == "skipped"),
+            failed_count=sum(1 for item in results if item.status == "failed"),
         )
 
     def get_document(self, document_id: str) -> Document:

{morphik-1.2.3 → morphik-1.2.4}/morphik/tests/test_scoped_ops_unit.py

@@ -1,3 +1,5 @@
+import json
+
 import httpx
 import jwt
 import pytest
@@ -117,39 +119,6 @@ def test_sync_list_documents_payloads_across_scopes():
         client.close()
 
 
-def test_sync_list_documents_fields_projection():
-    client, calls = _make_sync_client()
-    try:
-        # external_id + content_type are always added so the response parses into a Document;
-        # metadata_types is added so typed metadata values are reconstructed, not left as strings.
-        client.list_documents(fields=["metadata"])
-        assert calls.pop()["data"]["fields"] == ["external_id", "content_type", "metadata", "metadata_types"]
-
-        # Already-included required fields are not duplicated; order is preserved.
-        client.list_documents(fields=["external_id", "filename", "metadata"])
-        assert calls.pop()["data"]["fields"] == [
-            "external_id",
-            "content_type",
-            "filename",
-            "metadata",
-            "metadata_types",
-        ]
-
-        # Nested metadata paths also trigger metadata_types.
-        client.list_documents(fields=["metadata.client"])
-        assert calls.pop()["data"]["fields"] == ["external_id", "content_type", "metadata.client", "metadata_types"]
-
-        # Non-metadata projection does not pull metadata_types.
-        client.list_documents(fields=["filename"])
-        assert calls.pop()["data"]["fields"] == ["external_id", "content_type", "filename"]
-
-        # No fields -> no projection requested (full documents).
-        client.list_documents()
-        assert "fields" not in calls.pop()["data"]
-    finally:
-        client.close()
-
-
 def test_async_client_http2_toggle(monkeypatch):
     captured = []
 
@@ -338,6 +307,59 @@ def test_sync_get_document_by_filename_scoped_params_and_encoding():
         client.close()
 
 
+def test_sync_ingest_migrated_document_posts_migration_payload():
+    client = Morphik()
+    calls = []
+
+    def fake_request(method, endpoint, data=None, files=None, params=None):
+        calls.append({"method": method, "endpoint": endpoint, "data": data, "files": files, "params": params})
+        return {
+            "status": "created",
+            "document": {
+                "external_id": "source-doc-1",
+                "content_type": "application/pdf",
+                "filename": "report.pdf",
+            },
+        }
+
+    client._request = fake_request  # type: ignore[attr-defined]
+    try:
+        status, doc = client._ingest_migrated_document(
+            source_document_id="source-doc-1",
+            file_content=b"pdf-bytes",
+            filename="report.pdf",
+            content_type="application/pdf",
+            metadata={"category": "finance", "_morphik_migration": {"source_document_id": "source-doc-1"}},
+            metadata_types={"category": "string", "_morphik_migration": "object"},
+            folder_name="/finance/reports",
+            end_user_id="customer-1",
+            use_colpali=True,
+            on_conflict="skip",
+        )
+        call = calls.pop()
+        assert call["method"] == "POST"
+        assert call["endpoint"] == "migrate/document"
+        assert call["data"]["source_document_id"] == "source-doc-1"
+        assert call["data"]["folder_name"] == "/finance/reports"
+        assert call["data"]["end_user_id"] == "customer-1"
+        assert call["data"]["use_colpali"] == "true"
+        assert call["data"]["on_conflict"] == "skip"
+        assert json.loads(call["data"]["metadata"]) == {
+            "category": "finance",
+            "_morphik_migration": {"source_document_id": "source-doc-1"},
+        }
+        assert json.loads(call["data"]["metadata_types"]) == {
+            "category": "string",
+            "_morphik_migration": "object",
+        }
+        assert call["files"]["file"][0] == "report.pdf"
+        assert call["files"]["file"][2] == "application/pdf"
+        assert status == "created"
+        assert doc.external_id == "source-doc-1"
+    finally:
+        client.close()
+
+
 def test_sync_folder_get_document_by_filename_scoped():
     client, calls = _make_sync_client()
     try:

{morphik-1.2.3 → morphik-1.2.4}/morphik/tests/test_shared_helpers.py

@@ -7,6 +7,7 @@ from morphik._shared import (
     build_document_by_filename_params,
     build_list_apps_params,
     build_logs_params,
+    build_migration_metadata,
     build_rename_app_params,
     build_requeue_payload,
     build_rotate_app_params,
@@ -14,7 +15,7 @@ from morphik._shared import (
     merge_folders,
     normalize_additional_folders,
 )
-from morphik.models import RequeueIngestionJob
+from morphik.models import Document, RequeueIngestionJob
 
 
 def test_merge_folders_variants():
@@ -119,3 +120,37 @@ def test_normalize_additional_folders_alias():
     assert normalize_additional_folders(None, "b") == ["b"]
     assert normalize_additional_folders(["a"], "b") == ["a", "b"]
     assert normalize_additional_folders(["a"], ["b", "c"]) == ["a", "b", "c"]
+
+
+def test_build_migration_metadata_strips_managed_fields_and_records_source():
+    doc = Document(
+        external_id="source-doc-1",
+        content_type="application/pdf",
+        filename="report.pdf",
+        app_id="source-app",
+        folder_path="/finance/reports",
+        end_user_id="customer-1",
+        metadata={
+            "external_id": "source-doc-1",
+            "folder_name": "/finance/reports",
+            "end_user_id": "customer-1",
+            "category": "finance",
+        },
+        metadata_types={
+            "external_id": "string",
+            "folder_name": "string",
+            "end_user_id": "string",
+            "category": "string",
+        },
+        system_metadata={"created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z"},
+    )
+
+    metadata, metadata_types = build_migration_metadata(doc)
+
+    assert metadata["category"] == "finance"
+    assert "external_id" not in metadata
+    assert "folder_name" not in metadata
+    assert "end_user_id" not in metadata
+    assert metadata["_morphik_migration"]["source_document_id"] == "source-doc-1"
+    assert metadata["_morphik_migration"]["source_app_id"] == "source-app"
+    assert metadata_types == {"category": "string", "_morphik_migration": "object"}

{morphik-1.2.3 → morphik-1.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "morphik"
-version = "1.2.3"
+version = "1.2.4"
 authors = [
     { name = "Morphik", email = "founders@morphik.ai" },
 ]