morphik 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

morphik/sync.py

@@ -16,6 +16,7 @@ from .models import (
     IngestTextRequest,
     ChunkSource,
     Graph,
+    FolderInfo,
     # Prompt override models
     GraphPromptOverrides,
     QueryPromptOverrides,
@@ -58,16 +59,43 @@ class Folder:
     Args:
         client: The Morphik client instance
         name: The name of the folder
+        folder_id: Optional folder ID (if already known)
     """
 
-    def __init__(self, client: "Morphik", name: str):
+    def __init__(self, client: "Morphik", name: str, folder_id: Optional[str] = None):
         self._client = client
         self._name = name
+        self._id = folder_id
 
     @property
     def name(self) -> str:
         """Returns the folder name."""
         return self._name
+        
+    @property
+    def id(self) -> Optional[str]:
+        """Returns the folder ID if available."""
+        return self._id
+        
+    def get_info(self) -> Dict[str, Any]:
+        """
+        Get detailed information about this folder.
+        
+        Returns:
+            Dict[str, Any]: Detailed folder information
+        """
+        if not self._id:
+            # If we don't have the ID, find the folder by name first
+            folders = self._client.list_folders()
+            for folder in folders:
+                if folder.name == self._name:
+                    self._id = folder.id
+                    break
+            if not self._id:
+                raise ValueError(f"Folder '{self._name}' not found")
+        
+        return self._client._request("GET", f"folders/{self._id}")
+        
 
     def signin(self, end_user_id: str) -> "UserScope":
         """
@@ -144,11 +172,13 @@ class Folder:
                 metadata, rules, self._name, None
             )
 
+            # use_colpali should be a query parameter as defined in the API
             response = self._client._request(
                 "POST",
-                f"ingest/file?use_colpali={str(use_colpali).lower()}",
+                "ingest/file",
                 data=form_data,
                 files=files,
+                params={"use_colpali": str(use_colpali).lower()},
             )
             doc = self._client._logic._parse_document_response(response)
             doc._client = self._client
@@ -188,7 +218,13 @@ class Folder:
                 metadata, rules, use_colpali, parallel, self._name, None
             )
 
-            response = self._client._request("POST", "ingest/files", data=data, files=file_objects)
+            response = self._client._request(
+                "POST", 
+                "ingest/files", 
+                data=data, 
+                files=file_objects,
+                params={"use_colpali": str(use_colpali).lower()},
+            )
 
             if response.get("errors"):
                 # Log errors but don't raise exception
@@ -641,12 +677,14 @@ class UserScope:
             # Add folder name if scoped to a folder
             if self._folder_name:
                 form_data["folder_name"] = self._folder_name
-
+                
+            # use_colpali should be a query parameter as defined in the API
             response = self._client._request(
                 "POST",
-                f"ingest/file?use_colpali={str(use_colpali).lower()}",
+                "ingest/file",
                 data=form_data,
                 files=files,
+                params={"use_colpali": str(use_colpali).lower()},
             )
             doc = self._client._logic._parse_document_response(response)
             doc._client = self._client
@@ -706,7 +744,7 @@ class UserScope:
             data = {
                 "metadata": json.dumps(metadata or {}),
                 "rules": json.dumps(converted_rules),
-                "use_colpali": str(use_colpali).lower() if use_colpali is not None else None,
+                # Remove use_colpali from form data - it should be a query param
                 "parallel": str(parallel).lower(),
                 "end_user_id": self._end_user_id,  # Add end user ID here
             }
@@ -715,7 +753,13 @@ class UserScope:
             if self._folder_name:
                 data["folder_name"] = self._folder_name
 
-            response = self._client._request("POST", "ingest/files", data=data, files=file_objects)
+            response = self._client._request(
+                "POST", 
+                "ingest/files", 
+                data=data, 
+                files=file_objects,
+                params={"use_colpali": str(use_colpali).lower()},
+            )
 
             if response.get("errors"):
                 # Log errors but don't raise exception
@@ -1125,9 +1169,17 @@ class Morphik:
 
         # Configure request data based on type
         if files:
-            # Multipart form data for files
-            request_data = {"files": files, "data": data}
-            # Don't set Content-Type, let httpx handle it
+            # When uploading files, we need to make sure not to set Content-Type
+            # Remove Content-Type if it exists - httpx will set the correct multipart boundary
+            if "Content-Type" in headers:
+                del headers["Content-Type"]
+                
+            # For file uploads with form data, use form data (not json)
+            request_data = {"files": files}
+            if data:
+                request_data["data"] = data
+                
+            # Files are now properly handled
         else:
             # JSON for everything else
             headers["Content-Type"] = "application/json"
@@ -1147,19 +1199,30 @@ class Morphik:
         """Convert a rule to a dictionary format"""
         return self._logic._convert_rule(rule)
 
-    def create_folder(self, name: str) -> Folder:
+    def create_folder(self, name: str, description: Optional[str] = None) -> Folder:
         """
         Create a folder to scope operations.
 
         Args:
             name: The name of the folder
+            description: Optional description for the folder
 
         Returns:
-            Folder: A folder object for scoped operations
+            Folder: A folder object ready for scoped operations
         """
-        return Folder(self, name)
-
-    def get_folder(self, name: str) -> Folder:
+        payload = {
+            "name": name
+        }
+        if description:
+            payload["description"] = description
+            
+        response = self._request("POST", "folders", data=payload)
+        folder_info = FolderInfo(**response)
+        
+        # Return a usable Folder object with the ID from the response
+        return Folder(self, name, folder_id=folder_info.id)
+    
+    def get_folder_by_name(self, name: str) -> Folder:
         """
         Get a folder by name to scope operations.
 
@@ -1170,6 +1233,57 @@ class Morphik:
             Folder: A folder object for scoped operations
         """
         return Folder(self, name)
+        
+    def get_folder(self, folder_id: str) -> Folder:
+        """
+        Get a folder by ID.
+
+        Args:
+            folder_id: ID of the folder
+
+        Returns:
+            Folder: A folder object for scoped operations
+        """
+        response = self._request("GET", f"folders/{folder_id}")
+        return Folder(self, response["name"], folder_id)
+
+    def list_folders(self) -> List[Folder]:
+        """
+        List all folders the user has access to as Folder objects.
+        
+        Returns:
+            List[Folder]: List of Folder objects ready for operations
+        """
+        folder_infos = self._request("GET", "folders")
+        return [Folder(self, info["name"], info["id"]) for info in folder_infos]
+        
+    def add_document_to_folder(self, folder_id: str, document_id: str) -> Dict[str, str]:
+        """
+        Add a document to a folder.
+
+        Args:
+            folder_id: ID of the folder
+            document_id: ID of the document
+
+        Returns:
+            Dict[str, str]: Success status
+        """
+        response = self._request("POST", f"folders/{folder_id}/documents/{document_id}")
+        return response
+        
+    def remove_document_from_folder(self, folder_id: str, document_id: str) -> Dict[str, str]:
+        """
+        Remove a document from a folder.
+
+        Args:
+            folder_id: ID of the folder
+            document_id: ID of the document
+
+        Returns:
+            Dict[str, str]: Success status
+        """
+        response = self._request("DELETE", f"folders/{folder_id}/documents/{document_id}")
+        return response
 
     def signin(self, end_user_id: str) -> UserScope:
         """
@@ -1290,11 +1404,13 @@ class Morphik:
             # Create form data
             form_data = self._logic._prepare_ingest_file_form_data(metadata, rules, None, None)
 
+            # use_colpali should be a query parameter as defined in the API
             response = self._request(
                 "POST",
-                f"ingest/file?use_colpali={str(use_colpali).lower()}",
+                "ingest/file",
                 data=form_data,
                 files=files,
+                params={"use_colpali": str(use_colpali).lower()},
             )
             doc = self._logic._parse_document_response(response)
             doc._client = self
@@ -1333,11 +1449,18 @@ class Morphik:
 
         try:
             # Prepare form data
+            # Prepare form data - use_colpali should be a query parameter, not form data
             data = self._logic._prepare_ingest_files_form_data(
                 metadata, rules, use_colpali, parallel, None, None
             )
 
-            response = self._request("POST", "ingest/files", data=data, files=file_objects)
+            response = self._request(
+                "POST", 
+                "ingest/files", 
+                data=data, 
+                files=file_objects,
+                params={"use_colpali": str(use_colpali).lower()},
+            )
 
             if response.get("errors"):
                 # Log errors but don't raise exception
@@ -1618,6 +1741,76 @@ class Morphik:
         doc = self._logic._parse_document_response(response)
         doc._client = self
         return doc
+        
+    def get_document_status(self, document_id: str) -> Dict[str, Any]:
+        """
+        Get the current processing status of a document.
+        
+        Args:
+            document_id: ID of the document to check
+            
+        Returns:
+            Dict[str, Any]: Status information including current status, potential errors, and other metadata
+            
+        Example:
+            ```python
+            status = db.get_document_status("doc_123")
+            if status["status"] == "completed":
+                print("Document processing complete")
+            elif status["status"] == "failed":
+                print(f"Processing failed: {status['error']}")
+            else:
+                print("Document still processing...")
+            ```
+        """
+        response = self._request("GET", f"documents/{document_id}/status")
+        return response
+    
+    def wait_for_document_completion(self, document_id: str, timeout_seconds=300, check_interval_seconds=2) -> Document:
+        """
+        Wait for a document's processing to complete.
+        
+        Args:
+            document_id: ID of the document to wait for
+            timeout_seconds: Maximum time to wait for completion (default: 300 seconds)
+            check_interval_seconds: Time between status checks (default: 2 seconds)
+            
+        Returns:
+            Document: Updated document with the latest status
+            
+        Raises:
+            TimeoutError: If processing doesn't complete within the timeout period
+            ValueError: If processing fails with an error
+            
+        Example:
+            ```python
+            # Upload a file and wait for processing to complete
+            doc = db.ingest_file("large_document.pdf")
+            try:
+                completed_doc = db.wait_for_document_completion(doc.external_id)
+                print(f"Processing complete! Document has {len(completed_doc.chunk_ids)} chunks")
+            except TimeoutError:
+                print("Processing is taking too long")
+            except ValueError as e:
+                print(f"Processing failed: {e}")
+            ```
+        """
+        import time
+        start_time = time.time()
+        
+        while (time.time() - start_time) < timeout_seconds:
+            status = self.get_document_status(document_id)
+            
+            if status["status"] == "completed":
+                # Get the full document now that it's complete
+                return self.get_document(document_id)
+            elif status["status"] == "failed":
+                raise ValueError(f"Document processing failed: {status.get('error', 'Unknown error')}")
+            
+            # Wait before checking again
+            time.sleep(check_interval_seconds)
+        
+        raise TimeoutError(f"Document processing did not complete within {timeout_seconds} seconds")
 
     def get_document_by_filename(self, filename: str) -> Document:
         """
@@ -1991,7 +2184,8 @@ class Morphik:
                 print(f"Document {doc.external_id}: {doc.metadata.get('title')}")
             ```
         """
-        response = self._request("POST", "batch/documents", data=document_ids)
+        # API expects a dict with document_ids key, not a direct list
+        response = self._request("POST", "batch/documents", data={"document_ids": document_ids})
         docs = self._logic._parse_document_list_response(response)
         for doc in docs:
             doc._client = self

morphik/tests/README.md

@@ -0,0 +1,41 @@
+# Morphik SDK Tests
+
+This directory contains tests and example code for the Morphik SDK.
+
+## Test Types
+
+- `test_sync.py` - Tests for the synchronous client
+- `test_async.py` - Tests for the asynchronous client
+
+### Test Data
+- `test_docs/` - Sample text files for testing document ingestion
+
+### Example Code
+- `example_usage.py` - Example script demonstrating basic usage of the SDK
+
+## Running Tests
+
+```bash
+# Using default localhost:8000 URL
+pytest test_sync.py test_async.py -v
+
+# Tests connect to localhost:8000 by default
+# No need to specify a URL unless you want to test against a different server
+
+# With a custom server URL (optional)
+MORPHIK_TEST_URL=http://custom-url:8000 pytest test_sync.py -v
+```
+
+### Example Usage Script
+```bash
+# Run synchronous example
+python example_usage.py
+
+# Run asynchronous example
+python example_usage.py --async
+```
+
+## Environment Variables
+
+- `MORPHIK_TEST_URL` - The URL of the Morphik server to use for tests (default: http://localhost:8000)
+- `SKIP_LIVE_TESTS` - Set to "1" to skip tests that require a running server

morphik/tests/example_usage.py

@@ -0,0 +1,280 @@
+#!/usr/bin/env python
+"""
+Example script demonstrating basic usage of the Morphik SDK.
+This can be run to verify that the SDK is working correctly.
+
+Usage:
+    python example_usage.py [--async]
+    
+Options:
+    --async    Run the example using the async client
+"""
+
+import os
+import sys
+import time
+from pathlib import Path
+import argparse
+
+
+def run_sync_example():
+    """Run synchronous SDK examples"""
+    from morphik import Morphik
+    
+    # Get the test files directory - this script is in the tests directory
+    test_docs_dir = Path(__file__).parent / "test_docs"
+    
+    print("Running Morphik SDK Sync Example")
+    print("===============================")
+    
+    # Initialize the client - using default localhost:8000
+    print("\n1. Initializing Morphik client...")
+    db = Morphik()  # Connects to localhost:8000 by default
+    print(f"   Connected to {db._logic._base_url}")
+    
+    try:
+        # Ingest a text document
+        print("\n2. Ingesting a text document...")
+        text_doc = db.ingest_text(
+            content="This is a sample document created using the Morphik SDK. "
+                  "It demonstrates the text ingestion capabilities.",
+            filename="sdk_example.txt",
+            metadata={"source": "sdk_example", "type": "text"}
+        )
+        print(f"   Document created with ID: {text_doc.external_id}")
+        print(f"   Filename: {text_doc.filename}")
+        print(f"   Metadata: {text_doc.metadata}")
+        
+        # Ingest a file
+        print("\n3. Ingesting a file from disk...")
+        file_path = test_docs_dir / "sample1.txt"
+        file_doc = db.ingest_file(
+            file=file_path,
+            metadata={"source": "sdk_example", "type": "file"}
+        )
+        print(f"   Document created with ID: {file_doc.external_id}")
+        print(f"   Filename: {file_doc.filename}")
+        
+        # Create a folder
+        print("\n4. Creating a folder...")
+        folder = db.create_folder(name="sdk_example_folder", description="Example folder created by SDK")
+        print(f"   Folder created with name: {folder.name}")
+        print(f"   Folder ID: {folder.id}")
+        
+        # Ingest document into folder
+        print("\n5. Ingesting a document into the folder...")
+        folder_doc = folder.ingest_text(
+            content="This document is stored in a specific folder.",
+            filename="folder_example.txt",
+            metadata={"source": "sdk_example", "type": "folder_doc"}
+        )
+        print(f"   Document created with ID: {folder_doc.external_id}")
+        
+        # Create a user scope
+        print("\n6. Creating a user scope...")
+        user = db.signin("sdk_example_user")
+        print(f"   User scope created for: {user.end_user_id}")
+        
+        # Ingest document as user
+        print("\n7. Ingesting a document as this user...")
+        user_doc = user.ingest_text(
+            content="This document is associated with a specific user.",
+            filename="user_example.txt",
+            metadata={"source": "sdk_example", "type": "user_doc"}
+        )
+        print(f"   Document created with ID: {user_doc.external_id}")
+        
+        # Wait for processing to complete
+        print("\n8. Waiting for documents to be processed...")
+        for _ in range(10):
+            status = db.get_document_status(text_doc.external_id)
+            if status.get("status") == "completed":
+                print(f"   Document {text_doc.external_id} is now processed")
+                break
+            print(f"   Document status: {status.get('status')}. Waiting...")
+            time.sleep(3)
+        
+        # Search using retrieve_chunks
+        print("\n9. Retrieving relevant chunks...")
+        chunks = db.retrieve_chunks(
+            query="What is this document about?",
+            filters={"source": "sdk_example"},
+            k=2
+        )
+        print(f"   Found {len(chunks)} chunks")
+        for i, chunk in enumerate(chunks):
+            print(f"   Chunk {i+1}: Score {chunk.score}")
+            print(f"      Content: {chunk.content[:50]}...")
+        
+        # Query using RAG
+        print("\n10. Generating a completion using RAG...")
+        completion = db.query(
+            query="Summarize what these documents contain",
+            filters={"source": "sdk_example"},
+            k=3,
+            temperature=0.7
+        )
+        print(f"   Completion: {completion.completion}")
+        print(f"   Using {len(completion.sources)} sources")
+        for i, source in enumerate(completion.sources):
+            print(f"      Source {i+1}: Document {source.document_id}, Chunk {source.chunk_number}")
+        
+        # List documents
+        print("\n11. Listing documents...")
+        docs = db.list_documents(filters={"source": "sdk_example"})
+        print(f"   Found {len(docs)} documents")
+        for i, doc in enumerate(docs):
+            print(f"   Document {i+1}: {doc.filename} (ID: {doc.external_id})")
+        
+        # Cleanup
+        print("\n12. Cleaning up test documents...")
+        # Delete the documents in reverse order (won't delete folder)
+        doc_ids = [user_doc.external_id, folder_doc.external_id, file_doc.external_id, text_doc.external_id]
+        for doc_id in doc_ids:
+            result = db.delete_document(doc_id)
+            print(f"   Deleted document {doc_id}: {result.get('message', 'No message')}")
+        
+        print("\nExample completed successfully!")
+    
+    finally:
+        db.close()
+
+
+async def run_async_example():
+    """Run asynchronous SDK examples"""
+    import asyncio
+    from morphik.async_ import AsyncMorphik
+    
+    # Get the test files directory - this script is in the tests directory
+    test_docs_dir = Path(__file__).parent / "test_docs"
+    
+    print("Running Morphik SDK Async Example")
+    print("================================")
+    
+    # Initialize the client - using default localhost:8000
+    print("\n1. Initializing AsyncMorphik client...")
+    async with AsyncMorphik() as db:  # Connects to localhost:8000 by default
+        print(f"   Connected to {db._logic._base_url}")
+        
+        try:
+            # Ingest a text document
+            print("\n2. Ingesting a text document...")
+            text_doc = await db.ingest_text(
+                content="This is a sample document created using the Morphik SDK async client. "
+                      "It demonstrates the text ingestion capabilities.",
+                filename="async_sdk_example.txt",
+                metadata={"source": "async_sdk_example", "type": "text"}
+            )
+            print(f"   Document created with ID: {text_doc.external_id}")
+            print(f"   Filename: {text_doc.filename}")
+            print(f"   Metadata: {text_doc.metadata}")
+            
+            # Ingest a file
+            print("\n3. Ingesting a file from disk...")
+            file_path = test_docs_dir / "sample2.txt"
+            file_doc = await db.ingest_file(
+                file=file_path,
+                metadata={"source": "async_sdk_example", "type": "file"}
+            )
+            print(f"   Document created with ID: {file_doc.external_id}")
+            print(f"   Filename: {file_doc.filename}")
+            
+            # Create a folder
+            print("\n4. Creating a folder...")
+            folder = await db.create_folder(name="async_sdk_example_folder", description="Example folder created by SDK")
+            print(f"   Folder created with name: {folder.name}")
+            print(f"   Folder ID: {folder.id}")
+            
+            # Ingest document into folder
+            print("\n5. Ingesting a document into the folder...")
+            folder_doc = await folder.ingest_text(
+                content="This document is stored in a specific folder using the async client.",
+                filename="async_folder_example.txt",
+                metadata={"source": "async_sdk_example", "type": "folder_doc"}
+            )
+            print(f"   Document created with ID: {folder_doc.external_id}")
+            
+            # Create a user scope
+            print("\n6. Creating a user scope...")
+            user = db.signin("async_sdk_example_user")
+            print(f"   User scope created for: {user.end_user_id}")
+            
+            # Ingest document as user
+            print("\n7. Ingesting a document as this user...")
+            user_doc = await user.ingest_text(
+                content="This document is associated with a specific user using the async client.",
+                filename="async_user_example.txt",
+                metadata={"source": "async_sdk_example", "type": "user_doc"}
+            )
+            print(f"   Document created with ID: {user_doc.external_id}")
+            
+            # Wait for processing to complete
+            print("\n8. Waiting for documents to be processed...")
+            for _ in range(10):
+                status = await db.get_document_status(text_doc.external_id)
+                if status.get("status") == "completed":
+                    print(f"   Document {text_doc.external_id} is now processed")
+                    break
+                print(f"   Document status: {status.get('status')}. Waiting...")
+                await asyncio.sleep(3)
+            
+            # Search using retrieve_chunks
+            print("\n9. Retrieving relevant chunks...")
+            chunks = await db.retrieve_chunks(
+                query="What is this document about?",
+                filters={"source": "async_sdk_example"},
+                k=2
+            )
+            print(f"   Found {len(chunks)} chunks")
+            for i, chunk in enumerate(chunks):
+                print(f"   Chunk {i+1}: Score {chunk.score}")
+                print(f"      Content: {chunk.content[:50]}...")
+            
+            # Query using RAG
+            print("\n10. Generating a completion using RAG...")
+            completion = await db.query(
+                query="Summarize what these documents contain",
+                filters={"source": "async_sdk_example"},
+                k=3,
+                temperature=0.7
+            )
+            print(f"   Completion: {completion.completion}")
+            print(f"   Using {len(completion.sources)} sources")
+            for i, source in enumerate(completion.sources):
+                print(f"      Source {i+1}: Document {source.document_id}, Chunk {source.chunk_number}")
+            
+            # List documents
+            print("\n11. Listing documents...")
+            docs = await db.list_documents(filters={"source": "async_sdk_example"})
+            print(f"   Found {len(docs)} documents")
+            for i, doc in enumerate(docs):
+                print(f"   Document {i+1}: {doc.filename} (ID: {doc.external_id})")
+            
+            # Cleanup
+            print("\n12. Cleaning up test documents...")
+            # Delete the documents in reverse order (won't delete folder)
+            doc_ids = [user_doc.external_id, folder_doc.external_id, file_doc.external_id, text_doc.external_id]
+            for doc_id in doc_ids:
+                result = await db.delete_document(doc_id)
+                print(f"   Deleted document {doc_id}: {result.get('message', 'No message')}")
+            
+            print("\nAsync example completed successfully!")
+        
+        except Exception as e:
+            print(f"Error in async example: {e}")
+            raise
+
+
+if __name__ == "__main__":
+    # Parse command line arguments
+    parser = argparse.ArgumentParser(description="Morphik SDK example script")
+    parser.add_argument("--async", action="store_true", help="Run the async example")
+    args = parser.parse_args()
+    
+    if args.async:
+        # Run the async example
+        import asyncio
+        asyncio.run(run_async_example())
+    else:
+        # Run the sync example
+        run_sync_example()