aio-sf 0.1.0b9__tar.gz → 0.1.0b11__tar.gz

{aio_sf-0.1.0b9 → aio_sf-0.1.0b11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aio-sf
-Version: 0.1.0b9
+Version: 0.1.0b11
 Summary: Async Salesforce library for Python
 Project-URL: Homepage, https://github.com/callawaycloud/aio-salesforce
 Project-URL: Repository, https://github.com/callawaycloud/aio-salesforce
@@ -166,15 +166,19 @@ async with SalesforceClient(auth_strategy=auth) as sf:
 
 **Advanced - With Retries, Concurrency Scaling, and Progress:**
 ```python
-from aio_sf.api.collections import ProgressInfo
+from aio_sf.api.collections import ResultInfo
 
-async def on_progress(info: ProgressInfo):
+async def on_result(info: ResultInfo):
+    # Called after each batch completes with successes and errors split
     print(
-        f"Attempt {info['current_attempt']}: "
-        f"{info['records_succeeded']} succeeded, "
-        f"{info['records_failed']} failed, "
+        f"Batch: {len(info['successes'])} succeeded, {len(info['errors'])} failed | "
+        f"Attempt {info['current_attempt']}, "
+        f"Overall: {info['records_succeeded']} OK, {info['records_failed']} failed, "
         f"{info['records_pending']} pending"
     )
+    # Inspect errors (includes both API errors and HTTP failures)
+    for error in info['errors']:
+        print(f"  Error: {error['errors']}")
 
 async with SalesforceClient(auth_strategy=auth) as sf:
     results = await sf.collections.insert(
@@ -183,7 +187,7 @@ async with SalesforceClient(auth_strategy=auth) as sf:
         batch_size=[200, 100, 25],         # Shrink batch size on retry
         max_concurrent_batches=[5, 3, 1],  # Reduce concurrency on retry
         max_attempts=5,                    # Retry up to 5 times
-        on_batch_complete=on_progress,     # Progress callback
+        on_result=on_result,               # Callback with results
     )
 ```
 

{aio_sf-0.1.0b9 → aio_sf-0.1.0b11}/README.md

@@ -103,15 +103,19 @@ async with SalesforceClient(auth_strategy=auth) as sf:
 
 **Advanced - With Retries, Concurrency Scaling, and Progress:**
 ```python
-from aio_sf.api.collections import ProgressInfo
+from aio_sf.api.collections import ResultInfo
 
-async def on_progress(info: ProgressInfo):
+async def on_result(info: ResultInfo):
+    # Called after each batch completes with successes and errors split
     print(
-        f"Attempt {info['current_attempt']}: "
-        f"{info['records_succeeded']} succeeded, "
-        f"{info['records_failed']} failed, "
+        f"Batch: {len(info['successes'])} succeeded, {len(info['errors'])} failed | "
+        f"Attempt {info['current_attempt']}, "
+        f"Overall: {info['records_succeeded']} OK, {info['records_failed']} failed, "
         f"{info['records_pending']} pending"
     )
+    # Inspect errors (includes both API errors and HTTP failures)
+    for error in info['errors']:
+        print(f"  Error: {error['errors']}")
 
 async with SalesforceClient(auth_strategy=auth) as sf:
     results = await sf.collections.insert(
@@ -120,7 +124,7 @@ async with SalesforceClient(auth_strategy=auth) as sf:
         batch_size=[200, 100, 25],         # Shrink batch size on retry
         max_concurrent_batches=[5, 3, 1],  # Reduce concurrency on retry
         max_attempts=5,                    # Retry up to 5 times
-        on_batch_complete=on_progress,     # Progress callback
+        on_result=on_result,               # Callback with results
     )
 ```
 

{aio_sf-0.1.0b9 → aio_sf-0.1.0b11}/src/aio_sf/api/collections/__init__.py

@@ -1,7 +1,7 @@
 """Salesforce Collections API module."""
 
 from .client import CollectionsAPI
-from .batch import ProgressInfo, ProgressCallback
+from .batch import ResultInfo, ResultCallback
 from .retry import ShouldRetryCallback, default_should_retry
 from .types import (
     CollectionError,
@@ -14,8 +14,8 @@ from .types import (
 
 __all__ = [
     "CollectionsAPI",
-    "ProgressInfo",
-    "ProgressCallback",
+    "ResultInfo",
+    "ResultCallback",
     "ShouldRetryCallback",
     "default_should_retry",
     "CollectionError",

{aio_sf-0.1.0b9 → aio_sf-0.1.0b11}/src/aio_sf/api/collections/batch.py

@@ -17,21 +17,25 @@ from .types import CollectionResult
 logger = logging.getLogger(__name__)
 
 
-class ProgressInfo(TypedDict):
-    """Progress information for batch operations."""
+class ResultInfo(TypedDict):
+    """Result information provided after each batch completes."""
 
+    successes: List[CollectionResult]  # Successful results from this batch
+    errors: List[
+        CollectionResult
+    ]  # Failed results from this batch (API and HTTP errors)
     total_records: int  # Total records being processed
     records_completed: int  # Records finished (succeeded or failed permanently)
-    records_succeeded: int  # Records that succeeded
-    records_failed: int  # Records that failed permanently (exhausted retries)
+    records_succeeded: int  # Records that succeeded so far
+    records_failed: int  # Records that failed permanently so far
     records_pending: int  # Records still being retried
     current_attempt: int  # Current retry attempt number (1-indexed)
     current_batch_size: int  # Batch size for current attempt
     current_concurrency: int  # Concurrency level for current attempt
 
 
-# Type alias for progress callback
-ProgressCallback = Callable[[ProgressInfo], Awaitable[None]]
+# Type alias for result callback
+ResultCallback = Callable[[ResultInfo], Awaitable[None]]
 
 
 def split_into_batches(
@@ -65,8 +69,9 @@ async def process_batches_concurrently(
     operation_func,
     max_concurrent_batches: int,
     total_records: int,
-    on_batch_complete: Optional[ProgressCallback] = None,
+    on_result: Optional[ResultCallback] = None,
     progress_state: Optional[Dict[str, int]] = None,
+    final_results: Optional[List] = None,
     *args,
     **kwargs,
 ) -> List[Any]:
@@ -80,8 +85,8 @@ async def process_batches_concurrently(
     :param operation_func: Function to call for each batch
     :param max_concurrent_batches: Maximum number of concurrent batch operations
     :param total_records: Total number of records being processed
-    :param on_batch_complete: Optional callback invoked after each batch completes
-    :param progress_state: Dict with progress state (updated by caller)
+    :param on_result: Optional callback invoked after each batch completes with results
+    :param progress_state: Dict with progress state (to include in callback)
     :param args: Additional positional arguments for operation_func
     :param kwargs: Additional keyword arguments for operation_func
     :returns: List of results from all batches in the same order as input
@@ -91,7 +96,7 @@ async def process_batches_concurrently(
         raise ValueError("max_concurrent_batches must be greater than 0")
 
     semaphore = asyncio.Semaphore(max_concurrent_batches)
-    callback_lock = asyncio.Lock() if on_batch_complete else None
+    callback_lock = asyncio.Lock() if on_result else None
 
     async def process_batch_with_semaphore(batch_index: int, batch):
         async with semaphore:
@@ -104,10 +109,37 @@ async def process_batches_concurrently(
                 )
                 result = [e for _ in range(len(batch))]
 
-            # Invoke progress callback if provided
-            if on_batch_complete and callback_lock and progress_state:
+            # Invoke callback if provided, with results and progress state
+            if on_result and callback_lock and progress_state:
                 async with callback_lock:
-                    progress_info: ProgressInfo = {
+                    # Split results into successes and errors
+                    successes: List[CollectionResult] = []
+                    errors: List[CollectionResult] = []
+
+                    for item in result:
+                        # Convert exceptions to CollectionResult format
+                        if isinstance(item, Exception):
+                            error_result = convert_exception_to_result(item)
+                            errors.append(error_result)
+                        elif item.get("success", False):
+                            successes.append(item)
+                        else:
+                            errors.append(item)
+
+                    # Update progress_state cumulatively for per-batch reporting
+                    # We only count successes as completed here (errors remain pending until final)
+                    progress_state["records_succeeded"] += len(successes)
+                    progress_state["records_completed"] += len(successes)
+                    progress_state["records_pending"] = progress_state[
+                        "total_records"
+                    ] - (
+                        progress_state["records_succeeded"]
+                        + progress_state["records_failed"]
+                    )
+
+                    result_info: ResultInfo = {
+                        "successes": successes,
+                        "errors": errors,
                         "total_records": progress_state["total_records"],
                         "records_completed": progress_state["records_completed"],
                         "records_succeeded": progress_state["records_succeeded"],
@@ -117,7 +149,7 @@ async def process_batches_concurrently(
                         "current_batch_size": progress_state["current_batch_size"],
                         "current_concurrency": progress_state["current_concurrency"],
                     }
-                    await on_batch_complete(progress_info)
+                    await on_result(result_info)
 
             return result
 
@@ -141,7 +173,7 @@ async def process_with_retries(
     max_attempts: int,
     should_retry_callback: Optional[ShouldRetryCallback],
     max_concurrent_batches: Union[int, List[int]],
-    on_batch_complete: Optional[ProgressCallback],
+    on_result: Optional[ResultCallback],
     max_limit: int,
     *args,
     **kwargs,
@@ -155,7 +187,7 @@ async def process_with_retries(
     :param max_attempts: Maximum number of attempts per record
     :param should_retry_callback: Optional callback to determine if record should be retried
     :param max_concurrent_batches: Maximum concurrent batches (int or list of ints per attempt)
-    :param on_batch_complete: Progress callback
+    :param on_result: Callback invoked after each batch completes with results and progress
     :param max_limit: Maximum batch size limit for the operation
     :param args: Additional args for operation_func
     :param kwargs: Additional kwargs for operation_func
@@ -203,14 +235,15 @@ async def process_with_retries(
         records_to_process = [r.record for r in current_records]
         batches = split_into_batches(records_to_process, current_batch_size, max_limit)
 
-        # Process batches with current concurrency level (no callback here)
+        # Process batches - callback will be invoked for each batch with results
         batch_results = await process_batches_concurrently(
             batches,
             operation_func,
             current_concurrency,
             len(records_to_process),
-            None,  # Don't invoke callback during batch processing
-            None,
+            on_result,
+            progress_state,
+            final_results,
             *args,
             **kwargs,
         )
@@ -224,11 +257,7 @@ async def process_with_retries(
             final_results,
         )
 
-        # Update progress state based on results
-        # Count completed records (those not being retried)
-        records_completed_this_round = len(current_records) - len(records_to_retry)
-
-        # Count successes and failures in final_results so far
+        # Update progress state based on results for next iteration
         records_succeeded = sum(
             1 for r in final_results if r is not None and r.get("success", False)
         )
@@ -241,20 +270,6 @@ async def process_with_retries(
         progress_state["records_failed"] = records_failed
         progress_state["records_pending"] = len(records_to_retry)
 
-        # Invoke progress callback after we know the results
-        if on_batch_complete:
-            progress_info: ProgressInfo = {
-                "total_records": progress_state["total_records"],
-                "records_completed": progress_state["records_completed"],
-                "records_succeeded": progress_state["records_succeeded"],
-                "records_failed": progress_state["records_failed"],
-                "records_pending": progress_state["records_pending"],
-                "current_attempt": progress_state["current_attempt"],
-                "current_batch_size": progress_state["current_batch_size"],
-                "current_concurrency": progress_state["current_concurrency"],
-            }
-            await on_batch_complete(progress_info)
-
         if records_to_retry:
             logger.info(
                 f"Retrying {len(records_to_retry)} failed records "

{aio_sf-0.1.0b9 → aio_sf-0.1.0b11}/src/aio_sf/api/collections/client.py

@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from ..client import SalesforceClient
 
-from .batch import ProgressCallback, process_with_retries
+from .batch import ResultCallback, process_with_retries
 from .records import (
     detect_record_type_and_sobject,
     prepare_records,
@@ -92,7 +92,7 @@ class CollectionsAPI:
         batch_size: Union[int, List[int]] = 200,
         max_concurrent_batches: Union[int, List[int]] = 5,
         api_version: Optional[str] = None,
-        on_batch_complete: Optional[ProgressCallback] = None,
+        on_result: Optional[ResultCallback] = None,
         max_attempts: int = 1,
         should_retry: Optional[ShouldRetryCallback] = None,
     ) -> CollectionInsertResponse:
@@ -109,7 +109,7 @@ class CollectionsAPI:
         :param batch_size: Batch size (int for same size, or list of ints per attempt). Max 200.
         :param max_concurrent_batches: Maximum number of concurrent batch operations
         :param api_version: API version to use
-        :param on_batch_complete: Optional async callback invoked after each batch completes
+        :param on_result: Optional async callback invoked after each batch completes with results
         :param max_attempts: Maximum number of attempts per record (default: 1, no retries)
         :param should_retry: Optional callback to determine if a failed record should be retried
         :returns: List of results for each record, in same order as input
@@ -139,7 +139,7 @@ class CollectionsAPI:
             max_attempts,
             should_retry,
             max_concurrent_batches,
-            on_batch_complete,
+            on_result,
             self.MAX_RECORDS_INSERT,
             actual_sobject_type,
             all_or_none,
@@ -200,7 +200,7 @@ class CollectionsAPI:
         batch_size: Union[int, List[int]] = 200,
         max_concurrent_batches: Union[int, List[int]] = 5,
         api_version: Optional[str] = None,
-        on_batch_complete: Optional[ProgressCallback] = None,
+        on_result: Optional[ResultCallback] = None,
         max_attempts: int = 1,
         should_retry: Optional[ShouldRetryCallback] = None,
     ) -> CollectionUpdateResponse:
@@ -217,7 +217,7 @@ class CollectionsAPI:
         :param batch_size: Batch size (int for same size, or list of ints per attempt). Max 200.
         :param max_concurrent_batches: Maximum number of concurrent batch operations
         :param api_version: API version to use
-        :param on_batch_complete: Optional async callback invoked after each batch completes
+        :param on_result: Optional async callback invoked after each batch completes with results
         :param max_attempts: Maximum number of attempts per record (default: 1, no retries)
         :param should_retry: Optional callback to determine if a failed record should be retried
         :returns: List of results for each record, in same order as input
@@ -247,7 +247,7 @@ class CollectionsAPI:
             max_attempts,
             should_retry,
             max_concurrent_batches,
-            on_batch_complete,
+            on_result,
             self.MAX_RECORDS_UPDATE,
             actual_sobject_type,
             all_or_none,
@@ -313,7 +313,7 @@ class CollectionsAPI:
         batch_size: Union[int, List[int]] = 200,
         max_concurrent_batches: Union[int, List[int]] = 5,
         api_version: Optional[str] = None,
-        on_batch_complete: Optional[ProgressCallback] = None,
+        on_result: Optional[ResultCallback] = None,
         max_attempts: int = 1,
         should_retry: Optional[ShouldRetryCallback] = None,
     ) -> CollectionUpsertResponse:
@@ -331,7 +331,7 @@ class CollectionsAPI:
         :param batch_size: Batch size (int for same size, or list of ints per attempt). Max 200.
         :param max_concurrent_batches: Maximum number of concurrent batch operations
         :param api_version: API version to use
-        :param on_batch_complete: Optional async callback invoked after each batch completes
+        :param on_result: Optional async callback invoked after each batch completes with results
         :param max_attempts: Maximum number of attempts per record (default: 1, no retries)
         :param should_retry: Optional callback to determine if a failed record should be retried
         :returns: List of results for each record, in same order as input
@@ -361,7 +361,7 @@ class CollectionsAPI:
             max_attempts,
             should_retry,
             max_concurrent_batches,
-            on_batch_complete,
+            on_result,
             self.MAX_RECORDS_UPSERT,
             external_id_field,
             actual_sobject_type,
@@ -411,7 +411,7 @@ class CollectionsAPI:
         batch_size: Union[int, List[int]] = 200,
         max_concurrent_batches: Union[int, List[int]] = 5,
         api_version: Optional[str] = None,
-        on_batch_complete: Optional[ProgressCallback] = None,
+        on_result: Optional[ResultCallback] = None,
         max_attempts: int = 1,
         should_retry: Optional[ShouldRetryCallback] = None,
     ) -> CollectionDeleteResponse:
@@ -427,7 +427,7 @@ class CollectionsAPI:
         :param batch_size: Batch size (int for same size, or list of ints per attempt). Max 200.
         :param max_concurrent_batches: Maximum number of concurrent batch operations
         :param api_version: API version to use
-        :param on_batch_complete: Optional async callback invoked after each batch completes
+        :param on_result: Optional async callback invoked after each batch completes with results
         :param max_attempts: Maximum number of attempts per record (default: 1, no retries)
         :param should_retry: Optional callback to determine if a failed record should be retried
         :returns: List of results for each record, in same order as input
@@ -453,7 +453,7 @@ class CollectionsAPI:
             max_attempts,
             should_retry,
             max_concurrent_batches,
-            on_batch_complete,
+            on_result,
             self.MAX_RECORDS_DELETE,
             all_or_none,
             api_version,

{aio_sf-0.1.0b9 → aio_sf-0.1.0b11}/tests/test_retry_and_batch.py

@@ -2,7 +2,7 @@
 
 import pytest
 import asyncio
-from aio_sf.api.collections import CollectionsAPI, ProgressInfo
+from aio_sf.api.collections import CollectionsAPI, ResultInfo
 
 
 class MockClient:
@@ -463,31 +463,38 @@ class TestProgressTracking:
             ]
         )
 
-        progress_calls = []
+        result_calls = []
 
-        async def progress_callback(progress: ProgressInfo):
-            progress_calls.append(dict(progress))
+        async def result_callback(result: ResultInfo):
+            result_calls.append(dict(result))
 
         results = await collections_api.insert(
             records,
             sobject_type="Account",
             batch_size=200,
-            on_batch_complete=progress_callback,
+            on_result=result_callback,
         )
 
         assert len(results) == 300
-        # With the new design, callback is invoked once per attempt (after all batches)
-        # Since all records succeed on first attempt, we get 1 callback
-        assert len(progress_calls) == 1
-
-        # Verify progress data - after attempt completes, all succeeded
-        assert progress_calls[0]["total_records"] == 300
-        assert progress_calls[0]["current_batch_size"] == 200
-        assert progress_calls[0]["current_concurrency"] == 5  # default
-        assert progress_calls[0]["current_attempt"] == 1
-        assert progress_calls[0]["records_succeeded"] == 300
-        assert progress_calls[0]["records_failed"] == 0
-        assert progress_calls[0]["records_pending"] == 0
+        # With the new design, callback is invoked once per batch
+        # 300 records with batch_size=200 means 2 batches
+        assert len(result_calls) == 2
+
+        # Both callbacks split successes and errors
+        assert len(result_calls[0]["successes"]) == 200
+        assert len(result_calls[0]["errors"]) == 0
+        assert len(result_calls[1]["successes"]) == 100
+        assert len(result_calls[1]["errors"]) == 0
+
+        # Verify successes are properly typed CollectionResults
+        assert all(r.get("success") for r in result_calls[0]["successes"])
+        assert all(r.get("id") is not None for r in result_calls[0]["successes"])
+
+        # Context information is provided
+        assert result_calls[0]["total_records"] == 300
+        assert result_calls[0]["current_batch_size"] == 200
+        assert result_calls[0]["current_concurrency"] == 5
+        assert result_calls[0]["current_attempt"] == 1
 
     @pytest.mark.asyncio
     async def test_progress_with_retries(self, client):
@@ -517,36 +524,39 @@ class TestProgressTracking:
             ]
         )
 
-        progress_calls = []
+        result_calls = []
 
-        async def progress_callback(progress: ProgressInfo):
-            progress_calls.append(dict(progress))
+        async def result_callback(result: ResultInfo):
+            result_calls.append(dict(result))
 
         results = await collections_api.insert(
             records,
             sobject_type="Account",
             batch_size=200,
             max_attempts=2,
-            on_batch_complete=progress_callback,
+            on_result=result_callback,
         )
 
         assert len(results) == 10
         assert all(r["success"] for r in results)
 
-        # Should have 2 progress callbacks (initial + retry)
-        assert len(progress_calls) == 2
-
-        # First attempt: all failed, all pending retry
-        assert progress_calls[0]["current_attempt"] == 1
-        assert progress_calls[0]["records_succeeded"] == 0
-        assert progress_calls[0]["records_failed"] == 0
-        assert progress_calls[0]["records_pending"] == 10
-
-        # Second attempt: all succeeded
-        assert progress_calls[1]["current_attempt"] == 2
-        assert progress_calls[1]["records_succeeded"] == 10
-        assert progress_calls[1]["records_failed"] == 0
-        assert progress_calls[1]["records_pending"] == 0
+        # Should have 2 callbacks (initial attempt + retry attempt)
+        assert len(result_calls) == 2
+
+        # First attempt: all failed - errors array contains them
+        assert result_calls[0]["current_attempt"] == 1
+        assert len(result_calls[0]["successes"]) == 0
+        assert len(result_calls[0]["errors"]) == 10
+        # Can inspect error codes in errors array
+        for error in result_calls[0]["errors"]:
+            assert not error.get("success")
+            assert error["errors"][0]["statusCode"] == "UNABLE_TO_LOCK_ROW"
+
+        # Second attempt: all succeeded - successes array contains them
+        assert result_calls[1]["current_attempt"] == 2
+        assert len(result_calls[1]["successes"]) == 10
+        assert len(result_calls[1]["errors"]) == 0
+        assert all(r.get("success") for r in result_calls[1]["successes"])
 
 
 class TestConcurrencyScaling: