sf-queue-sdk 0.5.0b4__tar.gz → 0.5.0b30__tar.gz

{sf_queue_sdk-0.5.0b4 → sf_queue_sdk-0.5.0b30}/.gitignore

@@ -65,3 +65,4 @@ scripts/docx/
 scripts/mp3-results/
 scripts/studyset/.env
 scripts/studyset/downloads/
+scripts/failed-files/

sf_queue_sdk-0.5.0b4/README.md → sf_queue_sdk-0.5.0b30/PKG-INFO

@@ -1,3 +1,11 @@
+Metadata-Version: 2.5
+Name: sf-queue-sdk
+Version: 0.5.0b30
+Summary: Python SDK for sf-queue - Redis-based queue system
+Requires-Python: >=3.11
+Requires-Dist: redis>=5.0.0
+Description-Content-Type: text/markdown
+
 # sf-queue-sdk
 
 Python SDK for sf-queue. Enqueues emails via Redis Streams with optional blocking confirmation from the Go consumer service.
@@ -546,6 +554,33 @@ result = client.document_upload.send(
 print("Enqueued:", result.message_id)
 ```
 
+### Idempotent verified uploads
+
+Pass `idempotency_key` when retrying a verified upload. Calls with the same
+environment, queue, and key create one Redis stream entry and return the same
+`message_id`.
+
+```python
+result = client.document_upload.send(
+    uploaded_file_id="file_abc123",
+    study_set_id="set_xyz",
+    user_id="user_123",
+    text_extract_s3_key="uploadedfiles/file_abc123/textContent",
+    pdf_s3_key="uploads/document.pdf",
+    file_type="application/pdf",
+    effective_file_type="application/pdf",
+    filename="Chapter 5 - Thermodynamics.pdf",
+    material_title="Thermodynamics",
+    url="https://storage.example.com/uploads/document.pdf",
+    generate_notes=True,
+    idempotency_key="verified-upload:file_abc123",
+)
+```
+
+The key is scoped to `document-upload` and the configured environment. The SDK
+stores a SHA-256 digest of the caller key in Redis and keeps the idempotency
+mapping for seven days.
+
 ### With optional fields
 
 ```python
@@ -606,6 +641,7 @@ result = client.document_upload.send(
 | `skip_material_creation` | `Optional[bool]` | No | Skip folder and material creation when the material already exists (e.g. live lectures) |
 | `existing_material_id` | `Optional[str]` | No | ID of an existing material to link to topics instead of creating new ones (used with `skip_material_creation`) |
 | `params` | `Optional[dict]` | No | Typed extensible flags for pipeline behavior (see below) |
+| `idempotency_key` | `Optional[str]` | No | Caller key for retry-safe enqueueing. Duplicate environment/queue/key sends return the original `message_id`. |
 
 #### `params` fields
 

sf_queue_sdk-0.5.0b4/PKG-INFO → sf_queue_sdk-0.5.0b30/README.md

@@ -1,11 +1,3 @@
-Metadata-Version: 2.5
-Name: sf-queue-sdk
-Version: 0.5.0b4
-Summary: Python SDK for sf-queue - Redis-based queue system
-Requires-Python: >=3.11
-Requires-Dist: redis>=5.0.0
-Description-Content-Type: text/markdown
-
 # sf-queue-sdk
 
 Python SDK for sf-queue. Enqueues emails via Redis Streams with optional blocking confirmation from the Go consumer service.
@@ -554,6 +546,33 @@ result = client.document_upload.send(
 print("Enqueued:", result.message_id)
 ```
 
+### Idempotent verified uploads
+
+Pass `idempotency_key` when retrying a verified upload. Calls with the same
+environment, queue, and key create one Redis stream entry and return the same
+`message_id`.
+
+```python
+result = client.document_upload.send(
+    uploaded_file_id="file_abc123",
+    study_set_id="set_xyz",
+    user_id="user_123",
+    text_extract_s3_key="uploadedfiles/file_abc123/textContent",
+    pdf_s3_key="uploads/document.pdf",
+    file_type="application/pdf",
+    effective_file_type="application/pdf",
+    filename="Chapter 5 - Thermodynamics.pdf",
+    material_title="Thermodynamics",
+    url="https://storage.example.com/uploads/document.pdf",
+    generate_notes=True,
+    idempotency_key="verified-upload:file_abc123",
+)
+```
+
+The key is scoped to `document-upload` and the configured environment. The SDK
+stores a SHA-256 digest of the caller key in Redis and keeps the idempotency
+mapping for seven days.
+
 ### With optional fields
 
 ```python
@@ -614,6 +633,7 @@ result = client.document_upload.send(
 | `skip_material_creation` | `Optional[bool]` | No | Skip folder and material creation when the material already exists (e.g. live lectures) |
 | `existing_material_id` | `Optional[str]` | No | ID of an existing material to link to topics instead of creating new ones (used with `skip_material_creation`) |
 | `params` | `Optional[dict]` | No | Typed extensible flags for pipeline behavior (see below) |
+| `idempotency_key` | `Optional[str]` | No | Caller key for retry-safe enqueueing. Duplicate environment/queue/key sends return the original `message_id`. |
 
 #### `params` fields
 

{sf_queue_sdk-0.5.0b4 → sf_queue_sdk-0.5.0b30}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sf-queue-sdk"
-version = "0.5.0b4"
+version = "0.5.0b30"
 description = "Python SDK for sf-queue - Redis-based queue system"
 readme = "README.md"
 requires-python = ">=3.11"

{sf_queue_sdk-0.5.0b4 → sf_queue_sdk-0.5.0b30}/queue_sdk/__init__.py

@@ -1,6 +1,6 @@
 """sf-queue Python SDK - Redis-based queue system."""
 
-__version__ = "0.5.0b4"
+__version__ = "0.5.0b30"
 
 from queue_sdk.client import QueueClient
 from queue_sdk.query import QueryClient

{sf_queue_sdk-0.5.0b4 → sf_queue_sdk-0.5.0b30}/queue_sdk/generated/queue/email/v1/email_pb2.py

@@ -2,7 +2,7 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # NO CHECKED-IN PROTOBUF GENCODE
 # source: queue/email/v1/email.proto
-# Protobuf Python Version: 7.35.1
+# Protobuf Python Version: 7.36.0
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
@@ -12,8 +12,8 @@ from google.protobuf.internal import builder as _builder
 _runtime_version.ValidateProtobufRuntimeVersion(
     _runtime_version.Domain.PUBLIC,
     7,
-    35,
-    1,
+    36,
+    0,
     '',
     'queue/email/v1/email.proto'
 )

{sf_queue_sdk-0.5.0b4 → sf_queue_sdk-0.5.0b30}/queue_sdk/generated/queue/prompt_ingest/v1/prompt_ingest_pb2.py

@@ -2,7 +2,7 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # NO CHECKED-IN PROTOBUF GENCODE
 # source: queue/prompt_ingest/v1/prompt_ingest.proto
-# Protobuf Python Version: 7.35.1
+# Protobuf Python Version: 7.36.0
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
@@ -12,8 +12,8 @@ from google.protobuf.internal import builder as _builder
 _runtime_version.ValidateProtobufRuntimeVersion(
     _runtime_version.Domain.PUBLIC,
     7,
-    35,
-    1,
+    36,
+    0,
     '',
     'queue/prompt_ingest/v1/prompt_ingest.proto'
 )

{sf_queue_sdk-0.5.0b4 → sf_queue_sdk-0.5.0b30}/queue_sdk/generated/queue/tool_assignment/v1/tool_assignment_pb2.py

@@ -2,7 +2,7 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # NO CHECKED-IN PROTOBUF GENCODE
 # source: queue/tool_assignment/v1/tool_assignment.proto
-# Protobuf Python Version: 7.35.1
+# Protobuf Python Version: 7.36.0
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
@@ -12,8 +12,8 @@ from google.protobuf.internal import builder as _builder
 _runtime_version.ValidateProtobufRuntimeVersion(
     _runtime_version.Domain.PUBLIC,
     7,
-    35,
-    1,
+    36,
+    0,
     '',
     'queue/tool_assignment/v1/tool_assignment.proto'
 )

{sf_queue_sdk-0.5.0b4 → sf_queue_sdk-0.5.0b30}/queue_sdk/generated/queue/topic_assignment/v1/topic_assignment_pb2.py

@@ -2,7 +2,7 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # NO CHECKED-IN PROTOBUF GENCODE
 # source: queue/topic_assignment/v1/topic_assignment.proto
-# Protobuf Python Version: 7.35.1
+# Protobuf Python Version: 7.36.0
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
@@ -12,8 +12,8 @@ from google.protobuf.internal import builder as _builder
 _runtime_version.ValidateProtobufRuntimeVersion(
     _runtime_version.Domain.PUBLIC,
     7,
-    35,
-    1,
+    36,
+    0,
     '',
     'queue/topic_assignment/v1/topic_assignment.proto'
 )

{sf_queue_sdk-0.5.0b4 → sf_queue_sdk-0.5.0b30}/queue_sdk/queues/base_queue.py

@@ -7,10 +7,32 @@ from typing import Any, Optional
 
 import redis as redis_lib
 
-from queue_sdk.redis_client import response_key, stream_name
+from queue_sdk.redis_client import idempotency_key, response_key, stream_name
 from queue_sdk.types import EmailResponse
 
 DEFAULT_TIMEOUT = 30  # seconds
+IDEMPOTENCY_TTL_SECONDS = 7 * 24 * 60 * 60
+IDEMPOTENT_ENQUEUE_SCRIPT = """
+local existing_request_id = redis.call("GET", KEYS[2])
+if existing_request_id then
+  return {existing_request_id, 0}
+end
+
+redis.call(
+  "XADD",
+  KEYS[1],
+  "*",
+  "requestId",
+  ARGV[1],
+  "payload",
+  ARGV[2],
+  "createdAt",
+  ARGV[3]
+)
+redis.call("SET", KEYS[2], ARGV[1], "EX", ARGV[4])
+
+return {ARGV[1], 1}
+"""
 
 
 class BaseQueue:
@@ -35,6 +57,35 @@ class BaseQueue:
         self._redis.xadd(self._stream, message)
         return request_id
 
+    def _enqueue_idempotently(
+        self, payload: dict[str, Any], caller_key: str
+    ) -> tuple[str, bool]:
+        """Add one stream entry for a caller key."""
+        if not caller_key.strip():
+            raise ValueError("idempotency_key must not be empty")
+
+        request_id = str(uuid.uuid4())
+        created_at = datetime.now(timezone.utc).isoformat()
+        result = self._redis.eval(
+            IDEMPOTENT_ENQUEUE_SCRIPT,
+            2,
+            self._stream,
+            idempotency_key(self._environment, self._queue_name, caller_key),
+            request_id,
+            json.dumps(payload),
+            created_at,
+            IDEMPOTENCY_TTL_SECONDS,
+        )
+        if (
+            not isinstance(result, list)
+            or len(result) != 2
+            or not isinstance(result[0], str)
+            or result[1] not in (0, 1)
+        ):
+            raise RuntimeError("Redis returned an invalid idempotent enqueue result")
+
+        return result[0], result[1] == 1
+
     def _enqueue_and_wait(
         self, payload: dict[str, Any], timeout: Optional[int] = None
     ) -> EmailResponse:

{sf_queue_sdk-0.5.0b4 → sf_queue_sdk-0.5.0b30}/queue_sdk/queues/document_upload_queue.py

@@ -53,6 +53,7 @@ class DocumentUploadQueue(BaseQueue):
         skip_material_creation: Optional[bool] = None,
         existing_material_id: Optional[str] = None,
         params: Optional[dict] = None,
+        idempotency_key: Optional[str] = None,
     ) -> SendResult:
         """Enqueue a document upload processing request (fire and forget)."""
         self._validate(uploaded_file_id, study_set_id, user_id, pdf_s3_key, filename)
@@ -91,8 +92,16 @@ class DocumentUploadQueue(BaseQueue):
             existing_material_id=existing_material_id,
             params=params,
         )
+        if idempotency_key is not None:
+            message_id, enqueued = self._enqueue_idempotently(payload, idempotency_key)
+            if enqueued:
+                self._set_initial_status(message_id)
+
+            return SendResult(message_id=message_id)
+
         message_id = self._enqueue(payload)
         self._set_initial_status(message_id)
+
         return SendResult(message_id=message_id)
 
     def get_status(self, request_id: str) -> DocumentUploadStatus:

{sf_queue_sdk-0.5.0b4 → sf_queue_sdk-0.5.0b30}/queue_sdk/redis_client.py

@@ -1,5 +1,6 @@
 """Redis connection factory."""
 
+import hashlib
 from urllib.parse import urlparse
 
 import redis
@@ -45,6 +46,15 @@ def stream_name(env: str, base: str) -> str:
     return f"{env}:{base}"
 
 
+def idempotency_key(env: str, queue: str, caller_key: str) -> str:
+    """Return a bounded key in the queue's Redis Cluster hash slot."""
+    digest = hashlib.sha256(caller_key.encode("utf-8")).hexdigest()
+    key = f"{{{queue}}}:idempotency:{digest}"
+    if not env:
+        return key
+    return f"{env}:{key}"
+
+
 def response_key(env: str, queue: str, request_id: str) -> str:
     """Return the response list key for a given queue and request ID."""
     key = f"{queue}:response:{request_id}"

{sf_queue_sdk-0.5.0b4 → sf_queue_sdk-0.5.0b30}/queue_sdk/types.py

@@ -178,6 +178,7 @@ class DocumentUploadParams:
     """Extensible flags for pipeline behavior."""
 
     live_lecture: Optional[bool] = None
+    source: Optional[str] = None
 
 
 @dataclass

sf_queue_sdk-0.5.0b30/tests/test_idempotency.py

@@ -0,0 +1,109 @@
+"""Focused idempotent enqueue tests."""
+
+import unittest
+
+from queue_sdk.queues.document_upload_queue import DocumentUploadQueue
+
+
+class FakeRedis:
+    """Redis test double for the idempotent enqueue script."""
+
+    def __init__(self) -> None:
+        self.idempotency: dict[str, str] = {}
+        self.entries: list[dict[str, str]] = []
+        self.status_writes = 0
+
+    def eval(
+        self,
+        _script: str,
+        key_count: int,
+        stream: str,
+        deduplication_key: str,
+        request_id: str,
+        payload: str,
+        created_at: str,
+        ttl_seconds: int,
+    ) -> list[str | int]:
+        if key_count != 2:
+            raise AssertionError("expected two Redis keys")
+        if "{document-upload}" not in stream:
+            raise AssertionError("stream is missing the queue hash tag")
+        if "{document-upload}:idempotency:" not in deduplication_key:
+            raise AssertionError("idempotency key is in a different hash slot")
+        if ttl_seconds != 7 * 24 * 60 * 60:
+            raise AssertionError("unexpected idempotency key TTL")
+
+        existing_request_id = self.idempotency.get(deduplication_key)
+        if existing_request_id is not None:
+            return [existing_request_id, 0]
+
+        self.entries.append(
+            {
+                "stream": stream,
+                "request_id": request_id,
+                "payload": payload,
+                "created_at": created_at,
+            }
+        )
+        self.idempotency[deduplication_key] = request_id
+
+        return [request_id, 1]
+
+    def hset(self, *_args: object, **_kwargs: object) -> None:
+        self.status_writes += 1
+
+    def expire(self, *_args: object, **_kwargs: object) -> None:
+        pass
+
+
+def send_upload(
+    queue: DocumentUploadQueue, idempotency_key: str
+):
+    """Send the minimum valid document upload payload."""
+    return queue.send(
+        uploaded_file_id="file-1",
+        study_set_id="set-1",
+        user_id="user-1",
+        text_extract_s3_key="text/key",
+        pdf_s3_key="pdf/key",
+        file_type="application/pdf",
+        effective_file_type="application/pdf",
+        filename="notes.pdf",
+        material_title="Notes",
+        url="https://example.com/notes.pdf",
+        generate_notes=True,
+        idempotency_key=idempotency_key,
+    )
+
+
+class IdempotentDocumentUploadTests(unittest.TestCase):
+    """Document upload idempotency contract."""
+
+    def setUp(self) -> None:
+        self.redis = FakeRedis()
+        self.queue = DocumentUploadQueue(self.redis, "staging")
+
+    def test_duplicate_key_enqueues_once_and_returns_one_request_id(self) -> None:
+        first = send_upload(self.queue, "verified-upload-1")
+        duplicate = send_upload(self.queue, "verified-upload-1")
+
+        self.assertEqual(duplicate.message_id, first.message_id)
+        self.assertEqual(len(self.redis.entries), 1)
+        self.assertEqual(self.redis.status_writes, 1)
+
+    def test_different_keys_create_different_stream_entries(self) -> None:
+        first = send_upload(self.queue, "verified-upload-1")
+        second = send_upload(self.queue, "verified-upload-2")
+
+        self.assertNotEqual(second.message_id, first.message_id)
+        self.assertEqual(len(self.redis.entries), 2)
+
+    def test_empty_key_is_rejected(self) -> None:
+        with self.assertRaisesRegex(ValueError, "idempotency_key must not be empty"):
+            send_upload(self.queue, "  ")
+
+        self.assertEqual(len(self.redis.entries), 0)
+
+
+if __name__ == "__main__":
+    unittest.main()