sf-queue-sdk 0.4.0b139__tar.gz → 0.5.0__tar.gz

{sf_queue_sdk-0.4.0b139 → sf_queue_sdk-0.5.0}/.gitignore

@@ -59,3 +59,10 @@ TODO-document-upload-queue.md
 # Local pipeline test output
 scripts/.env
 scripts/local-pdfs/
+scripts/mp3/
+scripts/ppt/
+scripts/docx/
+scripts/mp3-results/
+scripts/studyset/.env
+scripts/studyset/downloads/
+scripts/failed-files/

sf_queue_sdk-0.4.0b139/README.md → sf_queue_sdk-0.5.0/PKG-INFO

@@ -1,3 +1,11 @@
+Metadata-Version: 2.5
+Name: sf-queue-sdk
+Version: 0.5.0
+Summary: Python SDK for sf-queue - Redis-based queue system
+Requires-Python: >=3.11
+Requires-Dist: redis>=5.0.0
+Description-Content-Type: text/markdown
+
 # sf-queue-sdk
 
 Python SDK for sf-queue. Enqueues emails via Redis Streams with optional blocking confirmation from the Go consumer service.
@@ -546,6 +554,33 @@ result = client.document_upload.send(
 print("Enqueued:", result.message_id)
 ```
 
+### Idempotent verified uploads
+
+Pass `idempotency_key` when retrying a verified upload. Calls with the same
+environment, queue, and key create one Redis stream entry and return the same
+`message_id`.
+
+```python
+result = client.document_upload.send(
+    uploaded_file_id="file_abc123",
+    study_set_id="set_xyz",
+    user_id="user_123",
+    text_extract_s3_key="uploadedfiles/file_abc123/textContent",
+    pdf_s3_key="uploads/document.pdf",
+    file_type="application/pdf",
+    effective_file_type="application/pdf",
+    filename="Chapter 5 - Thermodynamics.pdf",
+    material_title="Thermodynamics",
+    url="https://storage.example.com/uploads/document.pdf",
+    generate_notes=True,
+    idempotency_key="verified-upload:file_abc123",
+)
+```
+
+The key is scoped to `document-upload` and the configured environment. The SDK
+stores a SHA-256 digest of the caller key in Redis and keeps the idempotency
+mapping for seven days.
+
 ### With optional fields
 
 ```python
@@ -591,7 +626,7 @@ result = client.document_upload.send(
 | `generate_notes` | `bool` | Yes | Whether to generate study notes |
 | `generate_study_path` | `bool` | Yes | Whether to generate a study path |
 | `splitting` | `str` | No | Splitting strategy for sources with at least 40,000 tokens (default: `"auto"`). Smaller sources always persist as one material after per-topic note generation. |
-| `language` | `str` | No | Target language code (default: `"en"`) |
+| `language` | `str` | No | Output language as an app ISO code, e.g. `"de"` — not a locale like `"de-DE"`. Note the non-standard `cz` (Czech) and `gr` (Greek). (default: `"en"`) |
 | `use_markdown_ocr` | `bool` | No | Use Markdown OCR for image interpretation (default: `False`) |
 | `flashcard_amount` | `int` | No | Number of flashcards to generate (default: `12`) |
 | `new_url` | `Optional[str]` | No | Alternative URL if file was re-uploaded |
@@ -605,6 +640,14 @@ result = client.document_upload.send(
 | `youtube_title` | `Optional[str]` | No | Title override for YouTube video transcripts |
 | `skip_material_creation` | `Optional[bool]` | No | Skip folder and material creation when the material already exists (e.g. live lectures) |
 | `existing_material_id` | `Optional[str]` | No | ID of an existing material to link to topics instead of creating new ones (used with `skip_material_creation`) |
+| `params` | `Optional[dict]` | No | Typed extensible flags for pipeline behavior (see below) |
+| `idempotency_key` | `Optional[str]` | No | Caller key for retry-safe enqueueing. Duplicate environment/queue/key sends return the original `message_id`. |
+
+#### `params` fields
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `liveLecture` | `bool` | When true, forces `generateNotes=false` and `skipMaterialCreation=true` in the pipeline. Clustering still runs to produce sections/topics for study path generation. Use with `existing_material_id` and `extracted_content`. |
 
 ### Response Type
 
@@ -787,7 +830,7 @@ result = client.document_persist.send(
 | `study_set_id` | `str` | Yes | Study set to persist materials into |
 | `user_id` | `str` | Yes | User who uploaded the document |
 | `stash_key` | `str` | Yes | GCS key of the clustering stash JSON |
-| `language` | `str` | No | Target language code (default: `"en"`) |
+| `language` | `str` | No | Output language as an app ISO code, e.g. `"de"` — not a locale like `"de-DE"`. Note the non-standard `cz` (Czech) and `gr` (Greek). (default: `"en"`) |
 | `callback_url` | `Optional[str]` | No | URL to POST results to after persistence |
 | `callback_api_key` | `Optional[str]` | No | Auth key for the callback |
 | `approved_topic_ids` | `Optional[list[str]]` | No | Filter to only these topic IDs from the stash (omit to persist all) |

sf_queue_sdk-0.4.0b139/PKG-INFO → sf_queue_sdk-0.5.0/README.md

@@ -1,11 +1,3 @@
-Metadata-Version: 2.4
-Name: sf-queue-sdk
-Version: 0.4.0b139
-Summary: Python SDK for sf-queue - Redis-based queue system
-Requires-Python: >=3.11
-Requires-Dist: redis>=5.0.0
-Description-Content-Type: text/markdown
-
 # sf-queue-sdk
 
 Python SDK for sf-queue. Enqueues emails via Redis Streams with optional blocking confirmation from the Go consumer service.
@@ -554,6 +546,33 @@ result = client.document_upload.send(
 print("Enqueued:", result.message_id)
 ```
 
+### Idempotent verified uploads
+
+Pass `idempotency_key` when retrying a verified upload. Calls with the same
+environment, queue, and key create one Redis stream entry and return the same
+`message_id`.
+
+```python
+result = client.document_upload.send(
+    uploaded_file_id="file_abc123",
+    study_set_id="set_xyz",
+    user_id="user_123",
+    text_extract_s3_key="uploadedfiles/file_abc123/textContent",
+    pdf_s3_key="uploads/document.pdf",
+    file_type="application/pdf",
+    effective_file_type="application/pdf",
+    filename="Chapter 5 - Thermodynamics.pdf",
+    material_title="Thermodynamics",
+    url="https://storage.example.com/uploads/document.pdf",
+    generate_notes=True,
+    idempotency_key="verified-upload:file_abc123",
+)
+```
+
+The key is scoped to `document-upload` and the configured environment. The SDK
+stores a SHA-256 digest of the caller key in Redis and keeps the idempotency
+mapping for seven days.
+
 ### With optional fields
 
 ```python
@@ -599,7 +618,7 @@ result = client.document_upload.send(
 | `generate_notes` | `bool` | Yes | Whether to generate study notes |
 | `generate_study_path` | `bool` | Yes | Whether to generate a study path |
 | `splitting` | `str` | No | Splitting strategy for sources with at least 40,000 tokens (default: `"auto"`). Smaller sources always persist as one material after per-topic note generation. |
-| `language` | `str` | No | Target language code (default: `"en"`) |
+| `language` | `str` | No | Output language as an app ISO code, e.g. `"de"` — not a locale like `"de-DE"`. Note the non-standard `cz` (Czech) and `gr` (Greek). (default: `"en"`) |
 | `use_markdown_ocr` | `bool` | No | Use Markdown OCR for image interpretation (default: `False`) |
 | `flashcard_amount` | `int` | No | Number of flashcards to generate (default: `12`) |
 | `new_url` | `Optional[str]` | No | Alternative URL if file was re-uploaded |
@@ -613,6 +632,14 @@ result = client.document_upload.send(
 | `youtube_title` | `Optional[str]` | No | Title override for YouTube video transcripts |
 | `skip_material_creation` | `Optional[bool]` | No | Skip folder and material creation when the material already exists (e.g. live lectures) |
 | `existing_material_id` | `Optional[str]` | No | ID of an existing material to link to topics instead of creating new ones (used with `skip_material_creation`) |
+| `params` | `Optional[dict]` | No | Typed extensible flags for pipeline behavior (see below) |
+| `idempotency_key` | `Optional[str]` | No | Caller key for retry-safe enqueueing. Duplicate environment/queue/key sends return the original `message_id`. |
+
+#### `params` fields
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `liveLecture` | `bool` | When true, forces `generateNotes=false` and `skipMaterialCreation=true` in the pipeline. Clustering still runs to produce sections/topics for study path generation. Use with `existing_material_id` and `extracted_content`. |
 
 ### Response Type
 
@@ -795,7 +822,7 @@ result = client.document_persist.send(
 | `study_set_id` | `str` | Yes | Study set to persist materials into |
 | `user_id` | `str` | Yes | User who uploaded the document |
 | `stash_key` | `str` | Yes | GCS key of the clustering stash JSON |
-| `language` | `str` | No | Target language code (default: `"en"`) |
+| `language` | `str` | No | Output language as an app ISO code, e.g. `"de"` — not a locale like `"de-DE"`. Note the non-standard `cz` (Czech) and `gr` (Greek). (default: `"en"`) |
 | `callback_url` | `Optional[str]` | No | URL to POST results to after persistence |
 | `callback_api_key` | `Optional[str]` | No | Auth key for the callback |
 | `approved_topic_ids` | `Optional[list[str]]` | No | Filter to only these topic IDs from the stash (omit to persist all) |

{sf_queue_sdk-0.4.0b139 → sf_queue_sdk-0.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sf-queue-sdk"
-version = "0.4.0b139"
+version = "0.5.0"
 description = "Python SDK for sf-queue - Redis-based queue system"
 readme = "README.md"
 requires-python = ">=3.11"

{sf_queue_sdk-0.4.0b139 → sf_queue_sdk-0.5.0}/queue_sdk/__init__.py

@@ -1,6 +1,6 @@
 """sf-queue Python SDK - Redis-based queue system."""
 
-__version__ = "0.4.0b139"
+__version__ = "0.5.0"
 
 from queue_sdk.client import QueueClient
 from queue_sdk.query import QueryClient

{sf_queue_sdk-0.4.0b139 → sf_queue_sdk-0.5.0}/queue_sdk/queues/base_queue.py

@@ -7,10 +7,32 @@ from typing import Any, Optional
 
 import redis as redis_lib
 
-from queue_sdk.redis_client import response_key, stream_name
+from queue_sdk.redis_client import idempotency_key, response_key, stream_name
 from queue_sdk.types import EmailResponse
 
 DEFAULT_TIMEOUT = 30  # seconds
+IDEMPOTENCY_TTL_SECONDS = 7 * 24 * 60 * 60
+IDEMPOTENT_ENQUEUE_SCRIPT = """
+local existing_request_id = redis.call("GET", KEYS[2])
+if existing_request_id then
+  return {existing_request_id, 0}
+end
+
+redis.call(
+  "XADD",
+  KEYS[1],
+  "*",
+  "requestId",
+  ARGV[1],
+  "payload",
+  ARGV[2],
+  "createdAt",
+  ARGV[3]
+)
+redis.call("SET", KEYS[2], ARGV[1], "EX", ARGV[4])
+
+return {ARGV[1], 1}
+"""
 
 
 class BaseQueue:
@@ -35,6 +57,35 @@ class BaseQueue:
         self._redis.xadd(self._stream, message)
         return request_id
 
+    def _enqueue_idempotently(
+        self, payload: dict[str, Any], caller_key: str
+    ) -> tuple[str, bool]:
+        """Add one stream entry for a caller key."""
+        if not caller_key.strip():
+            raise ValueError("idempotency_key must not be empty")
+
+        request_id = str(uuid.uuid4())
+        created_at = datetime.now(timezone.utc).isoformat()
+        result = self._redis.eval(
+            IDEMPOTENT_ENQUEUE_SCRIPT,
+            2,
+            self._stream,
+            idempotency_key(self._environment, self._queue_name, caller_key),
+            request_id,
+            json.dumps(payload),
+            created_at,
+            IDEMPOTENCY_TTL_SECONDS,
+        )
+        if (
+            not isinstance(result, list)
+            or len(result) != 2
+            or not isinstance(result[0], str)
+            or result[1] not in (0, 1)
+        ):
+            raise RuntimeError("Redis returned an invalid idempotent enqueue result")
+
+        return result[0], result[1] == 1
+
     def _enqueue_and_wait(
         self, payload: dict[str, Any], timeout: Optional[int] = None
     ) -> EmailResponse:

{sf_queue_sdk-0.4.0b139 → sf_queue_sdk-0.5.0}/queue_sdk/queues/document_upload_queue.py

@@ -52,6 +52,8 @@ class DocumentUploadQueue(BaseQueue):
         study_plan_hook: Optional[bool] = None,
         skip_material_creation: Optional[bool] = None,
         existing_material_id: Optional[str] = None,
+        params: Optional[dict] = None,
+        idempotency_key: Optional[str] = None,
     ) -> SendResult:
         """Enqueue a document upload processing request (fire and forget)."""
         self._validate(uploaded_file_id, study_set_id, user_id, pdf_s3_key, filename)
@@ -88,9 +90,18 @@ class DocumentUploadQueue(BaseQueue):
             study_plan_hook=study_plan_hook,
             skip_material_creation=skip_material_creation,
             existing_material_id=existing_material_id,
+            params=params,
         )
+        if idempotency_key is not None:
+            message_id, enqueued = self._enqueue_idempotently(payload, idempotency_key)
+            if enqueued:
+                self._set_initial_status(message_id)
+
+            return SendResult(message_id=message_id)
+
         message_id = self._enqueue(payload)
         self._set_initial_status(message_id)
+
         return SendResult(message_id=message_id)
 
     def get_status(self, request_id: str) -> DocumentUploadStatus:
@@ -174,6 +185,7 @@ class DocumentUploadQueue(BaseQueue):
         study_plan_hook: Optional[bool],
         skip_material_creation: Optional[bool],
         existing_material_id: Optional[str],
+        params: Optional[dict],
     ) -> dict[str, Any]:
         """Build the document upload payload (snake_case -> camelCase)."""
         payload: dict[str, Any] = {
@@ -227,6 +239,8 @@ class DocumentUploadQueue(BaseQueue):
             payload["skipMaterialCreation"] = skip_material_creation
         if existing_material_id is not None:
             payload["existingMaterialId"] = existing_material_id
+        if params is not None:
+            payload["params"] = params
 
         return payload
 

{sf_queue_sdk-0.4.0b139 → sf_queue_sdk-0.5.0}/queue_sdk/redis_client.py

@@ -1,5 +1,6 @@
 """Redis connection factory."""
 
+import hashlib
 from urllib.parse import urlparse
 
 import redis
@@ -45,6 +46,15 @@ def stream_name(env: str, base: str) -> str:
     return f"{env}:{base}"
 
 
+def idempotency_key(env: str, queue: str, caller_key: str) -> str:
+    """Return a bounded key in the queue's Redis Cluster hash slot."""
+    digest = hashlib.sha256(caller_key.encode("utf-8")).hexdigest()
+    key = f"{{{queue}}}:idempotency:{digest}"
+    if not env:
+        return key
+    return f"{env}:{key}"
+
+
 def response_key(env: str, queue: str, request_id: str) -> str:
     """Return the response list key for a given queue and request ID."""
     key = f"{queue}:response:{request_id}"

{sf_queue_sdk-0.4.0b139 → sf_queue_sdk-0.5.0}/queue_sdk/types.py

@@ -173,6 +173,14 @@ class BatchRecommendationResponse:
     processed_at: Optional[str] = None
 
 
+@dataclass
+class DocumentUploadParams:
+    """Extensible flags for pipeline behavior."""
+
+    live_lecture: Optional[bool] = None
+    source: Optional[str] = None
+
+
 @dataclass
 class DocumentUploadData:
     """Data for a document upload processing request."""
@@ -206,6 +214,7 @@ class DocumentUploadData:
     callback_url: Optional[str] = None
     callback_api_key: Optional[str] = None
     study_plan_hook: Optional[bool] = None
+    params: Optional[DocumentUploadParams] = None
 
 
 @dataclass

sf_queue_sdk-0.5.0/tests/test_idempotency.py

@@ -0,0 +1,109 @@
+"""Focused idempotent enqueue tests."""
+
+import unittest
+
+from queue_sdk.queues.document_upload_queue import DocumentUploadQueue
+
+
+class FakeRedis:
+    """Redis test double for the idempotent enqueue script."""
+
+    def __init__(self) -> None:
+        self.idempotency: dict[str, str] = {}
+        self.entries: list[dict[str, str]] = []
+        self.status_writes = 0
+
+    def eval(
+        self,
+        _script: str,
+        key_count: int,
+        stream: str,
+        deduplication_key: str,
+        request_id: str,
+        payload: str,
+        created_at: str,
+        ttl_seconds: int,
+    ) -> list[str | int]:
+        if key_count != 2:
+            raise AssertionError("expected two Redis keys")
+        if "{document-upload}" not in stream:
+            raise AssertionError("stream is missing the queue hash tag")
+        if "{document-upload}:idempotency:" not in deduplication_key:
+            raise AssertionError("idempotency key is in a different hash slot")
+        if ttl_seconds != 7 * 24 * 60 * 60:
+            raise AssertionError("unexpected idempotency key TTL")
+
+        existing_request_id = self.idempotency.get(deduplication_key)
+        if existing_request_id is not None:
+            return [existing_request_id, 0]
+
+        self.entries.append(
+            {
+                "stream": stream,
+                "request_id": request_id,
+                "payload": payload,
+                "created_at": created_at,
+            }
+        )
+        self.idempotency[deduplication_key] = request_id
+
+        return [request_id, 1]
+
+    def hset(self, *_args: object, **_kwargs: object) -> None:
+        self.status_writes += 1
+
+    def expire(self, *_args: object, **_kwargs: object) -> None:
+        pass
+
+
+def send_upload(
+    queue: DocumentUploadQueue, idempotency_key: str
+):
+    """Send the minimum valid document upload payload."""
+    return queue.send(
+        uploaded_file_id="file-1",
+        study_set_id="set-1",
+        user_id="user-1",
+        text_extract_s3_key="text/key",
+        pdf_s3_key="pdf/key",
+        file_type="application/pdf",
+        effective_file_type="application/pdf",
+        filename="notes.pdf",
+        material_title="Notes",
+        url="https://example.com/notes.pdf",
+        generate_notes=True,
+        idempotency_key=idempotency_key,
+    )
+
+
+class IdempotentDocumentUploadTests(unittest.TestCase):
+    """Document upload idempotency contract."""
+
+    def setUp(self) -> None:
+        self.redis = FakeRedis()
+        self.queue = DocumentUploadQueue(self.redis, "staging")
+
+    def test_duplicate_key_enqueues_once_and_returns_one_request_id(self) -> None:
+        first = send_upload(self.queue, "verified-upload-1")
+        duplicate = send_upload(self.queue, "verified-upload-1")
+
+        self.assertEqual(duplicate.message_id, first.message_id)
+        self.assertEqual(len(self.redis.entries), 1)
+        self.assertEqual(self.redis.status_writes, 1)
+
+    def test_different_keys_create_different_stream_entries(self) -> None:
+        first = send_upload(self.queue, "verified-upload-1")
+        second = send_upload(self.queue, "verified-upload-2")
+
+        self.assertNotEqual(second.message_id, first.message_id)
+        self.assertEqual(len(self.redis.entries), 2)
+
+    def test_empty_key_is_rejected(self) -> None:
+        with self.assertRaisesRegex(ValueError, "idempotency_key must not be empty"):
+            send_upload(self.queue, "  ")
+
+        self.assertEqual(len(self.redis.entries), 0)
+
+
+if __name__ == "__main__":
+    unittest.main()