sf-queue-sdk 0.5.0b56__tar.gz → 0.5.0b79__tar.gz

{sf_queue_sdk-0.5.0b56 → sf_queue_sdk-0.5.0b79}/.gitignore

@@ -57,6 +57,7 @@ coverage/
 TODO-document-upload-queue.md
 
 # Local pipeline test output
+benchmark-results/
 scripts/.env
 scripts/local-pdfs/
 scripts/mp3/
@@ -66,3 +67,4 @@ scripts/mp3-results/
 scripts/studyset/.env
 scripts/studyset/downloads/
 scripts/failed-files/
+benchmark-results/

sf_queue_sdk-0.5.0b56/README.md → sf_queue_sdk-0.5.0b79/PKG-INFO

@@ -1,3 +1,11 @@
+Metadata-Version: 2.5
+Name: sf-queue-sdk
+Version: 0.5.0b79
+Summary: Python SDK for sf-queue - Redis-based queue system
+Requires-Python: >=3.11
+Requires-Dist: redis>=5.0.0
+Description-Content-Type: text/markdown
+
 # sf-queue-sdk
 
 Python SDK for sf-queue. Enqueues emails via Redis Streams with optional blocking confirmation from the Go consumer service.
@@ -598,9 +606,19 @@ result = client.document_upload.send(
     material_group_id="group_123",
     folder_id="folder_456",
     upload_batch_id="batch_789",
+    content_hash="sha256:verified-content-hash",
 )
 ```
 
+### Cancel a document
+
+Record a manual cancellation before or during processing. Cancellation is
+scoped to the configured environment and retained for seven days.
+
+```python
+client.document_upload.cancel_document("file_abc123")
+```
+
 ### Document Upload Fields
 
 | Field | Type | Required | Description |
@@ -629,6 +647,7 @@ result = client.document_upload.send(
 | `material_group_id` | `Optional[str]` | No | Material group for organization |
 | `folder_id` | `Optional[str]` | No | Folder for organization |
 | `upload_batch_id` | `Optional[str]` | No | Batch ID when multiple files uploaded together |
+| `content_hash` | `Optional[str]` | No | Stable source-content identity used to supersede older duplicate batches |
 | `youtube_title` | `Optional[str]` | No | Title override for YouTube video transcripts |
 | `skip_material_creation` | `Optional[bool]` | No | Skip folder and material creation when the material already exists (e.g. live lectures) |
 | `existing_material_id` | `Optional[str]` | No | ID of an existing material to link to topics instead of creating new ones (used with `skip_material_creation`) |
@@ -670,7 +689,7 @@ result = client.document_upload.send(
 
 # Poll for progress
 status = client.document_upload.get_status(result.message_id)
-print(f"Status: {status.status}")                     # "queued", "processing", "completed", or "failed"
+print(f"Status: {status.status}")                     # Includes "cancelled" and "superseded" terminal states
 print(f"Step: {status.step} ({status.step_number}/{status.total_steps})")
 print(f"Progress: {status.percent_complete}%")
 print(f"Detail: {status.detail}")                      # e.g. "Generated notes for topic 3/7"

sf_queue_sdk-0.5.0b56/PKG-INFO → sf_queue_sdk-0.5.0b79/README.md

@@ -1,11 +1,3 @@
-Metadata-Version: 2.5
-Name: sf-queue-sdk
-Version: 0.5.0b56
-Summary: Python SDK for sf-queue - Redis-based queue system
-Requires-Python: >=3.11
-Requires-Dist: redis>=5.0.0
-Description-Content-Type: text/markdown
-
 # sf-queue-sdk
 
 Python SDK for sf-queue. Enqueues emails via Redis Streams with optional blocking confirmation from the Go consumer service.
@@ -606,9 +598,19 @@ result = client.document_upload.send(
     material_group_id="group_123",
     folder_id="folder_456",
     upload_batch_id="batch_789",
+    content_hash="sha256:verified-content-hash",
 )
 ```
 
+### Cancel a document
+
+Record a manual cancellation before or during processing. Cancellation is
+scoped to the configured environment and retained for seven days.
+
+```python
+client.document_upload.cancel_document("file_abc123")
+```
+
 ### Document Upload Fields
 
 | Field | Type | Required | Description |
@@ -637,6 +639,7 @@ result = client.document_upload.send(
 | `material_group_id` | `Optional[str]` | No | Material group for organization |
 | `folder_id` | `Optional[str]` | No | Folder for organization |
 | `upload_batch_id` | `Optional[str]` | No | Batch ID when multiple files uploaded together |
+| `content_hash` | `Optional[str]` | No | Stable source-content identity used to supersede older duplicate batches |
 | `youtube_title` | `Optional[str]` | No | Title override for YouTube video transcripts |
 | `skip_material_creation` | `Optional[bool]` | No | Skip folder and material creation when the material already exists (e.g. live lectures) |
 | `existing_material_id` | `Optional[str]` | No | ID of an existing material to link to topics instead of creating new ones (used with `skip_material_creation`) |
@@ -678,7 +681,7 @@ result = client.document_upload.send(
 
 # Poll for progress
 status = client.document_upload.get_status(result.message_id)
-print(f"Status: {status.status}")                     # "queued", "processing", "completed", or "failed"
+print(f"Status: {status.status}")                     # Includes "cancelled" and "superseded" terminal states
 print(f"Step: {status.step} ({status.step_number}/{status.total_steps})")
 print(f"Progress: {status.percent_complete}%")
 print(f"Detail: {status.detail}")                      # e.g. "Generated notes for topic 3/7"

{sf_queue_sdk-0.5.0b56 → sf_queue_sdk-0.5.0b79}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sf-queue-sdk"
-version = "0.5.0b56"
+version = "0.5.0b79"
 description = "Python SDK for sf-queue - Redis-based queue system"
 readme = "README.md"
 requires-python = ">=3.11"

{sf_queue_sdk-0.5.0b56 → sf_queue_sdk-0.5.0b79}/queue_sdk/__init__.py

@@ -1,6 +1,6 @@
 """sf-queue Python SDK - Redis-based queue system."""
 
-__version__ = "0.5.0b56"
+__version__ = "0.5.0b79"
 
 from queue_sdk.client import QueueClient
 from queue_sdk.query import QueryClient

{sf_queue_sdk-0.5.0b56 → sf_queue_sdk-0.5.0b79}/queue_sdk/generated/queue/email/v1/email_pb2.py

@@ -2,7 +2,7 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # NO CHECKED-IN PROTOBUF GENCODE
 # source: queue/email/v1/email.proto
-# Protobuf Python Version: 7.36.0
+# Protobuf Python Version: 7.35.1
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
@@ -12,8 +12,8 @@ from google.protobuf.internal import builder as _builder
 _runtime_version.ValidateProtobufRuntimeVersion(
     _runtime_version.Domain.PUBLIC,
     7,
-    36,
-    0,
+    35,
+    1,
     '',
     'queue/email/v1/email.proto'
 )

{sf_queue_sdk-0.5.0b56 → sf_queue_sdk-0.5.0b79}/queue_sdk/generated/queue/prompt_ingest/v1/prompt_ingest_pb2.py

@@ -2,7 +2,7 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # NO CHECKED-IN PROTOBUF GENCODE
 # source: queue/prompt_ingest/v1/prompt_ingest.proto
-# Protobuf Python Version: 7.36.0
+# Protobuf Python Version: 7.35.1
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
@@ -12,8 +12,8 @@ from google.protobuf.internal import builder as _builder
 _runtime_version.ValidateProtobufRuntimeVersion(
     _runtime_version.Domain.PUBLIC,
     7,
-    36,
-    0,
+    35,
+    1,
     '',
     'queue/prompt_ingest/v1/prompt_ingest.proto'
 )

{sf_queue_sdk-0.5.0b56 → sf_queue_sdk-0.5.0b79}/queue_sdk/generated/queue/tool_assignment/v1/tool_assignment_pb2.py

@@ -2,7 +2,7 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # NO CHECKED-IN PROTOBUF GENCODE
 # source: queue/tool_assignment/v1/tool_assignment.proto
-# Protobuf Python Version: 7.36.0
+# Protobuf Python Version: 7.35.1
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
@@ -12,8 +12,8 @@ from google.protobuf.internal import builder as _builder
 _runtime_version.ValidateProtobufRuntimeVersion(
     _runtime_version.Domain.PUBLIC,
     7,
-    36,
-    0,
+    35,
+    1,
     '',
     'queue/tool_assignment/v1/tool_assignment.proto'
 )

{sf_queue_sdk-0.5.0b56 → sf_queue_sdk-0.5.0b79}/queue_sdk/generated/queue/topic_assignment/v1/topic_assignment_pb2.py

@@ -2,7 +2,7 @@
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # NO CHECKED-IN PROTOBUF GENCODE
 # source: queue/topic_assignment/v1/topic_assignment.proto
-# Protobuf Python Version: 7.36.0
+# Protobuf Python Version: 7.35.1
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool
@@ -12,8 +12,8 @@ from google.protobuf.internal import builder as _builder
 _runtime_version.ValidateProtobufRuntimeVersion(
     _runtime_version.Domain.PUBLIC,
     7,
-    36,
-    0,
+    35,
+    1,
     '',
     'queue/topic_assignment/v1/topic_assignment.proto'
 )

{sf_queue_sdk-0.5.0b56 → sf_queue_sdk-0.5.0b79}/queue_sdk/queues/document_upload_queue.py

@@ -6,10 +6,11 @@ from typing import Any, Optional
 import redis as redis_lib
 
 from queue_sdk.queues.base_queue import BaseQueue
-from queue_sdk.redis_client import status_key
+from queue_sdk.redis_client import document_control_key, status_key
 from queue_sdk.types import DocumentUploadStatus, SendResult
 
-STATUS_TTL_SECONDS = 3600
+STATUS_TTL_SECONDS = 7200
+TERMINAL_CONTROL_TTL_SECONDS = 7 * 24 * 60 * 60
 
 
 class DocumentUploadQueue(BaseQueue):
@@ -45,6 +46,7 @@ class DocumentUploadQueue(BaseQueue):
         folder_id: Optional[str] = None,
         upload_batch_id: Optional[str] = None,
         batch_size: Optional[int] = None,
+        content_hash: Optional[str] = None,
         youtube_title: Optional[str] = None,
         extracted_content: Optional[str] = None,
         callback_url: Optional[str] = None,
@@ -83,6 +85,7 @@ class DocumentUploadQueue(BaseQueue):
             folder_id=folder_id,
             upload_batch_id=upload_batch_id,
             batch_size=batch_size,
+            content_hash=content_hash,
             youtube_title=youtube_title,
             extracted_content=extracted_content,
             callback_url=callback_url,
@@ -133,11 +136,29 @@ class DocumentUploadQueue(BaseQueue):
             error=data.get("error", ""),
         )
 
+    def cancel_document(self, uploaded_file_id: str) -> None:
+        """Record a manual cancellation before or during processing."""
+        if not uploaded_file_id.strip():
+            raise ValueError("uploaded_file_id must not be empty")
+
+        key = document_control_key(self._environment, uploaded_file_id)
+        pipe = self._redis.pipeline(transaction=False)
+        pipe.hset(
+            key,
+            mapping={
+                "manualCancel": "1",
+                "cancelReason": "manual",
+                "cancelledAt": datetime.now(timezone.utc).isoformat(),
+            },
+        )
+        pipe.expire(key, TERMINAL_CONTROL_TTL_SECONDS)
+        pipe.execute()
+
     def _set_initial_status(self, request_id: str) -> None:
         """Write the initial queued status hash after enqueuing."""
         key = status_key(self._environment, self._queue_name, request_id)
         now = datetime.now(timezone.utc).isoformat()
-        self._redis.hset(key, mapping={
+        initial_status = {
             "status": "queued",
             "step": "",
             "stepNumber": "0",
@@ -148,8 +169,12 @@ class DocumentUploadQueue(BaseQueue):
             "startedAt": "",
             "updatedAt": now,
             "error": "",
-        })
-        self._redis.expire(key, STATUS_TTL_SECONDS)
+        }
+        pipe = self._redis.pipeline(transaction=False)
+        for field, value in initial_status.items():
+            pipe.hsetnx(key, field, value)
+        pipe.expire(key, STATUS_TTL_SECONDS)
+        pipe.execute()
 
     @staticmethod
     def _build_payload(
@@ -178,6 +203,7 @@ class DocumentUploadQueue(BaseQueue):
         folder_id: Optional[str],
         upload_batch_id: Optional[str],
         batch_size: Optional[int],
+        content_hash: Optional[str],
         youtube_title: Optional[str],
         extracted_content: Optional[str],
         callback_url: Optional[str],
@@ -225,6 +251,8 @@ class DocumentUploadQueue(BaseQueue):
             payload["uploadBatchId"] = upload_batch_id
         if batch_size is not None:
             payload["batchSize"] = batch_size
+        if content_hash is not None:
+            payload["contentHash"] = content_hash
         if youtube_title is not None:
             payload["youtubeTitle"] = youtube_title
         if extracted_content is not None:

{sf_queue_sdk-0.5.0b56 → sf_queue_sdk-0.5.0b79}/queue_sdk/redis_client.py

@@ -55,6 +55,16 @@ def idempotency_key(env: str, queue: str, caller_key: str) -> str:
     return f"{env}:{key}"
 
 
+def document_control_key(env: str, uploaded_file_id: str) -> str:
+    """Return the cancellation-control key for an upload."""
+    digest = hashlib.sha256(uploaded_file_id.encode("utf-8")).hexdigest()
+    key = f"{{document-upload}}:ctl:document:{digest}"
+    if not env:
+        return key
+
+    return f"{env}:{key}"
+
+
 def response_key(env: str, queue: str, request_id: str) -> str:
     """Return the response list key for a given queue and request ID."""
     key = f"{queue}:response:{request_id}"

{sf_queue_sdk-0.5.0b56 → sf_queue_sdk-0.5.0b79}/queue_sdk/types.py

@@ -209,6 +209,7 @@ class DocumentUploadData:
     material_group_id: Optional[str] = None
     folder_id: Optional[str] = None
     upload_batch_id: Optional[str] = None
+    content_hash: Optional[str] = None
     youtube_title: Optional[str] = None
     extracted_content: Optional[str] = None
     callback_url: Optional[str] = None

{sf_queue_sdk-0.5.0b56 → sf_queue_sdk-0.5.0b79}/tests/test_idempotency.py

@@ -1,10 +1,43 @@
 """Focused idempotent enqueue tests."""
 
+import json
 import unittest
 
 from queue_sdk.queues.document_upload_queue import DocumentUploadQueue
 
 
+class FakePipeline:
+    def __init__(self, redis: "FakeRedis") -> None:
+        self.redis = redis
+        self.operations: list[tuple[str, tuple[object, ...]]] = []
+
+    def hsetnx(self, *args: object) -> "FakePipeline":
+        self.operations.append(("hsetnx", args))
+        return self
+
+    def hset(self, *args: object, **kwargs: object) -> "FakePipeline":
+        self.operations.append(("hset", (*args, kwargs)))
+        return self
+
+    def expire(self, *args: object) -> "FakePipeline":
+        self.operations.append(("expire", args))
+        return self
+
+    def execute(self) -> None:
+        hsetnx_count = sum(
+            operation == "hsetnx"
+            for operation, _ in self.operations
+        )
+        if hsetnx_count:
+            if hsetnx_count != 10:
+                raise AssertionError("unexpected initial status field count")
+            if self.operations[-1][1][1] != 7200:
+                raise AssertionError("unexpected status TTL")
+            self.redis.status_writes += 1
+        else:
+            self.redis.cancel_writes.append(self.operations)
+
+
 class FakeRedis:
     """Redis test double for the idempotent enqueue script."""
 
@@ -12,6 +45,7 @@ class FakeRedis:
         self.idempotency: dict[str, str] = {}
         self.entries: list[dict[str, str]] = []
         self.status_writes = 0
+        self.cancel_writes: list[list[tuple[str, tuple[object, ...]]]] = []
 
     def eval(
         self,
@@ -49,11 +83,10 @@ class FakeRedis:
 
         return [request_id, 1]
 
-    def hset(self, *_args: object, **_kwargs: object) -> None:
-        self.status_writes += 1
-
-    def expire(self, *_args: object, **_kwargs: object) -> None:
-        pass
+    def pipeline(self, *, transaction: bool) -> FakePipeline:
+        if transaction:
+            raise AssertionError("status initialization should not use a transaction")
+        return FakePipeline(self)
 
 
 def send_upload(
@@ -72,6 +105,7 @@ def send_upload(
         material_title="Notes",
         url="https://example.com/notes.pdf",
         generate_notes=True,
+        content_hash="sha256:content-1",
         idempotency_key=idempotency_key,
     )
 
@@ -104,6 +138,45 @@ class IdempotentDocumentUploadTests(unittest.TestCase):
 
         self.assertEqual(len(self.redis.entries), 0)
 
+    def test_content_hash_is_serialized(self) -> None:
+        send_upload(self.queue, "with-hash")
+
+        payload = json.loads(self.redis.entries[0]["payload"])
+
+        self.assertEqual(payload["contentHash"], "sha256:content-1")
+
+    def test_content_hash_remains_optional(self) -> None:
+        result = self.queue.send(
+            uploaded_file_id="file-1",
+            study_set_id="set-1",
+            user_id="user-1",
+            text_extract_s3_key="text/key",
+            pdf_s3_key="pdf/key",
+            file_type="application/pdf",
+            effective_file_type="application/pdf",
+            filename="notes.pdf",
+            material_title="Notes",
+            url="https://example.com/notes.pdf",
+            generate_notes=True,
+            idempotency_key="without-hash",
+        )
+
+        self.assertTrue(result.message_id)
+        payload = json.loads(self.redis.entries[0]["payload"])
+        self.assertNotIn("contentHash", payload)
+
+    def test_manual_cancellation_writes_scoped_tombstone(self) -> None:
+        self.queue.cancel_document("file-1")
+
+        self.assertEqual(len(self.redis.cancel_writes), 1)
+        operations = self.redis.cancel_writes[0]
+        self.assertRegex(
+            str(operations[0][1][0]),
+            r"^staging:\{document-upload\}:ctl:document:",
+        )
+        self.assertEqual(operations[0][1][-1]["mapping"]["manualCancel"], "1")
+        self.assertEqual(operations[-1][1][1], 7 * 24 * 60 * 60)
+
 
 if __name__ == "__main__":
     unittest.main()