dayhoff-tools 1.1.42__py3-none-any.whl → 1.1.43__py3-none-any.whl

dayhoff_tools/deployment/swarm.py

@@ -128,23 +128,58 @@ def publish_cards(
     names: List[str],
     firestore_collection: str,
 ):
-    """Publish cards to Firebase. Expects a list of filenames (not full paths),
-    which will each be published as a new document in the collection."""
+    """Publish cards to Firebase using batch writes for optimal performance.
+
+    Expects a list of filenames (not full paths), which will each be published
+    as a new document in the collection. Uses Firestore batch writes to minimize
+    network round-trips and improve performance.
+
+    Args:
+        names: List of packet filenames to publish as cards
+        firestore_collection: Name of the Firestore collection to write to
+    """
+    if not names:
+        print("No cards to upload.")
+        return
 
     initialize_firebase()
-    collection = firestore.client().collection(firestore_collection)
+    db = firestore.client()
+    collection = db.collection(firestore_collection)
+
+    # Firestore batch limit is 500 operations
+    BATCH_SIZE = 500
+    total_cards = len(names)
+    cards_processed = 0
+
+    # Process names in batches of up to 500
+    for i in range(0, total_cards, BATCH_SIZE):
+        batch = db.batch()
+        batch_names = names[i : i + BATCH_SIZE]
+
+        # Add all operations for this batch
+        for name in batch_names:
+            doc_ref = collection.document()  # Auto-generate document ID
+            batch.set(
+                doc_ref,
+                {
+                    "status": "available",
+                    "packet_filename": name,
+                    "created": datetime.now(ZoneInfo("America/Los_Angeles")),
+                },
+            )
 
-    for name in names:
-        collection.document().set(
-            {
-                "status": "available",
-                "packet_filename": name,
-                "created": datetime.now(ZoneInfo("America/Los_Angeles")),
-            }
+        # Commit the entire batch atomically
+        batch.commit()
+        cards_processed += len(batch_names)
+
+        print(
+            f"Batch {i // BATCH_SIZE + 1}: Created {len(batch_names)} cards "
+            f"({cards_processed}/{total_cards} total)"
         )
-        print(f"Creating card {name}")
 
-    print(f"Uploaded {len(names)} cards.")
+    print(
+        f"Successfully uploaded {total_cards} cards in {(total_cards + BATCH_SIZE - 1) // BATCH_SIZE} batch(es)."
+    )
 
 
 @transactional

{dayhoff_tools-1.1.42.dist-info → dayhoff_tools-1.1.43.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dayhoff-tools
-Version: 1.1.42
+Version: 1.1.43
 Summary: Common tools for all the repos at Dayhoff Labs
 Author: Daniel Martin-Alarcon
 Author-email: dma@dayhofflabs.com

{dayhoff_tools-1.1.42.dist-info → dayhoff_tools-1.1.43.dist-info}/RECORD

@@ -12,7 +12,7 @@ dayhoff_tools/deployment/deploy_gcp.py,sha256=xgaOVsUDmP6wSEMYNkm1yRNcVskfdz80qJ
 dayhoff_tools/deployment/deploy_utils.py,sha256=StFwbqnr2_FWiKVg3xnJF4kagTHzndqqDkpaIOaAn_4,26027
 dayhoff_tools/deployment/job_runner.py,sha256=hljvFpH2Bw96uYyUup5Ths72PZRL_X27KxlYzBMgguo,5086
 dayhoff_tools/deployment/processors.py,sha256=LM0CQbr4XCb3AtLbrcuDQm4tYPXsoNqgVJ4WQYDjzJc,12406
-dayhoff_tools/deployment/swarm.py,sha256=MGcS2_x4RNFtnVjWlU_SwNfhICz8NlGYr9cYBK4ZKDA,21688
+dayhoff_tools/deployment/swarm.py,sha256=YJfvVOcAS8cYcIj2fiN4qwC2leh0I9w5A4px8ZWSF6g,22833
 dayhoff_tools/embedders.py,sha256=fRkyWjHo8OmbNUBY_FwrgfvyiLqpmrpI57UAb1Szn1Y,36609
 dayhoff_tools/fasta.py,sha256=_kA2Cpiy7JAGbBqLrjElkzbcUD_p-nO2d5Aj1LVmOvc,50509
 dayhoff_tools/file_ops.py,sha256=JlGowvr-CUJFidV-4g_JmhUTN9bsYuaxtqKmnKomm-Q,8506
@@ -26,7 +26,7 @@ dayhoff_tools/intake/uniprot.py,sha256=BZYJQF63OtPcBBnQ7_P9gulxzJtqyorgyuDiPeOJq
 dayhoff_tools/logs.py,sha256=DKdeP0k0kliRcilwvX0mUB2eipO5BdWUeHwh-VnsICs,838
 dayhoff_tools/sqlite.py,sha256=jV55ikF8VpTfeQqqlHSbY8OgfyfHj8zgHNpZjBLos_E,18672
 dayhoff_tools/warehouse.py,sha256=TqV8nex1AluNaL4JuXH5zuu9P7qmE89lSo6f_oViy6U,14965
-dayhoff_tools-1.1.42.dist-info/METADATA,sha256=tXnXIVVY3rpyNqyzl8cfzciGdW8ajHlJPpdliTYMD0M,2843
-dayhoff_tools-1.1.42.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-dayhoff_tools-1.1.42.dist-info/entry_points.txt,sha256=iAf4jteNqW3cJm6CO6czLxjW3vxYKsyGLZ8WGmxamSc,49
-dayhoff_tools-1.1.42.dist-info/RECORD,,
+dayhoff_tools-1.1.43.dist-info/METADATA,sha256=S3WFgeHSXhXJUg6E8nHcSBmbGktzd9B-2A7LIdX1c9k,2843
+dayhoff_tools-1.1.43.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+dayhoff_tools-1.1.43.dist-info/entry_points.txt,sha256=iAf4jteNqW3cJm6CO6czLxjW3vxYKsyGLZ8WGmxamSc,49
+dayhoff_tools-1.1.43.dist-info/RECORD,,