TransferQueue 0.1.1.dev0__py3-none-any.whl → 0.1.1.dev2__py3-none-any.whl

transfer_queue/controller.py

@@ -47,8 +47,8 @@ from transfer_queue.utils.zmq_utils import (
 logger = logging.getLogger(__name__)
 logger.setLevel(os.getenv("TQ_LOGGING_LEVEL", logging.WARNING))
 
-TQ_CONTROLLER_GET_METADATA_TIMEOUT = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_TIMEOUT", 300))
-TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL", 1))
+TQ_CONTROLLER_GET_METADATA_TIMEOUT = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_TIMEOUT", 1))
+TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL", 0.2))
 TQ_CONTROLLER_CONNECTION_CHECK_INTERVAL = int(os.environ.get("TQ_CONTROLLER_CONNECTION_CHECK_INTERVAL", 2))
 
 TQ_INIT_SAMPLE_NUM = int(os.environ.get("TQ_INIT_SAMPLE_NUM", 10))  # Initial number of samples
@@ -753,6 +753,17 @@ class TransferQueueController:
                 ready_for_consume_indexes = self.scan_data_status(partition_id, data_fields, task_name, batch_size)
 
                 if len(ready_for_consume_indexes) < batch_size:
+                    if time.time() - start_time > TQ_CONTROLLER_GET_METADATA_TIMEOUT:
+                        raise TimeoutError(
+                            f"Timeout while waiting for sufficient data. "
+                            f"Required: {batch_size}, Available: {len(ready_for_consume_indexes)}"
+                        )
+                    logger.warning(
+                        f"Insufficient complete groups available. Required: {batch_size}, "
+                        f"Available: {len(ready_for_consume_indexes)}. Retrying in "
+                        f"{TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL}s..."
+                    )
+                    time.sleep(TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL)
                     continue
 
                 # Try sampling - if it returns empty lists, retry

transfer_queue/storage/managers/simple_backend_manager.py

@@ -34,6 +34,9 @@ from transfer_queue.utils.zmq_utils import ZMQMessage, ZMQRequestType, ZMQServer
 logger = logging.getLogger(__name__)
 logger.setLevel(os.getenv("TQ_LOGGING_LEVEL", logging.WARNING))
 
+TQ_SIMPLE_STORAGE_MANAGER_RECV_TIMEOUT = int(os.environ.get("TQ_SIMPLE_STORAGE_MANAGER_RECV_TIMEOUT", 200))  # seconds
+TQ_SIMPLE_STORAGE_MANAGER_SEND_TIMEOUT = int(os.environ.get("TQ_SIMPLE_STORAGE_MANAGER_SEND_TIMEOUT", 200))  # seconds
+
 
 @TransferQueueStorageManagerFactory.register("AsyncSimpleStorageManager")
 class AsyncSimpleStorageManager(TransferQueueStorageManager):
@@ -132,8 +135,8 @@ class AsyncSimpleStorageManager(TransferQueueStorageManager):
                 try:
                     sock.connect(address)
                     # Timeouts to avoid indefinite await on recv/send
-                    sock.setsockopt(zmq.RCVTIMEO, 10_000)  # 10s
-                    sock.setsockopt(zmq.SNDTIMEO, 10_000)  # 10s
+                    sock.setsockopt(zmq.RCVTIMEO, TQ_SIMPLE_STORAGE_MANAGER_RECV_TIMEOUT * 1000)
+                    sock.setsockopt(zmq.SNDTIMEO, TQ_SIMPLE_STORAGE_MANAGER_SEND_TIMEOUT * 1000)
                     logger.info(
                         f"[{self.storage_manager_id}]: Connected to StorageUnit {server_info.id} at {address} "
                         f"with identity {identity.decode()}"

transfer_queue/version/version

@@ -1 +1 @@
-0.1.1.dev
+0.1.1.dev2

{transferqueue-0.1.1.dev0.dist-info → transferqueue-0.1.1.dev2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: TransferQueue
-Version: 0.1.1.dev0
+Version: 0.1.1.dev2
 Summary: TransferQueue: An Asynchronous Streaming Data Management Module
 Author-email: The TransferQueue Team <hanzy19@tsinghua.org.cn>
 License: Apache-2.0

{transferqueue-0.1.1.dev0.dist-info → transferqueue-0.1.1.dev2.dist-info}/RECORD

@@ -12,7 +12,7 @@ tests/test_simple_storage_unit.py,sha256=Sczhw2bdCfTfa7RwnpW-aKCV1of8mO6z3l3q2PD
 tests/test_storage_client_factory.py,sha256=U0gS_l4bc_bP7K_uPhy8UlJ186HyTrA10TSJNdf5SBM,1708
 transfer_queue/__init__.py,sha256=68c0sBfqHPqTa7OdzO4sAZB52XvwtjpwLqP9BWAh4fA,1535
 transfer_queue/client.py,sha256=zDlH1beWwRbjz0a7S8QH9IOJ1wp5yEQ36XYwFwJTXOY,24949
-transfer_queue/controller.py,sha256=gFS3OMM7giSqwypA7qmMFZ_AxG22LIpCQIblbqC_Gm8,48988
+transfer_queue/controller.py,sha256=uc_MQAlG_QmJ8szxc5yPLaZvCeP2CRULnAADtcf41-8,49702
 transfer_queue/metadata.py,sha256=LHQhM7vixw5QcA05nZcqWbxSh7mhJ2ETJOo_1gFb3eI,17645
 transfer_queue/sampler/__init__.py,sha256=1oauDy2Dwb5GXhKi7tl5DWAHv8i4t2MQK1S4U36Sy4g,788
 transfer_queue/sampler/base.py,sha256=wFti4dNJb3YArYpGzxA_YDfyUTdTG8wVz6HclPDyZPw,3299
@@ -27,15 +27,15 @@ transfer_queue/storage/clients/yuanrong_client.py,sha256=rWOPQPLgHLav7cFdGCr8CeV
 transfer_queue/storage/managers/__init__.py,sha256=y5x5OzZwK_YormdHdzc-smnJNew3niuqU2g_kkaiXIk,876
 transfer_queue/storage/managers/base.py,sha256=ntlo6sLWIbTiAOUxgJTUyjLl5m4HBuAUUNCYZxq9BFM,20352
 transfer_queue/storage/managers/factory.py,sha256=58kp2mCKz1K8Ea7RWMsWxdDhN3y4ZhgE-G647AKq7-I,1752
-transfer_queue/storage/managers/simple_backend_manager.py,sha256=fCj-0BnS3RbzPn03KEcxmUWbxZVWK11cfxpj8tSf-yY,27381
+transfer_queue/storage/managers/simple_backend_manager.py,sha256=zn0RrRng2zo0kLHl3eKQ9Ksf9O1W9DCEzpuZes8uSjM,27684
 transfer_queue/storage/managers/yuanrong_manager.py,sha256=NjHC3LBW0fQwm30Oq_qEKoCQEq8oWO0D-AobcpQNPNg,777
 transfer_queue/utils/__init__.py,sha256=vki-5RVaRBKxVc6Q7XPQox3VNPio2DvJYvRz0SZtu-w,586
 transfer_queue/utils/serial_utils.py,sha256=9ZgsytTp-441YKtIRFqyH5NhNifSqRKa2h0FI44ltcc,10200
 transfer_queue/utils/utils.py,sha256=Pno4h3WjX_eT7q4xiVV6Jkhquc39Fp-Ycsg1cv0qNKQ,4544
 transfer_queue/utils/zmq_utils.py,sha256=jCg2pQfvy_IYdGyZq4nvL4CAwjJc7Li0Trp3T3GDBMg,5118
-transfer_queue/version/version,sha256=6GX4KkA_FGXCFr4Z-Mw9dUky3PbKBVWPbSJCCVmVPlw,9
-transferqueue-0.1.1.dev0.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-transferqueue-0.1.1.dev0.dist-info/METADATA,sha256=3YY8wlygPXiXf5Ct4x6Is4ezv6-ZR1bwnL3tyTxZf7E,19531
-transferqueue-0.1.1.dev0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-transferqueue-0.1.1.dev0.dist-info/top_level.txt,sha256=BiBclu7jWJ0AZ35vUr3hN9_cg8JL9EiH_hjFxquMxtw,33
-transferqueue-0.1.1.dev0.dist-info/RECORD,,
+transfer_queue/version/version,sha256=GLEQDboAHBDJ_23hOEatkTWzCRpxAQCOMSDn78NKSjw,11
+transferqueue-0.1.1.dev2.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+transferqueue-0.1.1.dev2.dist-info/METADATA,sha256=TfiqGrPVI9vQ2QH5FUi1lFtHIidhVPkv5auZM16TOJM,19531
+transferqueue-0.1.1.dev2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+transferqueue-0.1.1.dev2.dist-info/top_level.txt,sha256=BiBclu7jWJ0AZ35vUr3hN9_cg8JL9EiH_hjFxquMxtw,33
+transferqueue-0.1.1.dev2.dist-info/RECORD,,