TransferQueue 0.1.1.dev0__py3-none-any.whl → 0.1.1.dev1__py3-none-any.whl

transfer_queue/controller.py

@@ -47,8 +47,8 @@ from transfer_queue.utils.zmq_utils import (
 logger = logging.getLogger(__name__)
 logger.setLevel(os.getenv("TQ_LOGGING_LEVEL", logging.WARNING))
 
-TQ_CONTROLLER_GET_METADATA_TIMEOUT = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_TIMEOUT", 300))
-TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL", 1))
+TQ_CONTROLLER_GET_METADATA_TIMEOUT = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_TIMEOUT", 1))
+TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL = int(os.environ.get("TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL", 0.2))
 TQ_CONTROLLER_CONNECTION_CHECK_INTERVAL = int(os.environ.get("TQ_CONTROLLER_CONNECTION_CHECK_INTERVAL", 2))
 
 TQ_INIT_SAMPLE_NUM = int(os.environ.get("TQ_INIT_SAMPLE_NUM", 10))  # Initial number of samples
@@ -753,6 +753,17 @@ class TransferQueueController:
                 ready_for_consume_indexes = self.scan_data_status(partition_id, data_fields, task_name, batch_size)
 
                 if len(ready_for_consume_indexes) < batch_size:
+                    if time.time() - start_time > TQ_CONTROLLER_GET_METADATA_TIMEOUT:
+                        raise TimeoutError(
+                            f"Timeout while waiting for sufficient data. "
+                            f"Required: {batch_size}, Available: {len(ready_for_consume_indexes)}"
+                        )
+                    logger.warning(
+                        f"Insufficient complete groups available. Required: {batch_size}, "
+                        f"Available: {len(ready_for_consume_indexes)}. Retrying in "
+                        f"{TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL}s..."
+                    )
+                    time.sleep(TQ_CONTROLLER_GET_METADATA_CHECK_INTERVAL)
                     continue
 
                 # Try sampling - if it returns empty lists, retry

transfer_queue/version/version

@@ -1 +1 @@
-0.1.1.dev
+0.1.1.dev1

{transferqueue-0.1.1.dev0.dist-info → transferqueue-0.1.1.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: TransferQueue
-Version: 0.1.1.dev0
+Version: 0.1.1.dev1
 Summary: TransferQueue: An Asynchronous Streaming Data Management Module
 Author-email: The TransferQueue Team <hanzy19@tsinghua.org.cn>
 License: Apache-2.0

{transferqueue-0.1.1.dev0.dist-info → transferqueue-0.1.1.dev1.dist-info}/RECORD

@@ -12,7 +12,7 @@ tests/test_simple_storage_unit.py,sha256=Sczhw2bdCfTfa7RwnpW-aKCV1of8mO6z3l3q2PD
 tests/test_storage_client_factory.py,sha256=U0gS_l4bc_bP7K_uPhy8UlJ186HyTrA10TSJNdf5SBM,1708
 transfer_queue/__init__.py,sha256=68c0sBfqHPqTa7OdzO4sAZB52XvwtjpwLqP9BWAh4fA,1535
 transfer_queue/client.py,sha256=zDlH1beWwRbjz0a7S8QH9IOJ1wp5yEQ36XYwFwJTXOY,24949
-transfer_queue/controller.py,sha256=gFS3OMM7giSqwypA7qmMFZ_AxG22LIpCQIblbqC_Gm8,48988
+transfer_queue/controller.py,sha256=uc_MQAlG_QmJ8szxc5yPLaZvCeP2CRULnAADtcf41-8,49702
 transfer_queue/metadata.py,sha256=LHQhM7vixw5QcA05nZcqWbxSh7mhJ2ETJOo_1gFb3eI,17645
 transfer_queue/sampler/__init__.py,sha256=1oauDy2Dwb5GXhKi7tl5DWAHv8i4t2MQK1S4U36Sy4g,788
 transfer_queue/sampler/base.py,sha256=wFti4dNJb3YArYpGzxA_YDfyUTdTG8wVz6HclPDyZPw,3299
@@ -33,9 +33,9 @@ transfer_queue/utils/__init__.py,sha256=vki-5RVaRBKxVc6Q7XPQox3VNPio2DvJYvRz0SZt
 transfer_queue/utils/serial_utils.py,sha256=9ZgsytTp-441YKtIRFqyH5NhNifSqRKa2h0FI44ltcc,10200
 transfer_queue/utils/utils.py,sha256=Pno4h3WjX_eT7q4xiVV6Jkhquc39Fp-Ycsg1cv0qNKQ,4544
 transfer_queue/utils/zmq_utils.py,sha256=jCg2pQfvy_IYdGyZq4nvL4CAwjJc7Li0Trp3T3GDBMg,5118
-transfer_queue/version/version,sha256=6GX4KkA_FGXCFr4Z-Mw9dUky3PbKBVWPbSJCCVmVPlw,9
-transferqueue-0.1.1.dev0.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-transferqueue-0.1.1.dev0.dist-info/METADATA,sha256=3YY8wlygPXiXf5Ct4x6Is4ezv6-ZR1bwnL3tyTxZf7E,19531
-transferqueue-0.1.1.dev0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-transferqueue-0.1.1.dev0.dist-info/top_level.txt,sha256=BiBclu7jWJ0AZ35vUr3hN9_cg8JL9EiH_hjFxquMxtw,33
-transferqueue-0.1.1.dev0.dist-info/RECORD,,
+transfer_queue/version/version,sha256=IBGGJN0Ii7hsCVlxnB5t6XS0CWaPTLXLjPL0jyY3vKE,11
+transferqueue-0.1.1.dev1.dist-info/licenses/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+transferqueue-0.1.1.dev1.dist-info/METADATA,sha256=nYLoCMpxUeP1Go3Pnai9W7lVRc4wRAUocqrzyP4K5ds,19531
+transferqueue-0.1.1.dev1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+transferqueue-0.1.1.dev1.dist-info/top_level.txt,sha256=BiBclu7jWJ0AZ35vUr3hN9_cg8JL9EiH_hjFxquMxtw,33
+transferqueue-0.1.1.dev1.dist-info/RECORD,,