switch-api 0.6.15__tar.gz → 0.6.16__tar.gz

{switch_api-0.6.15 → switch_api-0.6.16}/HISTORY.md

@@ -1,5 +1,11 @@
 # History
 
+#0.6.16
+### Changed
+In the `controls` module:
+- Updated `submit_control_continue()` to send log control requests to the IoT endpoint in chunks of 50 per request, improving reliability and preventing oversized payloads.
+- Updated `submit_control_continue()` to also batch control requests sent via MQTT (50 per batch), ensuring more stable and efficient message delivery.
+
 #0.6.15
 ### Added
 In the `automation` module:

{switch_api-0.6.15 → switch_api-0.6.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: switch_api
-Version: 0.6.15
+Version: 0.6.16
 Summary: A complete package for data ingestion into the Switch Automation Platform.
 Home-page: UNKNOWN
 Author: Switch Automation Pty Ltd.
@@ -38,6 +38,12 @@ pip install switch_api
 
 # History
 
+#0.6.16
+### Changed
+In the `controls` module:
+- Updated `submit_control_continue()` to send log control requests to the IoT endpoint in chunks of 50 per request, improving reliability and preventing oversized payloads.
+- Updated `submit_control_continue()` to also batch control requests sent via MQTT (50 per batch), ensuring more stable and efficient message delivery.
+
 #0.6.15
 ### Added
 In the `automation` module:

{switch_api-0.6.15 → switch_api-0.6.16}/setup.py

@@ -16,7 +16,7 @@ setup(
     long_description_content_type='text/markdown',
     license='MIT License',
     name="switch_api",
-    version="0.6.15",
+    version="0.6.16",
     packages=find_packages(include=["switch_api", "switch_api.*"]),
     install_requires=['pandas==1.5.3', 'requests', 'azure-storage-blob', 'pandera[io]==0.7.1', 'azure-servicebus',
                       'msal>=1.11.0', 'paho-mqtt==1.6.1', 'uvicorn==0.22.0', 'fastapi==0.98.0', 'pyodbc==4.0.39', 'cachetools==5.5.0', 

{switch_api-0.6.15 → switch_api-0.6.16}/switch_api/__init__.py

@@ -42,4 +42,4 @@ from . import _guide
 # log.addHandler(_ch)
 # https://dev.to/joaomcteixeira/setting-up-python-logging-for-a-library-app-6ml
 
-__version__ = "0.6.15"
+__version__ = "0.6.16"

{switch_api-0.6.15 → switch_api-0.6.16}/switch_api/controls/controls.py

@@ -55,6 +55,43 @@ global _session
 _control_components = {}
 _switch_mqtt = None
 _session = None
+MAX_CONTROL_REQUEST_SENSORS = 50
+
+def _chunk_dataframe(df: pandas.DataFrame, chunk_size: int = MAX_CONTROL_REQUEST_SENSORS) -> List[pandas.DataFrame]:
+    """Split a dataframe into bounded chunks while preserving row order."""
+    if df.empty:
+        return []
+
+    return [
+        df.iloc[start_idx:start_idx + chunk_size].copy()
+        for start_idx in range(0, len(df), chunk_size)
+    ]
+
+def _build_chunk_session_ids(chunk_count: int, session_id: Union[uuid.UUID, str, None] = None) -> List[uuid.UUID]:
+    """Return one unique session id per chunk, reusing the provided id for the first chunk when valid."""
+    if chunk_count <= 0:
+        return []
+
+    chunk_session_ids: List[uuid.UUID] = []
+    seen_session_ids = set()
+
+    if is_valid_uuid(session_id):
+        first_session_id = uuid.UUID(str(session_id))
+    else:
+        first_session_id = uuid.uuid4()
+
+    chunk_session_ids.append(first_session_id)
+    seen_session_ids.add(str(first_session_id))
+
+    while len(chunk_session_ids) < chunk_count:
+        next_session_id = uuid.uuid4()
+        if str(next_session_id) in seen_session_ids:
+            continue
+
+        chunk_session_ids.append(next_session_id)
+        seen_session_ids.add(str(next_session_id))
+
+    return chunk_session_ids
 
 def submit_control_continue(api_inputs: ApiInputs, data: pandas.DataFrame, information_columns: list = [], 
                             is_send_control: bool = True, ws_timeout_in_seconds: int = WS_MQTT_CONNECTION_TIMEOUT):
@@ -409,39 +446,70 @@ def __send_control(api_inputs: ApiInputs, installation_id: Union[uuid.UUID, str]
     acknowledgement_df = format_acknowledgement_payload_df(
         df=acknowledgement_df, required_columns=required_columns, has_priority=has_priority)
 
+    control_request_df = control_request_df.copy()
+    control_request_df = format_control_payload_df(
+        required_columns=required_columns, has_priority=has_priority, df=control_request_df)
+
+    acknowledgement_chunks = _chunk_dataframe(acknowledgement_df)
+    control_request_chunks = _chunk_dataframe(control_request_df)
+    chunk_session_ids = _build_chunk_session_ids(
+        chunk_count=len(control_request_chunks),
+        session_id=session_id)
+
     if is_send_control:
-        _, result_df = log_control_request(
-            session=_session,
-            api_inputs=api_inputs, 
-            df=acknowledgement_df, 
-            session_id=session_id, 
-            installation_id=installation_id)
-        if result_df.empty:
+        if len(acknowledgement_chunks) != len(control_request_chunks):
+            logger.error("Chunk count mismatch between control acknowledgement and MQTT payloads.")
             df_matrix[COL_CONTROL_STATUS] = ControlStatus.NotSentToService.value
             return df_matrix
 
-    control_request_df = control_request_df.copy()
-    control_request_df = format_control_payload_df(
-        required_columns=required_columns, has_priority=has_priority, df=control_request_df)
+        successful_ack_chunks = []
+        successful_control_chunks = []
+        successful_session_ids = []
+
+        for acknowledgement_chunk, control_request_chunk, chunk_session_id in zip(
+            acknowledgement_chunks,
+            control_request_chunks,
+            chunk_session_ids,
+        ):
+            _, result_df = log_control_request(
+                session=_session,
+                api_inputs=api_inputs,
+                df=acknowledgement_chunk,
+                session_id=chunk_session_id,
+                installation_id=installation_id,
+            )
 
-    def process_paged_request(df, page_size: int = 20):
-        paged_results = pandas.DataFrame()
-        total_rows = len(df)
-        num_pages = (total_rows + page_size - 1) // page_size
+            if result_df.empty:
+                logger.warning(
+                    "Skipping failed chunk for session_id=%s",
+                    chunk_session_id,
+                )
+                continue
 
-        for page_num in range(num_pages):
-            start_idx = page_num * page_size
-            end_idx = min((page_num + 1) * page_size, total_rows)
+            successful_ack_chunks.append(acknowledgement_chunk)
+            successful_control_chunks.append(control_request_chunk)
+            successful_session_ids.append(chunk_session_id)
 
-            # Get the data for the current page
-            page_data = df.iloc[start_idx:end_idx]
+        #only successfuly chunks are used for the control request next
+        acknowledgement_chunks = successful_ack_chunks
+        control_request_chunks = successful_control_chunks
+        chunk_session_ids = successful_session_ids
 
-            result = send_control(page_data)
+        if not acknowledgement_chunks:
+            logger.error("All chunks failed during log control request.")
+            df_matrix[COL_CONTROL_STATUS] = ControlStatus.NotSentToService.value
+            return df_matrix
+            
+    def process_paged_request(chunks: List[pandas.DataFrame], chunk_session_ids: List[uuid.UUID]):
+        paged_results = pandas.DataFrame()
+
+        for page_data, chunk_session_id in zip(chunks, chunk_session_ids):
+            result = send_control(page_data, chunk_session_id)
             paged_results = pandas.concat([paged_results, result])
 
         return paged_results
 
-    def send_control(page_data):
+    def send_control(page_data: pandas.DataFrame, chunk_session_id: uuid.UUID):
         retry_count = 0
         max_retries = 3
         success_results = pandas.DataFrame()
@@ -451,7 +519,7 @@ def __send_control(api_inputs: ApiInputs, installation_id: Union[uuid.UUID, str]
 
         while retry_count < max_retries:
             success_response, missing_response = _switch_mqtt.send_control_request(
-                session_id=str(session_id),
+                session_id=str(chunk_session_id),
                 installation_id=installation_id,
                 sensors=dataframe_to_control.to_dict(orient='records'))
 
@@ -522,7 +590,7 @@ def __send_control(api_inputs: ApiInputs, installation_id: Union[uuid.UUID, str]
         return control_result
 
     if is_send_control:
-        control_results = process_paged_request(control_request_df)
+        control_results = process_paged_request(control_request_chunks, chunk_session_ids)
         control_results = control_results.rename(
             columns={'sensorId': COL_OBJECT_PROPERTY_ID})
         control_results = pandas.merge(control_results, df_matrix[[
@@ -673,12 +741,18 @@ def submit_control(api_inputs: ApiInputs, installation_id: Union[uuid.UUID, str]
     if _session == None:
         _session = requests_retry_session2(retries=5, backoff_factor=2)
 
-    _, result_df = log_control_request(session=_session, 
-                        api_inputs=api_inputs, 
-                        df=data_frame, 
-                        session_id=session_id, 
-                        installation_id=installation_id)
-    if result_df.empty:
+    acknowledgement_chunks = _chunk_dataframe(data_frame)
+    chunk_session_ids = _build_chunk_session_ids(
+        chunk_count=len(acknowledgement_chunks),
+        session_id=session_id)
+
+    for acknowledgement_chunk, chunk_session_id in zip(acknowledgement_chunks, chunk_session_ids):
+        _, result_df = log_control_request(session=_session,
+                            api_inputs=api_inputs,
+                            df=acknowledgement_chunk,
+                            session_id=chunk_session_id,
+                            installation_id=installation_id)
+        if result_df.empty:
             return pandas.DataFrame()
 
     # Proceeds when the control request is successful
@@ -722,24 +796,18 @@ def submit_control(api_inputs: ApiInputs, installation_id: Union[uuid.UUID, str]
         logger.info("Could not connect to MQTT Broker.")
         return 'Could not connect to MQTT Broker.', pandas.DataFrame()
 
-    def process_paged_request(df, page_size: int = 20):
-        paged_results = pandas.DataFrame()
-        total_rows = len(df)
-        num_pages = (total_rows + page_size - 1) // page_size
-
-        for page_num in range(num_pages):
-            start_idx = page_num * page_size
-            end_idx = min((page_num + 1) * page_size, total_rows)
+    control_request_chunks = _chunk_dataframe(data_frame)
 
-            # Get the data for the current page
-            page_data = df.iloc[start_idx:end_idx]
+    def process_paged_request(chunks: List[pandas.DataFrame], chunk_session_ids: List[uuid.UUID]):
+        paged_results = pandas.DataFrame()
 
-            result = send_control(page_data)
+        for page_data, chunk_session_id in zip(chunks, chunk_session_ids):
+            result = send_control(page_data, chunk_session_id)
             paged_results = pandas.concat([paged_results, result])
 
         return paged_results
 
-    def send_control(page_data):
+    def send_control(page_data: pandas.DataFrame, chunk_session_id: uuid.UUID):
         retry_count = 0
         max_retries = 3
         success_results = pandas.DataFrame()
@@ -749,7 +817,7 @@ def submit_control(api_inputs: ApiInputs, installation_id: Union[uuid.UUID, str]
 
         while retry_count < max_retries:
             success_response, missing_response = _switch_mqtt.send_control_request(
-                session_id=str(session_id),
+                session_id=str(chunk_session_id),
                 installation_id=installation_id,
                 sensors=dataframe_to_control.to_dict(orient='records'))
 
@@ -821,7 +889,7 @@ def submit_control(api_inputs: ApiInputs, installation_id: Union[uuid.UUID, str]
     
     # [Subscribe] to ClientId + Site State Topics
     _switch_mqtt.subscribe_installation(installation_id=installation_id)
-    control_results = process_paged_request(data_frame)
+    control_results = process_paged_request(control_request_chunks, chunk_session_ids)
 
     return control_results
  
@@ -983,6 +1051,8 @@ def log_control_request(session: requests.Session, api_inputs: ApiInputs, df: pa
     logger.info("Control Request Session Id: %s", session_id)
     logger.info("Control Request for User: %s=%s",
                 api_inputs.email_address, api_inputs.user_id)
+    logger.info("Log Control Request Payload:")
+    logger.info(json_payload)
     
     try:
         response = session_post(session, url, json_payload, headers)
@@ -1177,4 +1247,4 @@ def session_post(session, url, json_payload, headers):
         if "Connection refused" in str(ex):
             logger.error("Hard failure: Connection Refused. Not retrying.")
             raise
-        raise
+        raise

{switch_api-0.6.15 → switch_api-0.6.16}/switch_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: switch-api
-Version: 0.6.15
+Version: 0.6.16
 Summary: A complete package for data ingestion into the Switch Automation Platform.
 Home-page: UNKNOWN
 Author: Switch Automation Pty Ltd.
@@ -38,6 +38,12 @@ pip install switch_api
 
 # History
 
+#0.6.16
+### Changed
+In the `controls` module:
+- Updated `submit_control_continue()` to send log control requests to the IoT endpoint in chunks of 50 per request, improving reliability and preventing oversized payloads.
+- Updated `submit_control_continue()` to also batch control requests sent via MQTT (50 per batch), ensuring more stable and efficient message delivery.
+
 #0.6.15
 ### Added
 In the `automation` module: