moose-lib 0.4.276__py3-none-any.whl → 0.4.278__py3-none-any.whl

moose_lib/dmv2/olap_table.py

@@ -426,6 +426,17 @@ class OlapTable(TypedMooseResource, Generic[T]):
     def _to_json_each_row(self, records: list[dict]) -> bytes:
         return "\n".join(json.dumps(r, default=str) for r in records).encode("utf-8")
 
+    def _with_wait_end_settings(self, settings: dict) -> dict:
+        """Add wait_end_of_query setting to ensure at least once delivery for INSERT operations.
+        
+        Args:
+            settings: Base settings dictionary
+            
+        Returns:
+            Settings dictionary with wait_end_of_query added
+        """
+        return {**settings, "wait_end_of_query": 1}
+
     def _prepare_insert_options(
         self,
         table_name: str,
@@ -437,13 +448,14 @@ class OlapTable(TypedMooseResource, Generic[T]):
     ) -> tuple[str, bytes, dict]:
         """Prepare insert options for JSONEachRow raw SQL insert, returning settings dict."""
         # Base settings for all inserts
-        settings = {
+        base_settings = {
             "date_time_input_format": "best_effort",
             "max_insert_block_size": 100000 if is_stream else min(len(validated_data), 100000),
             "max_block_size": 65536,
             "async_insert": 1 if len(validated_data) > 1000 else 0,
             "wait_for_async_insert": 1,
         }
+        settings = self._with_wait_end_settings(base_settings)
         if (strategy == "discard" and options and
             (options.allow_errors is not None or options.allow_errors_ratio is not None)):
             if options.allow_errors is not None:
@@ -541,12 +553,13 @@ class OlapTable(TypedMooseResource, Generic[T]):
             batch = records_dict[i:i + RETRY_BATCH_SIZE]
             try:
                 sql = f"INSERT INTO {table_name} FORMAT JSONEachRow"
-                settings = {
+                base_settings = {
                     "date_time_input_format": "best_effort",
                     "max_insert_block_size": RETRY_BATCH_SIZE,
                     "max_block_size": RETRY_BATCH_SIZE,
                     "async_insert": 0
                 }
+                settings = self._with_wait_end_settings(base_settings)
                 json_lines = self._to_json_each_row(batch)
                 client.command(sql, data=json_lines, settings=settings)
                 successful.extend(records[i:i + RETRY_BATCH_SIZE])
@@ -554,12 +567,12 @@ class OlapTable(TypedMooseResource, Generic[T]):
                 for j, record_dict in enumerate(batch):
                     try:
                         sql = f"INSERT INTO {table_name} FORMAT JSONEachRow"
-                        settings = {
+                        individual_settings = self._with_wait_end_settings({
                             "date_time_input_format": "best_effort",
                             "async_insert": 0
-                        }
+                        })
                         json_line = self._to_json_each_row([record_dict])
-                        client.command(sql, data=json_line, settings=settings)
+                        client.command(sql, data=json_line, settings=individual_settings)
                         successful.append(records[i + j])
                     except ClickHouseError as error:
                         failed.append(FailedRecord(
@@ -675,14 +688,18 @@ class OlapTable(TypedMooseResource, Generic[T]):
                 if len(batch) >= 1000:  # Batch size
                     json_lines = self._to_json_each_row(batch)
                     sql = f"INSERT INTO {table_name} FORMAT JSONEachRow"
-                    client.command(sql, data=json_lines, settings=settings)
+                    # Add wait_end_of_query to batch settings using helper function
+                    batch_settings = self._with_wait_end_settings(settings)
+                    client.command(sql, data=json_lines, settings=batch_settings)
                     total_inserted += len(batch)
                     batch = []
 
             if batch:  # Insert any remaining records
                 json_lines = self._to_json_each_row(batch)
                 sql = f"INSERT INTO {table_name} FORMAT JSONEachRow"
-                client.command(sql, data=json_lines, settings=settings)
+                # Add wait_end_of_query to final batch settings using helper function
+                final_settings = self._with_wait_end_settings(settings)
+                client.command(sql, data=json_lines, settings=final_settings)
                 total_inserted += len(batch)
 
             return InsertResult(

{moose_lib-0.4.276.dist-info → moose_lib-0.4.278.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: moose_lib
-Version: 0.4.276
+Version: 0.4.278
 Home-page: https://www.fiveonefour.com/moose
 Author: Fiveonefour Labs Inc.
 Author-email: support@fiveonefour.com

{moose_lib-0.4.276.dist-info → moose_lib-0.4.278.dist-info}/RECORD

@@ -18,7 +18,7 @@ moose_lib/dmv2/consumption.py,sha256=vt1APul1O21yPHaB0MRjF_9ohz5MJs5QIpDCnzFwyJA
 moose_lib/dmv2/ingest_api.py,sha256=Snek9NGwaJl_BuImSWGtQq91m9D3AJ4qBoGiKZ-9yTQ,2323
 moose_lib/dmv2/ingest_pipeline.py,sha256=Y1gsvHZjlW07gMapLnBRJEsoAPv7ThvLABoLmVV7BHE,6714
 moose_lib/dmv2/materialized_view.py,sha256=kcx-sJFTM-cH3Uc1GoldgFGodjoz0AegAQEMmohdS38,3826
-moose_lib/dmv2/olap_table.py,sha256=U-ZdWzLKbxq1ANcG37rI3xJtouW3VM_NFGCWOcuwEs8,30567
+moose_lib/dmv2/olap_table.py,sha256=mQYPThAhIfZ4aTadb_tdRvQyXcjcwlY26yF1iKzuih0,31479
 moose_lib/dmv2/registry.py,sha256=AaGS6Xy0vKz-wHLPgRVxfKfSwW5KksMePjZ8N7-2OKU,2054
 moose_lib/dmv2/sql_resource.py,sha256=kUZoGqxhZMHMthtBZGYJBxTFjXkspXiWLXhJRYXgGUM,1864
 moose_lib/dmv2/stream.py,sha256=H5nzqVHIXulFNMNaGZUQnhGjNx7fIg0X95kxAO_qlls,10600
@@ -31,7 +31,7 @@ tests/__init__.py,sha256=0Gh4yzPkkC3TzBGKhenpMIxJcRhyrrCfxLSfpTZnPMQ,53
 tests/conftest.py,sha256=ZVJNbnr4DwbcqkTmePW6U01zAzE6QD0kNAEZjPG1f4s,169
 tests/test_moose.py,sha256=mBsx_OYWmL8ppDzL_7Bd7xR6qf_i3-pCIO3wm2iQNaA,2136
 tests/test_redis_client.py,sha256=d9_MLYsJ4ecVil_jPB2gW3Q5aWnavxmmjZg2uYI3LVo,3256
-moose_lib-0.4.276.dist-info/METADATA,sha256=XwfYCRSicgrtOuI-9rBGbxlO0IMAgCm4FBPPXnFHMhg,729
-moose_lib-0.4.276.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-moose_lib-0.4.276.dist-info/top_level.txt,sha256=XEns2-4aCmGp2XjJAeEH9TAUcGONLnSLy6ycT9FSJh8,16
-moose_lib-0.4.276.dist-info/RECORD,,
+moose_lib-0.4.278.dist-info/METADATA,sha256=99ouJnsUC33mfZl_aJfc2aimdrP2SsE2leoSaGW0b9I,729
+moose_lib-0.4.278.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+moose_lib-0.4.278.dist-info/top_level.txt,sha256=XEns2-4aCmGp2XjJAeEH9TAUcGONLnSLy6ycT9FSJh8,16
+moose_lib-0.4.278.dist-info/RECORD,,