python-databases 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

python_databases/elasticsearch_infrastructure/elasticsearch.py

@@ -55,13 +55,22 @@ class ElasticSearch(ABC):
         tracer = get_logger("elasticsearch")
         tracer.setLevel(debug_level)
 
-    def _set_list_of_docs_quick(self, list_of_docs: list[dict], request_timeout: int = 600) -> None:
+    def _set_list_of_docs_quick(
+        self,
+        list_of_docs: list[dict],
+        request_timeout: int = 600,
+        raise_on_error: bool = False,
+    ) -> None:
+        failed_response_list = []
         if failed_response := helpers.streaming_bulk(
-            self.elk_client, list_of_docs, raise_on_error=False, request_timeout=request_timeout, chunk_size=1000
+                self.elk_client, list_of_docs, raise_on_error=False, request_timeout=request_timeout, chunk_size=1000
         ):
             for item in failed_response:
                 if item[1]["index"]["status"] != 201 or item[1]["index"]["_shards"]["failed"] > 0:
                     self.logger.debug(f"Failed document: \n{item}")
+                    failed_response_list.append(item)
+        if raise_on_error and failed_response_list:
+            raise Exception(f"Failed to report {len(failed_response_list)} documents")
 
     def _set_list_of_docs_safe(self, list_of_docs: list[dict]) -> None:
         success_response, failed_response = helpers.bulk(self.elk_client, list_of_docs)
@@ -73,11 +82,21 @@ class ElasticSearch(ABC):
                 f"The documents that were not reported successfully: {failed_response}"
             )
 
-    def post_list_of_docs(self, list_of_docs: list[dict], request_timeout: int = 600, quick: bool = False) -> None:
+    def post_list_of_docs(
+        self,
+        list_of_docs: list[dict],
+        request_timeout: int = 600,
+        quick: bool = False,
+        raise_on_error: bool = False
+    ) -> None:
         self.logger.info("Start to report the documents to ELK")
 
         if quick:
-            self._set_list_of_docs_quick(list_of_docs=list_of_docs, request_timeout=request_timeout)
+            self._set_list_of_docs_quick(
+                list_of_docs=list_of_docs,
+                request_timeout=request_timeout,
+                raise_on_error=raise_on_error
+            )
         else:
             self._set_list_of_docs_safe(list_of_docs=list_of_docs)
         self.logger.info("Finish to report the documents to ELK")
@@ -135,16 +154,27 @@ class ElasticSearch(ABC):
         return list_of_docs
 
     def post_list_of_docs_as_bulk_chunk(
-        self, list_of_docs: list[dict], chunk_size: int = 1000, time_sleep: int = 60, quick: bool = False
+        self,
+        list_of_docs: list[dict],
+        chunk_size: int = 1000,
+        time_sleep: int = 60,
+        quick: bool = False,
+        raise_on_error: bool = False,
     ) -> None:
         for i in range(0, len(list_of_docs), chunk_size):
             self.logger.info(f"index: {i} - {i + chunk_size} / {len(list_of_docs)}")
-            chunk = list_of_docs[i : i + chunk_size]
-            self.post_list_of_docs(list_of_docs=chunk, quick=quick)
+            chunk = list_of_docs[i: i + chunk_size]
+            self.post_list_of_docs(list_of_docs=chunk, quick=quick, raise_on_error=raise_on_error)
         time.sleep(time_sleep)
 
     def fill_elk_index_as_bulk(  # post_list_of_docs_as_bulk
-        self, data: list[dict], doc_index_name: str, chunk_size: int = 1000, time_sleep: int = 1, quick: bool = True
+        self,
+        data: list[dict],
+        doc_index_name: str,
+        chunk_size: int = 1000,
+        time_sleep: int = 1,
+        quick: bool = True,
+        raise_on_error: bool = False,
     ) -> None:  # use as the main function to send data to the elastic search
         list_of_docs = self._prepare_documents_for_bulk(
             data=data,
@@ -156,6 +186,7 @@ class ElasticSearch(ABC):
             chunk_size=chunk_size,
             time_sleep=time_sleep,
             quick=quick,
+            raise_on_error=raise_on_error,
         )
 
     def check_if_index_exists(self, index: str) -> bool:

{python_databases-1.1.0.dist-info → python_databases-1.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-databases
-Version: 1.1.0
+Version: 1.2.0
 Summary: A Python library for managing and interacting with multiple databases seamlessly.
 Author: Avi Zaguri
 License: MIT
@@ -10,7 +10,7 @@ Project-URL: Issues, https://github.com/aviz92/python-databases/issues
 Keywords: development,databases,ORM,SQL,NoSQL,database management
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
-Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: System :: Logging
 Requires-Python: >=3.12

{python_databases-1.1.0.dist-info → python_databases-1.2.0.dist-info}/RECORD

@@ -2,12 +2,12 @@ python_databases/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 python_databases/consts.py,sha256=ADn3PmhTXKBobtiVgDR4Tqq2Ncn_FTK0WjgREOLdBPs,33
 python_databases/elasticsearch_infrastructure/__init__.py,sha256=U1orkPi0Pg-1rtY1CPSF4-eVRfrMr36qpT9VNKjBUw4,297
 python_databases/elasticsearch_infrastructure/consts.py,sha256=tW4jgeUsf3FhqEpNHZQr3SjqPbuJjcWQDsSqxAvUYPg,98
-python_databases/elasticsearch_infrastructure/elasticsearch.py,sha256=SJkwV1kM2IrGpTbNtUJWp2K5dbAApMCCAnQ_jD13lPg,10047
+python_databases/elasticsearch_infrastructure/elasticsearch.py,sha256=8Mu1li0AT0puqjSh8SQdP9Drv4lQDHGGcQ4YNlBeIs4,10739
 python_databases/postgresql_infrastructure/__init__.py,sha256=yBjq7SLnNgnxW0DCQHEwCi_FrCNtjsJGJXIq7XtkUZg,150
 python_databases/postgresql_infrastructure/consts.py,sha256=Fz19Qj-B9mPCTGInNygm2CPmY0nWiamWeEMrJsv-XB4,102
 python_databases/postgresql_infrastructure/postgresql.py,sha256=tFdQr5IOPYbuqKE3r9oqSoNoGMbED5NSEy1anrUNEyY,3105
-python_databases-1.1.0.dist-info/licenses/LICENSE,sha256=cSikHY6SZFsPZSBizCDAJ0-Bjjzxt-JtX6TVbKxwimo,1067
-python_databases-1.1.0.dist-info/METADATA,sha256=CrCiF3Xm9os9Ize1JFmgsNk91QbZO9el7VlGrScmZ3E,2463
-python_databases-1.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-python_databases-1.1.0.dist-info/top_level.txt,sha256=fXcMUntaYJfKeFS7FjQ6DY4WVXYBsgdrDmhOudYs-vg,17
-python_databases-1.1.0.dist-info/RECORD,,
+python_databases-1.2.0.dist-info/licenses/LICENSE,sha256=cSikHY6SZFsPZSBizCDAJ0-Bjjzxt-JtX6TVbKxwimo,1067
+python_databases-1.2.0.dist-info/METADATA,sha256=xlmPxDkSjI0xl5UPEFtvclI71101SED6KkF5JrTDAys,2463
+python_databases-1.2.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+python_databases-1.2.0.dist-info/top_level.txt,sha256=fXcMUntaYJfKeFS7FjQ6DY4WVXYBsgdrDmhOudYs-vg,17
+python_databases-1.2.0.dist-info/RECORD,,

{python_databases-1.1.0.dist-info → python_databases-1.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (82.0.1)
+Generator: setuptools (84.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any