python-databases 1.2.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

python_databases/elasticsearch_infrastructure/consts.py

@@ -1,3 +1,3 @@
 from python_databases.consts import LOGGER_NAME
 
-ELK_LOGGER_NAME = f'{LOGGER_NAME}.elasticsearch'
+ELK_LOGGER_NAME = f"{LOGGER_NAME}.elasticsearch"

python_databases/elasticsearch_infrastructure/elasticsearch.py

@@ -63,7 +63,7 @@ class ElasticSearch(ABC):
     ) -> None:
         failed_response_list = []
         if failed_response := helpers.streaming_bulk(
-                self.elk_client, list_of_docs, raise_on_error=False, request_timeout=request_timeout, chunk_size=1000
+            self.elk_client, list_of_docs, raise_on_error=False, request_timeout=request_timeout, chunk_size=1000
         ):
             for item in failed_response:
                 if item[1]["index"]["status"] != 201 or item[1]["index"]["_shards"]["failed"] > 0:
@@ -83,19 +83,13 @@ class ElasticSearch(ABC):
             )
 
     def post_list_of_docs(
-        self,
-        list_of_docs: list[dict],
-        request_timeout: int = 600,
-        quick: bool = False,
-        raise_on_error: bool = False
+        self, list_of_docs: list[dict], request_timeout: int = 600, quick: bool = False, raise_on_error: bool = False
     ) -> None:
         self.logger.info("Start to report the documents to ELK")
 
         if quick:
             self._set_list_of_docs_quick(
-                list_of_docs=list_of_docs,
-                request_timeout=request_timeout,
-                raise_on_error=raise_on_error
+                list_of_docs=list_of_docs, request_timeout=request_timeout, raise_on_error=raise_on_error
             )
         else:
             self._set_list_of_docs_safe(list_of_docs=list_of_docs)
@@ -113,7 +107,7 @@ class ElasticSearch(ABC):
         doc.update(
             {
                 "doc_id": timestamp.strftime("%Y%m%dT%H%M%S%f"),
-                "timestamp": timestamp,
+                "upload_timestamp": timestamp,
                 "date_and_time_str": date_and_time_str,
                 "date_str": date,
                 "time_str": time_part,
@@ -145,10 +139,16 @@ class ElasticSearch(ABC):
     def _fill_list_of_docs(index_name: str, doc: dict[str, Any]) -> dict:
         return {"_index": index_name, "_source": deepcopy(doc)}
 
-    def _prepare_documents_for_bulk(self, data: list[dict], doc_index_name: str) -> list[dict]:
+    def _prepare_documents_for_bulk(
+        self,
+        data: list[dict],
+        doc_index_name: str,
+        log_progress: bool = False,
+    ) -> list[dict]:
         list_of_docs = []
         for index, row in enumerate(data):
-            self.logger.debug(f"index: {index + 1}/{len(data)}")
+            if log_progress:
+                self.logger.debug(f"index: {index + 1}/{len(data)}")
             doc = self._build_document(row=row)
             list_of_docs.append(self._fill_list_of_docs(index_name=doc_index_name, doc=doc))
         return list_of_docs
@@ -160,10 +160,12 @@ class ElasticSearch(ABC):
         time_sleep: int = 60,
         quick: bool = False,
         raise_on_error: bool = False,
+        log_progress: bool = False,
     ) -> None:
         for i in range(0, len(list_of_docs), chunk_size):
-            self.logger.info(f"index: {i} - {i + chunk_size} / {len(list_of_docs)}")
-            chunk = list_of_docs[i: i + chunk_size]
+            if log_progress:
+                self.logger.info(f"index: {i} - {i + chunk_size} / {len(list_of_docs)}")
+            chunk = list_of_docs[i : i + chunk_size]
             self.post_list_of_docs(list_of_docs=chunk, quick=quick, raise_on_error=raise_on_error)
         time.sleep(time_sleep)
 
@@ -175,10 +177,12 @@ class ElasticSearch(ABC):
         time_sleep: int = 1,
         quick: bool = True,
         raise_on_error: bool = False,
+        log_progress: bool = False,
     ) -> None:  # use as the main function to send data to the elastic search
         list_of_docs = self._prepare_documents_for_bulk(
             data=data,
             doc_index_name=doc_index_name,
+            log_progress=log_progress,
         )
 
         self.post_list_of_docs_as_bulk_chunk(
@@ -187,6 +191,7 @@ class ElasticSearch(ABC):
             time_sleep=time_sleep,
             quick=quick,
             raise_on_error=raise_on_error,
+            log_progress=log_progress,
         )
 
     def check_if_index_exists(self, index: str) -> bool:
@@ -213,11 +218,16 @@ class ElasticSearch(ABC):
         except Exception as e:
             raise Exception(f"Failed to get documents from index '{index}': {str(e)}") from e
 
-    def convert_dataframes_to_list_of_docs(self, dataframe: pd.DataFrame) -> list:
+    def convert_dataframes_to_list_of_docs(
+        self,
+        dataframe: pd.DataFrame,
+        log_progress: bool = False,
+    ) -> list:
         data = []
         dataframe_values = dataframe.values.tolist()
         for index, value in enumerate(dataframe_values):
-            self.logger.debug(f"index: {index}/{len(dataframe_values)}")
+            if log_progress:
+                self.logger.debug(f"index: {index}/{len(dataframe_values)}")
             data.append(value)
         return data
 

python_databases/postgresql_infrastructure/consts.py

@@ -1,3 +1,3 @@
 from python_databases.consts import LOGGER_NAME
 
-POSTGRESQL_LOGGER_NAME = f'{LOGGER_NAME}.postgresql'
+POSTGRESQL_LOGGER_NAME = f"{LOGGER_NAME}.postgresql"

{python_databases-1.2.1.dist-info → python_databases-2.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-databases
-Version: 1.2.1
+Version: 2.0.0
 Summary: A Python library for managing and interacting with multiple databases seamlessly.
 Author: Avi Zaguri
 License: MIT
@@ -16,16 +16,12 @@ Classifier: Topic :: System :: Logging
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: custom-python-logger>=4.0.0
+Requires-Dist: custom-python-logger>=4.0.2
 Requires-Dist: elasticsearch<9
 Requires-Dist: pandas>=2.3.3
-Requires-Dist: pre-commit>=4.5.0
 Requires-Dist: psycopg2-binary>=2.9.11
-Requires-Dist: pytest>=9.0.1
 Requires-Dist: python-dotenv>=1.2.1
 Requires-Dist: retrying>=1.4.2
-Requires-Dist: setuptools>=80.9.0
-Requires-Dist: wheel>=0.45.1
 Dynamic: license-file
 
 ![PyPI version](https://img.shields.io/pypi/v/python-databases)

{python_databases-1.2.1.dist-info → python_databases-2.0.0.dist-info}/RECORD

@@ -1,13 +1,13 @@
 python_databases/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 python_databases/consts.py,sha256=ADn3PmhTXKBobtiVgDR4Tqq2Ncn_FTK0WjgREOLdBPs,33
 python_databases/elasticsearch_infrastructure/__init__.py,sha256=U1orkPi0Pg-1rtY1CPSF4-eVRfrMr36qpT9VNKjBUw4,297
-python_databases/elasticsearch_infrastructure/consts.py,sha256=tW4jgeUsf3FhqEpNHZQr3SjqPbuJjcWQDsSqxAvUYPg,98
-python_databases/elasticsearch_infrastructure/elasticsearch.py,sha256=phbcAzM1HGXGl9AKk693IFNTrd7Mr1l-TSBsa8gvXbI,10756
+python_databases/elasticsearch_infrastructure/consts.py,sha256=MBZL90yjYUmipayzj65-XEBS-4qTRhClGatnEwy7OfA,98
+python_databases/elasticsearch_infrastructure/elasticsearch.py,sha256=GtWDvOpwoUCq6zvml6KFi9Xxlu34bKZTRZjZYlZmggw,11071
 python_databases/postgresql_infrastructure/__init__.py,sha256=yBjq7SLnNgnxW0DCQHEwCi_FrCNtjsJGJXIq7XtkUZg,150
-python_databases/postgresql_infrastructure/consts.py,sha256=Fz19Qj-B9mPCTGInNygm2CPmY0nWiamWeEMrJsv-XB4,102
+python_databases/postgresql_infrastructure/consts.py,sha256=1GPwiyRMyQJOgdz2np0H12cr9GrP9p3aR6APCli2Hs8,102
 python_databases/postgresql_infrastructure/postgresql.py,sha256=tFdQr5IOPYbuqKE3r9oqSoNoGMbED5NSEy1anrUNEyY,3105
-python_databases-1.2.1.dist-info/licenses/LICENSE,sha256=cSikHY6SZFsPZSBizCDAJ0-Bjjzxt-JtX6TVbKxwimo,1067
-python_databases-1.2.1.dist-info/METADATA,sha256=VLnVd7qQ_6K7KfYhFA__fcI0Z55wNP-yGqQGFLbDkvM,2463
-python_databases-1.2.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-python_databases-1.2.1.dist-info/top_level.txt,sha256=fXcMUntaYJfKeFS7FjQ6DY4WVXYBsgdrDmhOudYs-vg,17
-python_databases-1.2.1.dist-info/RECORD,,
+python_databases-2.0.0.dist-info/licenses/LICENSE,sha256=cSikHY6SZFsPZSBizCDAJ0-Bjjzxt-JtX6TVbKxwimo,1067
+python_databases-2.0.0.dist-info/METADATA,sha256=-lJYYQ9x82y0A5k1ISRlVTrMP8V8PKn3ml8niTaDfaY,2338
+python_databases-2.0.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+python_databases-2.0.0.dist-info/top_level.txt,sha256=fXcMUntaYJfKeFS7FjQ6DY4WVXYBsgdrDmhOudYs-vg,17
+python_databases-2.0.0.dist-info/RECORD,,