python-databases 1.0.0__tar.gz → 1.2.0__tar.gz

{python_databases-1.0.0/python_databases.egg-info → python_databases-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-databases
-Version: 1.0.0
+Version: 1.2.0
 Summary: A Python library for managing and interacting with multiple databases seamlessly.
 Author: Avi Zaguri
 License: MIT
@@ -10,13 +10,13 @@ Project-URL: Issues, https://github.com/aviz92/python-databases/issues
 Keywords: development,databases,ORM,SQL,NoSQL,database management
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
-Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: System :: Logging
-Requires-Python: >=3.11
+Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: custom-python-logger>=2.0.10
+Requires-Dist: custom-python-logger>=4.0.0
 Requires-Dist: elasticsearch<9
 Requires-Dist: pandas>=2.3.3
 Requires-Dist: pre-commit>=4.5.0

{python_databases-1.0.0 → python_databases-1.2.0}/pyproject.toml

@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "python-databases"
-version = "1.0.0"
+version = "1.2.0"
 description = "A Python library for managing and interacting with multiple databases seamlessly."
 readme = "README.md"
-requires-python = ">=3.11"
+requires-python = ">=3.12"
 license = { text = "MIT" }
 authors = [{ name = "Avi Zaguri" },]
 keywords = ["development", "databases", "ORM", "SQL", "NoSQL", "database management"]
@@ -15,13 +15,13 @@ keywords = ["development", "databases", "ORM", "SQL", "NoSQL", "database managem
 classifiers = [
     "Development Status :: 4 - Beta",
     "Intended Audience :: Developers",
-    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Operating System :: OS Independent",
     "Topic :: System :: Logging",
 ]
 
 dependencies = [
-    "custom-python-logger>=2.0.10",
+    "custom-python-logger>=4.0.0",
     "elasticsearch<9",
     "pandas>=2.3.3",
     "pre-commit>=4.5.0",

python_databases-1.2.0/python_databases/consts.py

@@ -0,0 +1 @@
+LOGGER_NAME = "python_databases"

python_databases-1.2.0/python_databases/elasticsearch_infrastructure/consts.py

@@ -0,0 +1,3 @@
+from python_databases.consts import LOGGER_NAME
+
+ELK_LOGGER_NAME = f'{LOGGER_NAME}.elasticsearch'

{python_databases-1.0.0 → python_databases-1.2.0}/python_databases/elasticsearch_infrastructure/elasticsearch.py

@@ -11,6 +11,8 @@ from custom_python_logger import get_logger
 from elasticsearch import Elasticsearch, helpers
 from retrying import retry
 
+from python_databases.elasticsearch_infrastructure.consts import ELK_LOGGER_NAME
+
 
 class UrlProtocol(Enum):
     HTTP = "http"
@@ -27,7 +29,7 @@ class ElasticSearch(ABC):
         username: str | None,
         password: str | None,
     ) -> None:
-        self.logger = get_logger(self.__class__.__name__)
+        self.logger = get_logger(ELK_LOGGER_NAME)
         self._change_elasticsearch_logger()
 
         self.protocol = protocol
@@ -53,13 +55,22 @@ class ElasticSearch(ABC):
         tracer = get_logger("elasticsearch")
         tracer.setLevel(debug_level)
 
-    def _set_list_of_docs_quick(self, list_of_docs: list[dict], request_timeout: int = 600) -> None:
+    def _set_list_of_docs_quick(
+        self,
+        list_of_docs: list[dict],
+        request_timeout: int = 600,
+        raise_on_error: bool = False,
+    ) -> None:
+        failed_response_list = []
         if failed_response := helpers.streaming_bulk(
-            self.elk_client, list_of_docs, raise_on_error=False, request_timeout=request_timeout, chunk_size=1000
+                self.elk_client, list_of_docs, raise_on_error=False, request_timeout=request_timeout, chunk_size=1000
         ):
             for item in failed_response:
                 if item[1]["index"]["status"] != 201 or item[1]["index"]["_shards"]["failed"] > 0:
                     self.logger.debug(f"Failed document: \n{item}")
+                    failed_response_list.append(item)
+        if raise_on_error and failed_response_list:
+            raise Exception(f"Failed to report {len(failed_response_list)} documents")
 
     def _set_list_of_docs_safe(self, list_of_docs: list[dict]) -> None:
         success_response, failed_response = helpers.bulk(self.elk_client, list_of_docs)
@@ -71,11 +82,21 @@ class ElasticSearch(ABC):
                 f"The documents that were not reported successfully: {failed_response}"
             )
 
-    def post_list_of_docs(self, list_of_docs: list[dict], request_timeout: int = 600, quick: bool = False) -> None:
+    def post_list_of_docs(
+        self,
+        list_of_docs: list[dict],
+        request_timeout: int = 600,
+        quick: bool = False,
+        raise_on_error: bool = False
+    ) -> None:
         self.logger.info("Start to report the documents to ELK")
 
         if quick:
-            self._set_list_of_docs_quick(list_of_docs=list_of_docs, request_timeout=request_timeout)
+            self._set_list_of_docs_quick(
+                list_of_docs=list_of_docs,
+                request_timeout=request_timeout,
+                raise_on_error=raise_on_error
+            )
         else:
             self._set_list_of_docs_safe(list_of_docs=list_of_docs)
         self.logger.info("Finish to report the documents to ELK")
@@ -133,16 +154,27 @@ class ElasticSearch(ABC):
         return list_of_docs
 
     def post_list_of_docs_as_bulk_chunk(
-        self, list_of_docs: list[dict], chunk_size: int = 1000, time_sleep: int = 60, quick: bool = False
+        self,
+        list_of_docs: list[dict],
+        chunk_size: int = 1000,
+        time_sleep: int = 60,
+        quick: bool = False,
+        raise_on_error: bool = False,
     ) -> None:
         for i in range(0, len(list_of_docs), chunk_size):
             self.logger.info(f"index: {i} - {i + chunk_size} / {len(list_of_docs)}")
-            chunk = list_of_docs[i : i + chunk_size]
-            self.post_list_of_docs(list_of_docs=chunk, quick=quick)
+            chunk = list_of_docs[i: i + chunk_size]
+            self.post_list_of_docs(list_of_docs=chunk, quick=quick, raise_on_error=raise_on_error)
         time.sleep(time_sleep)
 
     def fill_elk_index_as_bulk(  # post_list_of_docs_as_bulk
-        self, data: list[dict], doc_index_name: str, chunk_size: int = 1000, time_sleep: int = 1, quick: bool = True
+        self,
+        data: list[dict],
+        doc_index_name: str,
+        chunk_size: int = 1000,
+        time_sleep: int = 1,
+        quick: bool = True,
+        raise_on_error: bool = False,
     ) -> None:  # use as the main function to send data to the elastic search
         list_of_docs = self._prepare_documents_for_bulk(
             data=data,
@@ -154,6 +186,7 @@ class ElasticSearch(ABC):
             chunk_size=chunk_size,
             time_sleep=time_sleep,
             quick=quick,
+            raise_on_error=raise_on_error,
         )
 
     def check_if_index_exists(self, index: str) -> bool:
@@ -208,7 +241,7 @@ class ElasticSearchOnPrem(ElasticSearch):
             password=password,
         )
 
-        self.logger = get_logger(self.__class__.__name__)
+        self.logger = get_logger(ELK_LOGGER_NAME)
 
         self.connect_to_elasticsearch()
 
@@ -250,7 +283,7 @@ class ElasticSearchCloud(ElasticSearch):
             password=password,
         )
 
-        self.logger = get_logger(self.__class__.__name__)
+        self.logger = get_logger(ELK_LOGGER_NAME)
 
         self.connect_to_elasticsearch()
 

python_databases-1.2.0/python_databases/postgresql_infrastructure/consts.py

@@ -0,0 +1,3 @@
+from python_databases.consts import LOGGER_NAME
+
+POSTGRESQL_LOGGER_NAME = f'{LOGGER_NAME}.postgresql'

{python_databases-1.0.0 → python_databases-1.2.0}/python_databases/postgresql_infrastructure/postgresql.py

@@ -6,6 +6,8 @@ import psycopg2
 from custom_python_logger import get_logger
 from psycopg2 import extras
 
+from python_databases.postgresql_infrastructure.consts import POSTGRESQL_LOGGER_NAME
+
 
 class PostgreSQL:
     def __init__(
@@ -16,7 +18,7 @@ class PostgreSQL:
         username: str | None = None,
         password: str | None = None,
     ) -> None:
-        self.logger = get_logger(self.__class__.__name__)
+        self.logger = get_logger(POSTGRESQL_LOGGER_NAME)
 
         self.name = name or os.getenv("POSTGRESQL_NAME")
         self.host = host or os.getenv("POSTGRESQL_HOST")

{python_databases-1.0.0 → python_databases-1.2.0/python_databases.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-databases
-Version: 1.0.0
+Version: 1.2.0
 Summary: A Python library for managing and interacting with multiple databases seamlessly.
 Author: Avi Zaguri
 License: MIT
@@ -10,13 +10,13 @@ Project-URL: Issues, https://github.com/aviz92/python-databases/issues
 Keywords: development,databases,ORM,SQL,NoSQL,database management
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
-Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: System :: Logging
-Requires-Python: >=3.11
+Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: custom-python-logger>=2.0.10
+Requires-Dist: custom-python-logger>=4.0.0
 Requires-Dist: elasticsearch<9
 Requires-Dist: pandas>=2.3.3
 Requires-Dist: pre-commit>=4.5.0

{python_databases-1.0.0 → python_databases-1.2.0}/python_databases.egg-info/SOURCES.txt

@@ -3,13 +3,16 @@ MANIFEST.in
 README.md
 pyproject.toml
 python_databases/__init__.py
+python_databases/consts.py
 python_databases.egg-info/PKG-INFO
 python_databases.egg-info/SOURCES.txt
 python_databases.egg-info/dependency_links.txt
 python_databases.egg-info/requires.txt
 python_databases.egg-info/top_level.txt
 python_databases/elasticsearch_infrastructure/__init__.py
+python_databases/elasticsearch_infrastructure/consts.py
 python_databases/elasticsearch_infrastructure/elasticsearch.py
 python_databases/postgresql_infrastructure/__init__.py
+python_databases/postgresql_infrastructure/consts.py
 python_databases/postgresql_infrastructure/postgresql.py
 tests/test_postgres.py

{python_databases-1.0.0 → python_databases-1.2.0}/python_databases.egg-info/requires.txt

@@ -1,4 +1,4 @@
-custom-python-logger>=2.0.10
+custom-python-logger>=4.0.0
 elasticsearch<9
 pandas>=2.3.3
 pre-commit>=4.5.0