python-databases 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

python_databases/elasticsearch_infrastructure/__init__.py

@@ -1,17 +1,12 @@
 from dotenv import load_dotenv
 
 from python_databases.elasticsearch_infrastructure.elasticsearch import (
-    UrlProtocol,
     ElasticSearch,
+    ElasticSearchCloud,
     ElasticSearchOnPrem,
-    ElasticSearchCloud
+    UrlProtocol,
 )
 
 load_dotenv()
 
-__all__ = [
-    'UrlProtocol',
-    'ElasticSearch',
-    'ElasticSearchOnPrem',
-    'ElasticSearchCloud'
-]
+__all__ = ["UrlProtocol", "ElasticSearch", "ElasticSearchOnPrem", "ElasticSearchCloud"]

python_databases/elasticsearch_infrastructure/elasticsearch.py

@@ -2,9 +2,10 @@ import logging
 import time
 from abc import ABC, abstractmethod
 from copy import deepcopy
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from enum import Enum
-from typing import Optional, Any
+from typing import Any
+
 import pandas as pd
 from custom_python_logger import get_logger
 from elasticsearch import Elasticsearch, helpers
@@ -12,20 +13,20 @@ from retrying import retry
 
 
 class UrlProtocol(Enum):
-    HTTP = 'http'
-    HTTPS = 'https'
+    HTTP = "http"
+    HTTPS = "https"
 
 
 class ElasticSearch(ABC):
     def __init__(
         self,
         elk_hostname: str,
-        elasticsearch_port: Optional[int],
-        kibana_port: Optional[int],
+        elasticsearch_port: int | None,
+        kibana_port: int | None,
         protocol: UrlProtocol,
-        username: Optional[str],
-        password: Optional[str]
-    ):
+        username: str | None,
+        password: str | None,
+    ) -> None:
         self.logger = get_logger(self.__class__.__name__)
         self._change_elasticsearch_logger()
 
@@ -36,69 +37,65 @@ class ElasticSearch(ABC):
 
         self.elasticsearch_port = elasticsearch_port
         if not self.elasticsearch_port:
-            self.elasticsearch_url = f'{self.protocol.value}://{self.elk_hostname}'
+            self.elasticsearch_url = f"{self.protocol.value}://{self.elk_hostname}"
         else:
-            self.elasticsearch_url = f'{self.protocol.value}://{self.elk_hostname}:{self.elasticsearch_port}'
+            self.elasticsearch_url = f"{self.protocol.value}://{self.elk_hostname}:{self.elasticsearch_port}"
         self.kibana_port = kibana_port
 
         self.elk_client = None
 
     @abstractmethod
-    def connect_to_elasticsearch(self):
+    def connect_to_elasticsearch(self) -> None:
         pass
 
     @staticmethod
     def _change_elasticsearch_logger(debug_level: int = logging.CRITICAL) -> None:
-        tracer = get_logger('elasticsearch')
+        tracer = get_logger("elasticsearch")
         tracer.setLevel(debug_level)
 
     def _set_list_of_docs_quick(self, list_of_docs: list[dict], request_timeout: int = 600) -> None:
         if failed_response := helpers.streaming_bulk(
-                self.elk_client,
-                list_of_docs,
-                raise_on_error=False,
-                request_timeout=request_timeout,
-                chunk_size=1000
+            self.elk_client, list_of_docs, raise_on_error=False, request_timeout=request_timeout, chunk_size=1000
         ):
             for item in failed_response:
-                if item[1]['index']['status'] != 201 or item[1]['index']['_shards']['failed'] > 0:
-                    self.logger.debug(f'Failed document: \n{item}')
+                if item[1]["index"]["status"] != 201 or item[1]["index"]["_shards"]["failed"] > 0:
+                    self.logger.debug(f"Failed document: \n{item}")
 
     def _set_list_of_docs_safe(self, list_of_docs: list[dict]) -> None:
         success_response, failed_response = helpers.bulk(self.elk_client, list_of_docs)
         if success_response == len(list_of_docs):
-            self.logger.info('All documents were reported successfully')
+            self.logger.info("All documents were reported successfully")
         else:
             self.logger.error(
-                f'Not all documents were reported successfully.\n'
-                f'The documents that were not reported successfully: {failed_response}'
+                f"Not all documents were reported successfully.\n"
+                f"The documents that were not reported successfully: {failed_response}"
             )
 
     def post_list_of_docs(self, list_of_docs: list[dict], request_timeout: int = 600, quick: bool = False) -> None:
-        self.logger.info('Start to report the documents to ELK')
+        self.logger.info("Start to report the documents to ELK")
 
         if quick:
             self._set_list_of_docs_quick(list_of_docs=list_of_docs, request_timeout=request_timeout)
         else:
             self._set_list_of_docs_safe(list_of_docs=list_of_docs)
-        self.logger.info('Finish to report the documents to ELK')
+        self.logger.info("Finish to report the documents to ELK")
 
     @staticmethod
     def _get_doc_with_basic_info() -> dict:
         doc = {}
 
-        timestamp = datetime.now(tz=timezone.utc)
+        timestamp = datetime.now(tz=UTC)
         date_and_time_str = timestamp.isoformat()
-        date = date_and_time_str.split('T')[0]
-        time_part = date_and_time_str.split('T')[1].split('+')[0]
+        date = date_and_time_str.split("T")[0]
+        time_part = date_and_time_str.split("T")[1].split("+")[0]
 
         doc.update(
             {
-                'doc_id': timestamp.strftime('%Y%m%dT%H%M%S%f'),
-                'timestamp': timestamp,
-                'date_and_time_str': date_and_time_str,
-                'date_str': date,
-                'time_str': time_part,
+                "doc_id": timestamp.strftime("%Y%m%dT%H%M%S%f"),
+                "timestamp": timestamp,
+                "date_and_time_str": date_and_time_str,
+                "date_str": date,
+                "time_str": time_part,
             }
         )
         return doc
@@ -106,10 +103,10 @@ class ElasticSearch(ABC):
     @staticmethod
     def _add_list_values_as_str(doc: dict, column: str, value: Any) -> None:
         """Add list values as both list and string representation to the provided doc"""
-        if type(value) is list and all(type(item) is str for item in value):
-            filtered_value = list(filter(lambda item: item is not None, value))
-            doc[f'{column}'] = filtered_value
-            doc[f'{column}_str'] = ', '.join(filtered_value)
+        if isinstance(value, list) and all(isinstance(item, str) for item in value):
+            filtered_value = [item for item in value if item is not None]
+            doc[f"{column}"] = filtered_value
+            doc[f"{column}_str"] = ", ".join(filtered_value)
 
     def _build_document(self, row: dict) -> dict:
         doc = self._get_doc_with_basic_info()
@@ -125,41 +122,27 @@ class ElasticSearch(ABC):
 
     @staticmethod
     def _fill_list_of_docs(index_name: str, doc: dict[str, Any]) -> dict:
-        return {
-                "_index": index_name,
-                "_source": deepcopy(doc)
-            }
+        return {"_index": index_name, "_source": deepcopy(doc)}
 
     def _prepare_documents_for_bulk(self, data: list[dict], doc_index_name: str) -> list[dict]:
         list_of_docs = []
         for index, row in enumerate(data):
-            self.logger.debug(f'index: {index + 1}/{len(data)}')
+            self.logger.debug(f"index: {index + 1}/{len(data)}")
             doc = self._build_document(row=row)
-            list_of_docs.append(
-                self._fill_list_of_docs(index_name=doc_index_name, doc=doc)
-            )
+            list_of_docs.append(self._fill_list_of_docs(index_name=doc_index_name, doc=doc))
         return list_of_docs
 
     def post_list_of_docs_as_bulk_chunk(
-        self,
-        list_of_docs: list[dict],
-        chunk_size=1000,
-        time_sleep: int = 60,
-        quick: bool = False
+        self, list_of_docs: list[dict], chunk_size: int = 1000, time_sleep: int = 60, quick: bool = False
     ) -> None:
         for i in range(0, len(list_of_docs), chunk_size):
-            self.logger.info(f'index: {i} - {i + chunk_size} / {len(list_of_docs)}')
-            chunk = list_of_docs[i:i + chunk_size]
+            self.logger.info(f"index: {i} - {i + chunk_size} / {len(list_of_docs)}")
+            chunk = list_of_docs[i : i + chunk_size]
             self.post_list_of_docs(list_of_docs=chunk, quick=quick)
         time.sleep(time_sleep)
 
     def fill_elk_index_as_bulk(  # post_list_of_docs_as_bulk
-        self,
-        data: list[dict],
-        doc_index_name: str,
-        chunk_size=1000,
-        time_sleep: int = 1,
-        quick: bool = True
+        self, data: list[dict], doc_index_name: str, chunk_size: int = 1000, time_sleep: int = 1, quick: bool = True
     ) -> None:  # use as the main function to send data to the elastic search
         list_of_docs = self._prepare_documents_for_bulk(
             data=data,
@@ -177,9 +160,8 @@ class ElasticSearch(ABC):
         if self.elk_client.indices.exists(index=index):
             self.logger.info(f"The index '{index}' exists.")
             return True
-        else:
-            self.logger.info(f"The index '{index}' does not exist.")
-            return False
+        self.logger.info(f"The index '{index}' does not exist.")
+        return False
 
     def delete_index(self, index: str) -> None:
         self.elk_client.indices.delete(index=index)
@@ -189,21 +171,20 @@ class ElasticSearch(ABC):
         if self.check_if_index_exists(index=index):
             self.delete_index(index=index)
             return True
-        else:
-            return False
+        return False
 
     def get_documents(self, index: str, query: dict) -> list[dict]:
         try:
             res = self.elk_client.search(index=index, body=query)["hits"]["hits"]
             return [doc["_source"] for doc in res]
         except Exception as e:
-            raise Exception(f"Failed to get documents from index '{index}': {str(e)}")
+            raise Exception(f"Failed to get documents from index '{index}': {str(e)}") from e
 
     def convert_dataframes_to_list_of_docs(self, dataframe: pd.DataFrame) -> list:
         data = []
         dataframe_values = dataframe.values.tolist()
         for index, value in enumerate(dataframe_values):
-            self.logger.debug(f'index: {index}/{len(dataframe_values)}')
+            self.logger.debug(f"index: {index}/{len(dataframe_values)}")
             data.append(value)
         return data
 
@@ -212,19 +193,19 @@ class ElasticSearchOnPrem(ElasticSearch):
     def __init__(
         self,
         elk_hostname: str,
-        elasticsearch_port: Optional[int] = 9200,
-        kibana_port: Optional[int] = 5602,
+        elasticsearch_port: int | None = 9200,
+        kibana_port: int | None = 5602,
         protocol: UrlProtocol = UrlProtocol.HTTPS,
-        username: Optional[str] = None,
-        password: Optional[str] = None
-    ):
+        username: str | None = None,
+        password: str | None = None,
+    ) -> None:
         super().__init__(
             elk_hostname=elk_hostname,
             elasticsearch_port=elasticsearch_port,
             kibana_port=kibana_port,
             protocol=protocol,
             username=username,
-            password=password
+            password=password,
         )
 
         self.logger = get_logger(self.__class__.__name__)
@@ -232,42 +213,41 @@ class ElasticSearchOnPrem(ElasticSearch):
         self.connect_to_elasticsearch()
 
     @retry(stop_max_attempt_number=3, wait_fixed=180000)
-    def connect_to_elasticsearch(self):
+    def connect_to_elasticsearch(self) -> None:
         if self.username and self.password:
             self.elk_client = Elasticsearch(
                 hosts=[self.elasticsearch_url],
                 http_auth=(self.username, self.password),
                 # ca_certs="/etc/elasticsearch/certs/http_ca.crt",
-                verify_certs=False
+                verify_certs=False,
             )
         else:
-            self.elk_client = Elasticsearch(hosts=[f'{self.elasticsearch_url}'])
+            self.elk_client = Elasticsearch(hosts=[f"{self.elasticsearch_url}"])
 
         if self.elk_client.ping():
             self.logger.info("Elasticsearch on-prem Connection Successful")
         else:
             self.logger.error("Elasticsearch on-prem Connection Failed")
-            raise Exception(
-                f'Failed to connect to Elasticsearch on-prem on {self.elasticsearch_url}')
+            raise Exception(f"Failed to connect to Elasticsearch on-prem on {self.elasticsearch_url}")
 
 
 class ElasticSearchCloud(ElasticSearch):
     def __init__(
         self,
         elk_hostname: str,
-        elasticsearch_port: Optional[int] = 9200,
-        kibana_port: Optional[int] = 5602,
+        elasticsearch_port: int | None = 9200,
+        kibana_port: int | None = 5602,
         protocol: UrlProtocol = UrlProtocol.HTTPS,
-        username: Optional[str] = None,
-        password: Optional[str] = None
-    ):
+        username: str | None = None,
+        password: str | None = None,
+    ) -> None:
         super().__init__(
             elk_hostname=elk_hostname,
             elasticsearch_port=elasticsearch_port,
             kibana_port=kibana_port,
             protocol=protocol,
             username=username,
-            password=password
+            password=password,
         )
 
         self.logger = get_logger(self.__class__.__name__)
@@ -275,12 +255,11 @@ class ElasticSearchCloud(ElasticSearch):
         self.connect_to_elasticsearch()
 
     @retry(stop_max_attempt_number=3, wait_fixed=180000)
-    def connect_to_elasticsearch(self):
+    def connect_to_elasticsearch(self) -> None:
         self.elk_client = Elasticsearch(cloud_id=self.elk_hostname, http_auth=(self.username, self.password))
 
         if self.elk_client.ping():
             self.logger.info("Elasticsearch cloud Connection Successful")
         else:
             self.logger.error("Elasticsearch cloud Connection Failed")
-            raise Exception(
-                f'Failed to connect to Elasticsearch cloud on {self.elk_hostname}')
+            raise Exception(f"Failed to connect to Elasticsearch cloud on {self.elk_hostname}")

python_databases/postgresql_infrastructure/__init__.py

@@ -4,4 +4,4 @@ from python_databases.postgresql_infrastructure.postgresql import PostgreSQL
 
 load_dotenv()
 
-__all__ = ['PostgreSQL']
+__all__ = ["PostgreSQL"]

python_databases/postgresql_infrastructure/postgresql.py

@@ -1,21 +1,21 @@
-import logging
 import os
-from typing import Optional, Union, Any
+from types import TracebackType
+from typing import Any
+
 import psycopg2
 from custom_python_logger import get_logger
 from psycopg2 import extras
-from retrying import retry
 
 
 class PostgreSQL:
     def __init__(
         self,
-        name: Optional[str] = None,
-        host: Optional[str] = None,
-        port: Optional[str] = None,
-        username: Optional[str] = None,
-        password: Optional[str] = None
-    ):
+        name: str | None = None,
+        host: str | None = None,
+        port: str | None = None,
+        username: str | None = None,
+        password: str | None = None,
+    ) -> None:
         self.logger = get_logger(self.__class__.__name__)
 
         self.name = name or os.getenv("POSTGRESQL_NAME")
@@ -30,21 +30,21 @@ class PostgreSQL:
         self.create_connection()
 
     @property
-    def client(self):
+    def client(self) -> psycopg2.extensions.connection:
         if not self._client or self._client.closed:
             self.logger.warning("Re-establishing PostgreSQL connection")
             self.create_connection()
         return self._client
 
     @property
-    def cursor(self):
+    def cursor(self) -> psycopg2.extensions.cursor:
         if not self._cursor or self._cursor.closed:
             self.logger.warning("Re-creating PostgreSQL cursor")
             self.create_connection()
         return self._cursor
 
     # @retry(stop_max_attempt_number=3, wait_fixed=180000)
-    def create_connection(self):
+    def create_connection(self) -> None:
         self.logger.info(f"Create connection to {self.host}:{self.port}")
 
         self._client = psycopg2.connect(
@@ -56,15 +56,15 @@ class PostgreSQL:
         )
         self._cursor = self._client.cursor(cursor_factory=extras.DictCursor)
 
-    def execute_query(self, query: str, params: Optional[Union[tuple, dict[str, Any]]] = None) -> None:
+    def execute_query(self, query: str, params: tuple | dict[str, Any] | None = None) -> None:
         self.logger.debug(f"Fetch rows from DB by query {query} with params {params}")
         try:
             self.cursor.execute(query, params)
         except psycopg2.ProgrammingError as e:
             raise psycopg2.ProgrammingError(f"Error executing query: {query}") from e
-        self.logger.debug(f"Finished to fetch rows by query")
+        self.logger.debug("Finished to fetch rows by query")
 
-    def fetch_all(self, query: str, params: Optional[Union[tuple, dict[str, Any]]] = None) -> list[dict[str, Any]]:
+    def fetch_all(self, query: str, params: tuple | dict[str, Any] | None = None) -> list[dict[str, Any]]:
         self.execute_query(query, params)
         try:
             results = self.cursor.fetchall()
@@ -73,12 +73,17 @@ class PostgreSQL:
         self.logger.debug(f"Fetched {len(results)} rows")
         return results
 
-    def close(self):
+    def close(self) -> None:
         self.cursor.close()
         self.client.close()
 
-    def __enter__(self):
+    def __enter__(self) -> "PostgreSQL":
         return self
 
-    def __exit__(self, exc_type, exc_val, exc_tb):
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
         self.close()

{python_databases-0.1.5.dist-info → python_databases-0.1.7.dist-info}/METADATA

@@ -1,32 +1,32 @@
 Metadata-Version: 2.4
 Name: python-databases
-Version: 0.1.5
+Version: 0.1.7
 Summary: A Python library for managing and interacting with multiple databases seamlessly.
-Home-page: https://github.com/aviz92/python-databases
 Author: Avi Zaguri
-Author-email: 
+License: MIT
+Project-URL: Homepage, https://github.com/aviz92/python-databases
 Project-URL: Repository, https://github.com/aviz92/python-databases
+Project-URL: Issues, https://github.com/aviz92/python-databases/issues
+Keywords: development,databases,ORM,SQL,NoSQL,database management
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: System :: Logging
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: setuptools
-Requires-Dist: wheel
-Requires-Dist: dotenv
-Requires-Dist: custom-python-logger
-Requires-Dist: retrying
+Requires-Dist: custom-python-logger>=2.0.10
 Requires-Dist: elasticsearch<9
-Requires-Dist: pandas
-Requires-Dist: psycopg2-binary
-Requires-Dist: pytest
-Dynamic: author
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: pre-commit>=4.5.0
+Requires-Dist: psycopg2-binary>=2.9.11
+Requires-Dist: pytest>=9.0.1
+Requires-Dist: python-dotenv>=1.2.1
+Requires-Dist: retrying>=1.4.2
+Requires-Dist: setuptools>=80.9.0
+Requires-Dist: wheel>=0.45.1
 Dynamic: license-file
-Dynamic: project-url
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary
 
 # Python Databases
 A flexible and extensible Python library that provides a unified interface for interacting with multiple types of databases.

python_databases-0.1.7.dist-info/RECORD

@@ -0,0 +1,10 @@
+python_databases/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+python_databases/elasticsearch_infrastructure/__init__.py,sha256=U1orkPi0Pg-1rtY1CPSF4-eVRfrMr36qpT9VNKjBUw4,297
+python_databases/elasticsearch_infrastructure/elasticsearch.py,sha256=qIimoK4ykU38-pDlfqxgEpzsyXwZBP3ld5qXZE5ewb4,9989
+python_databases/postgresql_infrastructure/__init__.py,sha256=yBjq7SLnNgnxW0DCQHEwCi_FrCNtjsJGJXIq7XtkUZg,150
+python_databases/postgresql_infrastructure/postgresql.py,sha256=z1yBG-80Hj67mjDVlPvdIxfT5vl4IlbQE0rk2UEE4wY,3020
+python_databases-0.1.7.dist-info/licenses/LICENSE,sha256=cSikHY6SZFsPZSBizCDAJ0-Bjjzxt-JtX6TVbKxwimo,1067
+python_databases-0.1.7.dist-info/METADATA,sha256=iRGwbOsR4lJX2A5DCpFJqGx0eWMLfTIFhWydWeIC9BM,2087
+python_databases-0.1.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+python_databases-0.1.7.dist-info/top_level.txt,sha256=fXcMUntaYJfKeFS7FjQ6DY4WVXYBsgdrDmhOudYs-vg,17
+python_databases-0.1.7.dist-info/RECORD,,

python_databases-0.1.5.dist-info/RECORD

@@ -1,10 +0,0 @@
-python_databases/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-python_databases/elasticsearch_infrastructure/__init__.py,sha256=cjs95C1yBZpc3kEdkcXEamDvQGsO6ob5r5ttFCJfDaA,314
-python_databases/elasticsearch_infrastructure/elasticsearch.py,sha256=A5NwzqMIcAyuMLhVIlbpoIYO18jQFqWmtaZ7XyeLzzw,10259
-python_databases/postgresql_infrastructure/__init__.py,sha256=h1eJx0ogbtcF6UnN2c7uW5iCuadDab3So1VI15y3kWA,150
-python_databases/postgresql_infrastructure/postgresql.py,sha256=zH8y5w2k2djhHLefDJhlGwa6MtoZKONhJ5vMxIxDdlc,2856
-python_databases-0.1.5.dist-info/licenses/LICENSE,sha256=cSikHY6SZFsPZSBizCDAJ0-Bjjzxt-JtX6TVbKxwimo,1067
-python_databases-0.1.5.dist-info/METADATA,sha256=SccVOpAUU_gPpKNiHgZO2DANGUgjFw7pGcpf59hgUiA,1792
-python_databases-0.1.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-python_databases-0.1.5.dist-info/top_level.txt,sha256=fXcMUntaYJfKeFS7FjQ6DY4WVXYBsgdrDmhOudYs-vg,17
-python_databases-0.1.5.dist-info/RECORD,,