python-databases 0.1.4__tar.gz → 0.1.6__tar.gz

{python_databases-0.1.4/python_databases.egg-info → python_databases-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-databases
-Version: 0.1.4
+Version: 0.1.6
 Summary: A Python library for managing and interacting with multiple databases seamlessly.
 Home-page: https://github.com/aviz92/python-databases
 Author: Avi Zaguri
@@ -11,7 +11,8 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: setuptools
 Requires-Dist: wheel
-Requires-Dist: dotenv
+Requires-Dist: python-dotenv
+Requires-Dist: pre-commit
 Requires-Dist: custom-python-logger
 Requires-Dist: retrying
 Requires-Dist: elasticsearch<9

{python_databases-0.1.4 → python_databases-0.1.6}/python_databases/elasticsearch_infrastructure/__init__.py

@@ -1,12 +1,12 @@
 from dotenv import load_dotenv
 
 from python_databases.elasticsearch_infrastructure.elasticsearch import (
-    UrlProtocol,
     ElasticSearch,
+    ElasticSearchCloud,
     ElasticSearchOnPrem,
-    ElasticSearchCloud
+    UrlProtocol,
 )
 
 load_dotenv()
 
-__all__ = ['UrlProtocol', 'ElasticSearch', 'ElasticSearchOnPrem', 'ElasticSearchCloud']
+__all__ = ["UrlProtocol", "ElasticSearch", "ElasticSearchOnPrem", "ElasticSearchCloud"]

python_databases-0.1.6/python_databases/elasticsearch_infrastructure/elasticsearch.py

@@ -0,0 +1,265 @@
+import logging
+import time
+from abc import ABC, abstractmethod
+from copy import deepcopy
+from datetime import UTC, datetime
+from enum import Enum
+from typing import Any
+
+import pandas as pd
+from custom_python_logger import get_logger
+from elasticsearch import Elasticsearch, helpers
+from retrying import retry
+
+
+class UrlProtocol(Enum):
+    HTTP = "http"
+    HTTPS = "https"
+
+
+class ElasticSearch(ABC):
+    def __init__(
+        self,
+        elk_hostname: str,
+        elasticsearch_port: int | None,
+        kibana_port: int | None,
+        protocol: UrlProtocol,
+        username: str | None,
+        password: str | None,
+    ) -> None:
+        self.logger = get_logger(self.__class__.__name__)
+        self._change_elasticsearch_logger()
+
+        self.protocol = protocol
+        self.elk_hostname = elk_hostname
+        self.username = username
+        self.password = password
+
+        self.elasticsearch_port = elasticsearch_port
+        if not self.elasticsearch_port:
+            self.elasticsearch_url = f"{self.protocol.value}://{self.elk_hostname}"
+        else:
+            self.elasticsearch_url = f"{self.protocol.value}://{self.elk_hostname}:{self.elasticsearch_port}"
+        self.kibana_port = kibana_port
+
+        self.elk_client = None
+
+    @abstractmethod
+    def connect_to_elasticsearch(self) -> None:
+        pass
+
+    @staticmethod
+    def _change_elasticsearch_logger(debug_level: int = logging.CRITICAL) -> None:
+        tracer = get_logger("elasticsearch")
+        tracer.setLevel(debug_level)
+
+    def _set_list_of_docs_quick(self, list_of_docs: list[dict], request_timeout: int = 600) -> None:
+        if failed_response := helpers.streaming_bulk(
+            self.elk_client, list_of_docs, raise_on_error=False, request_timeout=request_timeout, chunk_size=1000
+        ):
+            for item in failed_response:
+                if item[1]["index"]["status"] != 201 or item[1]["index"]["_shards"]["failed"] > 0:
+                    self.logger.debug(f"Failed document: \n{item}")
+
+    def _set_list_of_docs_safe(self, list_of_docs: list[dict]) -> None:
+        success_response, failed_response = helpers.bulk(self.elk_client, list_of_docs)
+        if success_response == len(list_of_docs):
+            self.logger.info("All documents were reported successfully")
+        else:
+            self.logger.error(
+                f"Not all documents were reported successfully.\n"
+                f"The documents that were not reported successfully: {failed_response}"
+            )
+
+    def post_list_of_docs(self, list_of_docs: list[dict], request_timeout: int = 600, quick: bool = False) -> None:
+        self.logger.info("Start to report the documents to ELK")
+
+        if quick:
+            self._set_list_of_docs_quick(list_of_docs=list_of_docs, request_timeout=request_timeout)
+        else:
+            self._set_list_of_docs_safe(list_of_docs=list_of_docs)
+        self.logger.info("Finish to report the documents to ELK")
+
+    @staticmethod
+    def _get_doc_with_basic_info() -> dict:
+        doc = {}
+
+        timestamp = datetime.now(tz=UTC)
+        date_and_time_str = timestamp.isoformat()
+        date = date_and_time_str.split("T")[0]
+        time_part = date_and_time_str.split("T")[1].split("+")[0]
+
+        doc.update(
+            {
+                "doc_id": timestamp.strftime("%Y%m%dT%H%M%S%f"),
+                "timestamp": timestamp,
+                "date_and_time_str": date_and_time_str,
+                "date_str": date,
+                "time_str": time_part,
+            }
+        )
+        return doc
+
+    @staticmethod
+    def _add_list_values_as_str(doc: dict, column: str, value: Any) -> None:
+        """Add list values as both list and string representation to the provided doc"""
+        if isinstance(value, list) and all(isinstance(item, str) for item in value):
+            filtered_value = [item for item in value if item is not None]
+            doc[f"{column}"] = filtered_value
+            doc[f"{column}_str"] = ", ".join(filtered_value)
+
+    def _build_document(self, row: dict) -> dict:
+        doc = self._get_doc_with_basic_info()
+        for custom_field, value in row.items():
+            doc[custom_field] = value
+
+            self._add_list_values_as_str(
+                doc=doc,
+                column=custom_field,
+                value=value,
+            )
+        return doc
+
+    @staticmethod
+    def _fill_list_of_docs(index_name: str, doc: dict[str, Any]) -> dict:
+        return {"_index": index_name, "_source": deepcopy(doc)}
+
+    def _prepare_documents_for_bulk(self, data: list[dict], doc_index_name: str) -> list[dict]:
+        list_of_docs = []
+        for index, row in enumerate(data):
+            self.logger.debug(f"index: {index + 1}/{len(data)}")
+            doc = self._build_document(row=row)
+            list_of_docs.append(self._fill_list_of_docs(index_name=doc_index_name, doc=doc))
+        return list_of_docs
+
+    def post_list_of_docs_as_bulk_chunk(
+        self, list_of_docs: list[dict], chunk_size: int = 1000, time_sleep: int = 60, quick: bool = False
+    ) -> None:
+        for i in range(0, len(list_of_docs), chunk_size):
+            self.logger.info(f"index: {i} - {i + chunk_size} / {len(list_of_docs)}")
+            chunk = list_of_docs[i : i + chunk_size]
+            self.post_list_of_docs(list_of_docs=chunk, quick=quick)
+        time.sleep(time_sleep)
+
+    def fill_elk_index_as_bulk(  # post_list_of_docs_as_bulk
+        self, data: list[dict], doc_index_name: str, chunk_size: int = 1000, time_sleep: int = 1, quick: bool = True
+    ) -> None:  # use as the main function to send data to the elastic search
+        list_of_docs = self._prepare_documents_for_bulk(
+            data=data,
+            doc_index_name=doc_index_name,
+        )
+
+        self.post_list_of_docs_as_bulk_chunk(
+            list_of_docs=list_of_docs,
+            chunk_size=chunk_size,
+            time_sleep=time_sleep,
+            quick=quick,
+        )
+
+    def check_if_index_exists(self, index: str) -> bool:
+        if self.elk_client.indices.exists(index=index):
+            self.logger.info(f"The index '{index}' exists.")
+            return True
+        self.logger.info(f"The index '{index}' does not exist.")
+        return False
+
+    def delete_index(self, index: str) -> None:
+        self.elk_client.indices.delete(index=index)
+        self.logger.info(f"The index '{index}' was deleted.")
+
+    def check_if_index_exists_and_delete_if_exists(self, index: str) -> bool:
+        if self.check_if_index_exists(index=index):
+            self.delete_index(index=index)
+            return True
+        return False
+
+    def get_documents(self, index: str, query: dict) -> list[dict]:
+        try:
+            res = self.elk_client.search(index=index, body=query)["hits"]["hits"]
+            return [doc["_source"] for doc in res]
+        except Exception as e:
+            raise Exception(f"Failed to get documents from index '{index}': {str(e)}") from e
+
+    def convert_dataframes_to_list_of_docs(self, dataframe: pd.DataFrame) -> list:
+        data = []
+        dataframe_values = dataframe.values.tolist()
+        for index, value in enumerate(dataframe_values):
+            self.logger.debug(f"index: {index}/{len(dataframe_values)}")
+            data.append(value)
+        return data
+
+
+class ElasticSearchOnPrem(ElasticSearch):
+    def __init__(
+        self,
+        elk_hostname: str,
+        elasticsearch_port: int | None = 9200,
+        kibana_port: int | None = 5602,
+        protocol: UrlProtocol = UrlProtocol.HTTPS,
+        username: str | None = None,
+        password: str | None = None,
+    ) -> None:
+        super().__init__(
+            elk_hostname=elk_hostname,
+            elasticsearch_port=elasticsearch_port,
+            kibana_port=kibana_port,
+            protocol=protocol,
+            username=username,
+            password=password,
+        )
+
+        self.logger = get_logger(self.__class__.__name__)
+
+        self.connect_to_elasticsearch()
+
+    @retry(stop_max_attempt_number=3, wait_fixed=180000)
+    def connect_to_elasticsearch(self) -> None:
+        if self.username and self.password:
+            self.elk_client = Elasticsearch(
+                hosts=[self.elasticsearch_url],
+                http_auth=(self.username, self.password),
+                # ca_certs="/etc/elasticsearch/certs/http_ca.crt",
+                verify_certs=False,
+            )
+        else:
+            self.elk_client = Elasticsearch(hosts=[f"{self.elasticsearch_url}"])
+
+        if self.elk_client.ping():
+            self.logger.info("Elasticsearch on-prem Connection Successful")
+        else:
+            self.logger.error("Elasticsearch on-prem Connection Failed")
+            raise Exception(f"Failed to connect to Elasticsearch on-prem on {self.elasticsearch_url}")
+
+
+class ElasticSearchCloud(ElasticSearch):
+    def __init__(
+        self,
+        elk_hostname: str,
+        elasticsearch_port: int | None = 9200,
+        kibana_port: int | None = 5602,
+        protocol: UrlProtocol = UrlProtocol.HTTPS,
+        username: str | None = None,
+        password: str | None = None,
+    ) -> None:
+        super().__init__(
+            elk_hostname=elk_hostname,
+            elasticsearch_port=elasticsearch_port,
+            kibana_port=kibana_port,
+            protocol=protocol,
+            username=username,
+            password=password,
+        )
+
+        self.logger = get_logger(self.__class__.__name__)
+
+        self.connect_to_elasticsearch()
+
+    @retry(stop_max_attempt_number=3, wait_fixed=180000)
+    def connect_to_elasticsearch(self) -> None:
+        self.elk_client = Elasticsearch(cloud_id=self.elk_hostname, http_auth=(self.username, self.password))
+
+        if self.elk_client.ping():
+            self.logger.info("Elasticsearch cloud Connection Successful")
+        else:
+            self.logger.error("Elasticsearch cloud Connection Failed")
+            raise Exception(f"Failed to connect to Elasticsearch cloud on {self.elk_hostname}")

{python_databases-0.1.4 → python_databases-0.1.6}/python_databases/postgresql_infrastructure/__init__.py

@@ -4,4 +4,4 @@ from python_databases.postgresql_infrastructure.postgresql import PostgreSQL
 
 load_dotenv()
 
-__all__ = ['PostgreSQL']
+__all__ = ["PostgreSQL"]

{python_databases-0.1.4 → python_databases-0.1.6}/python_databases/postgresql_infrastructure/postgresql.py

@@ -1,21 +1,21 @@
-import logging
 import os
-from typing import Optional, Union, Any
+from types import TracebackType
+from typing import Any
+
 import psycopg2
 from custom_python_logger import get_logger
 from psycopg2 import extras
-from retrying import retry
 
 
 class PostgreSQL:
     def __init__(
         self,
-        name: Optional[str] = None,
-        host: Optional[str] = None,
-        port: Optional[str] = None,
-        username: Optional[str] = None,
-        password: Optional[str] = None
-    ):
+        name: str | None = None,
+        host: str | None = None,
+        port: str | None = None,
+        username: str | None = None,
+        password: str | None = None,
+    ) -> None:
         self.logger = get_logger(self.__class__.__name__)
 
         self.name = name or os.getenv("POSTGRESQL_NAME")
@@ -30,21 +30,21 @@ class PostgreSQL:
         self.create_connection()
 
     @property
-    def client(self):
+    def client(self) -> psycopg2.extensions.connection:
         if not self._client or self._client.closed:
             self.logger.warning("Re-establishing PostgreSQL connection")
             self.create_connection()
         return self._client
 
     @property
-    def cursor(self):
+    def cursor(self) -> psycopg2.extensions.cursor:
         if not self._cursor or self._cursor.closed:
             self.logger.warning("Re-creating PostgreSQL cursor")
             self.create_connection()
         return self._cursor
 
     # @retry(stop_max_attempt_number=3, wait_fixed=180000)
-    def create_connection(self):
+    def create_connection(self) -> None:
         self.logger.info(f"Create connection to {self.host}:{self.port}")
 
         self._client = psycopg2.connect(
@@ -56,15 +56,15 @@ class PostgreSQL:
         )
         self._cursor = self._client.cursor(cursor_factory=extras.DictCursor)
 
-    def execute_query(self, query: str, params: Optional[Union[tuple, dict[str, Any]]] = None) -> None:
+    def execute_query(self, query: str, params: tuple | dict[str, Any] | None = None) -> None:
         self.logger.debug(f"Fetch rows from DB by query {query} with params {params}")
         try:
             self.cursor.execute(query, params)
         except psycopg2.ProgrammingError as e:
             raise psycopg2.ProgrammingError(f"Error executing query: {query}") from e
-        self.logger.debug(f"Finished to fetch rows by query")
+        self.logger.debug("Finished to fetch rows by query")
 
-    def fetch_all(self, query: str, params: Optional[Union[tuple, dict[str, Any]]] = None) -> list[dict[str, Any]]:
+    def fetch_all(self, query: str, params: tuple | dict[str, Any] | None = None) -> list[dict[str, Any]]:
         self.execute_query(query, params)
         try:
             results = self.cursor.fetchall()
@@ -73,12 +73,17 @@ class PostgreSQL:
         self.logger.debug(f"Fetched {len(results)} rows")
         return results
 
-    def close(self):
+    def close(self) -> None:
         self.cursor.close()
         self.client.close()
 
-    def __enter__(self):
+    def __enter__(self) -> "PostgreSQL":
         return self
 
-    def __exit__(self, exc_type, exc_val, exc_tb):
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
         self.close()

{python_databases-0.1.4 → python_databases-0.1.6/python_databases.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-databases
-Version: 0.1.4
+Version: 0.1.6
 Summary: A Python library for managing and interacting with multiple databases seamlessly.
 Home-page: https://github.com/aviz92/python-databases
 Author: Avi Zaguri
@@ -11,7 +11,8 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: setuptools
 Requires-Dist: wheel
-Requires-Dist: dotenv
+Requires-Dist: python-dotenv
+Requires-Dist: pre-commit
 Requires-Dist: custom-python-logger
 Requires-Dist: retrying
 Requires-Dist: elasticsearch<9

{python_databases-0.1.4 → python_databases-0.1.6}/python_databases.egg-info/requires.txt

@@ -1,6 +1,7 @@
 setuptools
 wheel
-dotenv
+python-dotenv
+pre-commit
 custom-python-logger
 retrying
 elasticsearch<9

{python_databases-0.1.4 → python_databases-0.1.6}/requirements.txt

@@ -1,6 +1,7 @@
 setuptools
 wheel
-dotenv
+python-dotenv
+pre-commit
 custom-python-logger
 retrying
 elasticsearch<9

python_databases-0.1.6/setup.py

@@ -0,0 +1,35 @@
+from setuptools import find_packages, setup
+
+package_version = "0.1.6"
+
+package_name = "python-databases"
+package_description = "A Python library for managing and interacting with multiple databases seamlessly."
+
+package_name_ = package_name.replace("-", "_")
+package_long_description_content_type = "text/markdown"
+package_url = f"https://github.com/aviz92/{package_name}"
+package_python_requires = ">=3.11"
+package_author = "Avi Zaguri"
+
+with open("requirements.txt") as file:
+    package_install_requires = [line.strip() for line in file.readlines() if line.strip() and not line.startswith("#")]
+
+with open("README.md") as file:
+    package_long_description = file.read()
+
+setup(
+    name=package_name,
+    version=package_version,
+    packages=find_packages(include=[package_name_, f"{package_name_}.*"]),
+    install_requires=package_install_requires,
+    author=package_author,
+    author_email="",
+    description=package_description,
+    long_description=package_long_description,
+    long_description_content_type=package_long_description_content_type,
+    url=package_url,
+    project_urls={
+        "Repository": package_url,
+    },
+    python_requires=package_python_requires,
+)

{python_databases-0.1.4 → python_databases-0.1.6}/tests/test_postgres.py

@@ -1,7 +1,9 @@
 # test_postgresql.py
 
+from unittest.mock import MagicMock, patch
+
 import pytest
-from unittest.mock import patch, MagicMock
+
 from python_databases.postgresql_infrastructure import PostgreSQL
 
 
@@ -15,21 +17,21 @@ def mock_connection():
         yield mock_conn, mock_cursor
 
 
-def test_connection_created(mock_connection):
+def test_connection_created(mock_connection: MagicMock):  # pylint: disable=W0621
     conn, _ = mock_connection
     db = PostgreSQL(name="test_db", host="localhost", port="5432", username="user", password="pass")
     assert db.client == conn
     assert conn.cursor.call_count == 2
 
 
-def test_execute_query_success(mock_connection):
+def test_execute_query_success(mock_connection: MagicMock):  # pylint: disable=W0621
     _, cursor = mock_connection
     db = PostgreSQL(name="test_db", host="localhost", port="5432", username="user", password="pass")
     db.execute_query("SELECT 1", None)
     cursor.execute.assert_called_with("SELECT 1", None)
 
 
-def test_fetch_all_success(mock_connection):
+def test_fetch_all_success(mock_connection: MagicMock):  # pylint: disable=W0621
     _, cursor = mock_connection
     cursor.fetchall.return_value = [{"id": 1}]
     db = PostgreSQL(name="test_db", host="localhost", port="5432", username="user", password="pass")
@@ -38,9 +40,9 @@ def test_fetch_all_success(mock_connection):
     assert results == [{"id": 1}]
 
 
-def test_context_manager_closes_connection(mock_connection):
+def test_context_manager_closes_connection(mock_connection: MagicMock):  # pylint: disable=W0621
     conn, cursor = mock_connection
-    with PostgreSQL(name="test_db", host="localhost", port="5432", username="user", password="pass") as db:
+    with PostgreSQL(name="test_db", host="localhost", port="5432", username="user", password="pass") as _:
         pass
     cursor.close.assert_called_once()
     conn.close.assert_called_once()

python_databases-0.1.4/python_databases/elasticsearch_infrastructure/elasticsearch.py

@@ -1,320 +0,0 @@
-import logging
-import time
-from abc import ABC, abstractmethod
-from copy import deepcopy
-from datetime import datetime, timezone
-from enum import Enum
-from typing import Optional, Any
-import pandas as pd
-from custom_python_logger import get_logger
-from elasticsearch import Elasticsearch, helpers
-from retrying import retry
-
-
-class UrlProtocol(Enum):
-    HTTP = 'http'
-    HTTPS = 'https'
-
-
-class ElasticSearch(ABC):
-    def __init__(
-            self,
-            elk_hostname: str,
-            elasticsearch_port: Optional[int],
-            kibana_port: Optional[int],
-            protocol: UrlProtocol,
-            username: Optional[str],
-            password: Optional[str]
-    ):
-        self.logger = get_logger(self.__class__.__name__)
-        self._change_elasticsearch_logger()
-
-        self.protocol = protocol
-        self.elk_hostname = elk_hostname
-        self.username = username
-        self.password = password
-
-        self.elasticsearch_port = elasticsearch_port
-        if not self.elasticsearch_port:
-            self.elasticsearch_url = f'{self.protocol.value}://{self.elk_hostname}'
-        else:
-            self.elasticsearch_url = f'{self.protocol.value}://{self.elk_hostname}:{self.elasticsearch_port}'
-        self.kibana_port = kibana_port
-
-        self.elk_client = None
-
-    def _change_elasticsearch_logger(self):
-        tracer = get_logger('elasticsearch')
-        tracer.setLevel(logging.CRITICAL)  # or desired level
-        # tracer.addHandler(logging.FileHandler('indexer.log'))
-
-    @abstractmethod
-    def connect_to_elasticsearch(self):
-        pass
-
-    def set_list_of_docs(self, list_of_docs: list[dict], request_timeout: int = 600, quick: bool = True) -> None:
-        self.logger.info('Start to report the documents to ELK')
-
-        if quick:
-            if failed_response := helpers.streaming_bulk(
-                    self.elk_client,
-                    list_of_docs,
-                    raise_on_error=False,
-                    request_timeout=request_timeout,
-                    chunk_size=1000
-            ):
-                for item in failed_response:
-                    if item[1]['index']['status'] != 201 or item[1]['index']['_shards']['failed'] > 0:
-                        self.logger.debug(f'Failed document: \n{item}')
-        else:
-            success_response, failed_response = helpers.bulk(self.elk_client, list_of_docs)
-            if success_response == len(list_of_docs):
-                self.logger.info('All documents were reported successfully')
-            else:
-                self.logger.error(
-                    f'Not all documents were reported successfully.\n'
-                    f'The documents that were not reported successfully: {failed_response}'
-                )
-        self.logger.info('Finish to report the documents to ELK')
-
-    def fill_list_of_docs(  # noqa
-            self,
-            list_of_docs: list[dict],
-            index_name: str,
-            doc: dict[str, Any]
-    ) -> None:
-        list_of_docs.append(
-            {
-                "_index": index_name,
-                "_source": deepcopy(doc)
-            }
-        )
-
-    def _add_basic_info(
-            self,
-            doc: dict,
-            doc_id: str,
-            timestamp: datetime,
-            date_and_time: str,
-            username: str = None
-    ) -> dict:
-        doc.update(
-            {
-                'doc_id': doc_id,
-                'timestamp': timestamp,
-                'date_str': f'{date_and_time}'.replace('-', '_')
-            }
-        )
-        if username:
-            doc['username'] = username
-        return doc
-
-    def _add_list_values_as_str(self, doc: dict, value: Any, column: str) -> None:
-        """Add list values as both list and string representation to the provided doc"""
-        if type(value) is list and all(type(item) is str for item in value):
-            filtered_value = list(filter(lambda item: item is not None, value))
-            doc[f'{column}'] = filtered_value
-            doc[f'{column}_str'] = ', '.join(filtered_value)
-
-    def _build_document(
-            self,
-            row: dict,
-            doc_id: str,
-            timestamp: datetime,
-            date_and_time: str,
-            username: str = None
-    ) -> dict:
-        doc = self._add_basic_info(
-            doc={},
-            doc_id=doc_id,
-            timestamp=timestamp,
-            date_and_time=date_and_time,
-            username=username
-        )
-
-        # Add custom fields
-        for custom_field, value in row.items():
-            doc[custom_field] = value
-            self._add_list_values_as_str(
-                doc=doc,
-                value=value,
-                column=custom_field
-            )
-
-        return doc
-
-    def _prepare_documents_for_bulk(
-            self,
-            data: list[dict],
-            doc_id: str,
-            doc_index_name: str,
-            timestamp: datetime,
-            date_and_time: str,
-            username: str = None
-    ) -> list[dict]:
-        list_of_docs = []
-        for index, row in enumerate(data):
-            self.logger.debug(f'index: {index + 1}/{len(data)}')
-
-            doc = self._build_document(
-                row=row,
-                doc_id=doc_id,
-                timestamp=timestamp,
-                date_and_time=date_and_time,
-                username=username
-            )
-
-            self.fill_list_of_docs(
-                list_of_docs=list_of_docs,
-                index_name=doc_index_name,
-                doc=doc
-            )
-
-        return list_of_docs
-
-    def fill_elk_index_as_bulk_chunk(self, list_of_docs: list[dict], chunk_size=1000, time_sleep: int = 60) -> None:
-        for i in range(0, len(list_of_docs), chunk_size):
-            self.logger.info(f'index: {i} - {i + chunk_size} / {len(list_of_docs)}')
-            chunk = list_of_docs[i:i + chunk_size]
-            self.set_list_of_docs(list_of_docs=chunk)
-        time.sleep(time_sleep)
-
-    def fill_elk_index_as_bulk(
-            self,
-            data: list[dict],
-            doc_index_name: str,
-            doc_id: Optional[str] = None,
-            timestamp: datetime = None,
-            date_and_time: Optional[str] = None,
-            username: Optional[str] = None,
-            chunk_size=1000,
-            time_sleep: int = 1
-    ) -> None:  # use as the main function to send data to the elastic search
-        timestamp = timestamp or datetime.now(tz=timezone.utc)
-        list_of_docs = self._prepare_documents_for_bulk(
-            data=data,
-            doc_index_name=doc_index_name,
-            timestamp=timestamp,
-            doc_id=doc_id or timestamp.strftime('%Y_%m_%dT%H_%M_%S_%f'),
-            date_and_time=date_and_time or timestamp.isoformat(),
-            username=username
-        )
-
-        self.fill_elk_index_as_bulk_chunk(
-            list_of_docs=list_of_docs,
-            chunk_size=chunk_size,
-            time_sleep=time_sleep
-        )
-
-    def convert_dataframes_to_list_of_docs(self, dataframe: pd.DataFrame) -> list:
-        data = []
-        dataframe_values = dataframe.values.tolist()
-        for index, value in enumerate(dataframe_values):
-            self.logger.debug(f'index: {index}/{len(dataframe_values)}')
-            data.append(value)
-        return data
-
-    def check_if_index_exists(self, index: str) -> bool:
-        if self.elk_client.indices.exists(index=index):
-            self.logger.info(f"The index '{index}' exists.")
-            return True
-        else:
-            self.logger.info(f"The index '{index}' does not exist.")
-            return False
-
-    def delete_index(self, index: str) -> None:
-        self.elk_client.indices.delete(index=index)
-        self.logger.info(f"The index '{index}' was deleted.")
-
-    def check_if_index_exists_and_delete_if_exists(self, index: str) -> bool:
-        if self.check_if_index_exists(index=index):
-            self.delete_index(index=index)
-            return True
-        else:
-            self.logger.info(f"The index '{index}' does not exist.")
-            return False
-
-    def get_documents(self, index: str, query: dict) -> list[dict]:
-        try:
-            res = self.elk_client.search(index=index, body=query)["hits"]["hits"]
-            return [doc["_source"] for doc in res]
-        except Exception as e:
-            raise Exception(f"Failed to get documents from index '{index}': {str(e)}")
-
-
-class ElasticSearchOnPrem(ElasticSearch):
-    def __init__(
-            self,
-            elk_hostname: str,
-            elasticsearch_port: Optional[int] = 9200,
-            kibana_port: Optional[int] = 5602,
-            protocol: UrlProtocol = UrlProtocol.HTTPS,
-            username: Optional[str] = None,
-            password: Optional[str] = None
-    ):
-        super().__init__(
-            elk_hostname=elk_hostname,
-            elasticsearch_port=elasticsearch_port,
-            kibana_port=kibana_port,
-            protocol=protocol,
-            username=username,
-            password=password
-        )
-
-        self.logger = get_logger(self.__class__.__name__)
-
-        self.connect_to_elasticsearch()
-
-    @retry(stop_max_attempt_number=3, wait_fixed=180000)
-    def connect_to_elasticsearch(self):
-        if self.username and self.password:
-            self.elk_client = Elasticsearch(
-                hosts=[self.elasticsearch_url],
-                http_auth=(self.username, self.password),
-                # ca_certs="/etc/elasticsearch/certs/http_ca.crt",
-                verify_certs=False, timeout=60
-            )
-        else:
-            self.elk_client = Elasticsearch(hosts=[f'{self.elasticsearch_url}'], timeout=20)
-
-        if self.elk_client.ping():
-            self.logger.info("Elasticsearch on-prem Connection Successful")
-        else:
-            self.logger.error("Elasticsearch on-prem Connection Failed")
-            raise Exception(
-                f'Failed to connect to Elasticsearch on-prem on {self.elasticsearch_url}')
-
-
-class ElasticSearchCloud(ElasticSearch):
-    def __init__(
-            self,
-            elk_hostname: str,
-            elasticsearch_port: Optional[int] = 9200,
-            kibana_port: Optional[int] = 5602,
-            protocol: UrlProtocol = UrlProtocol.HTTPS,
-            username: Optional[str] = None,
-            password: Optional[str] = None
-    ):
-        super().__init__(
-            elk_hostname=elk_hostname,
-            elasticsearch_port=elasticsearch_port,
-            kibana_port=kibana_port,
-            protocol=protocol,
-            username=username,
-            password=password
-        )
-
-        self.logger = get_logger(self.__class__.__name__)
-
-        self.connect_to_elasticsearch()
-
-    @retry(stop_max_attempt_number=3, wait_fixed=180000)
-    def connect_to_elasticsearch(self):
-        self.elk_client = Elasticsearch(cloud_id=self.elk_hostname, http_auth=(self.username, self.password))
-
-        if self.elk_client.ping():
-            self.logger.info("Elasticsearch cloud Connection Successful")
-        else:
-            self.logger.error("Elasticsearch cloud Connection Failed")
-            raise Exception(
-                f'Failed to connect to Elasticsearch cloud on {self.elk_hostname}')

python_databases-0.1.4/setup.py

@@ -1,37 +0,0 @@
-from setuptools import setup, find_packages
-
-package_version = '0.1.4'
-
-package_name = 'python-databases'
-package_description = 'A Python library for managing and interacting with multiple databases seamlessly.'
-
-package_name_ = package_name.replace('-', '_')
-package_long_description_content_type = 'text/markdown'
-package_url = f'https://github.com/aviz92/{package_name}'
-package_python_requires = '>=3.11'
-package_author = 'Avi Zaguri'
-
-with open('requirements.txt', 'r') as file:
-    package_install_requires = [
-        line.strip() for line in file.readlines() if line.strip() and not line.startswith('#')
-    ]
-
-with open('README.md', 'r') as file:
-    package_long_description = file.read()
-
-setup(
-    name=package_name,
-    version=package_version,
-    packages=find_packages(include=[package_name_, f'{package_name_}.*']),
-    install_requires=package_install_requires,
-    author=package_author,
-    author_email='',
-    description=package_description,
-    long_description=package_long_description,
-    long_description_content_type=package_long_description_content_type,
-    url=package_url,
-    project_urls={
-        'Repository': package_url,
-    },
-    python_requires=package_python_requires,
-)