python-databases 0.1.4__py3-none-any.whl → 0.1.5__py3-none-any.whl

python_databases/elasticsearch_infrastructure/__init__.py

@@ -9,4 +9,9 @@ from python_databases.elasticsearch_infrastructure.elasticsearch import (
 
 load_dotenv()
 
-__all__ = ['UrlProtocol', 'ElasticSearch', 'ElasticSearchOnPrem', 'ElasticSearchCloud']
+__all__ = [
+    'UrlProtocol',
+    'ElasticSearch',
+    'ElasticSearchOnPrem',
+    'ElasticSearchCloud'
+]

python_databases/elasticsearch_infrastructure/elasticsearch.py

@@ -18,13 +18,13 @@ class UrlProtocol(Enum):
 
 class ElasticSearch(ABC):
     def __init__(
-            self,
-            elk_hostname: str,
-            elasticsearch_port: Optional[int],
-            kibana_port: Optional[int],
-            protocol: UrlProtocol,
-            username: Optional[str],
-            password: Optional[str]
+        self,
+        elk_hostname: str,
+        elasticsearch_port: Optional[int],
+        kibana_port: Optional[int],
+        protocol: UrlProtocol,
+        username: Optional[str],
+        password: Optional[str]
     ):
         self.logger = get_logger(self.__class__.__name__)
         self._change_elasticsearch_logger()
@@ -43,177 +43,136 @@ class ElasticSearch(ABC):
 
         self.elk_client = None
 
-    def _change_elasticsearch_logger(self):
-        tracer = get_logger('elasticsearch')
-        tracer.setLevel(logging.CRITICAL)  # or desired level
-        # tracer.addHandler(logging.FileHandler('indexer.log'))
-
     @abstractmethod
     def connect_to_elasticsearch(self):
         pass
 
-    def set_list_of_docs(self, list_of_docs: list[dict], request_timeout: int = 600, quick: bool = True) -> None:
+    @staticmethod
+    def _change_elasticsearch_logger(debug_level: int = logging.CRITICAL) -> None:
+        tracer = get_logger('elasticsearch')
+        tracer.setLevel(debug_level)
+
+    def _set_list_of_docs_quick(self, list_of_docs: list[dict], request_timeout: int = 600) -> None:
+        if failed_response := helpers.streaming_bulk(
+                self.elk_client,
+                list_of_docs,
+                raise_on_error=False,
+                request_timeout=request_timeout,
+                chunk_size=1000
+        ):
+            for item in failed_response:
+                if item[1]['index']['status'] != 201 or item[1]['index']['_shards']['failed'] > 0:
+                    self.logger.debug(f'Failed document: \n{item}')
+
+    def _set_list_of_docs_safe(self, list_of_docs: list[dict]) -> None:
+        success_response, failed_response = helpers.bulk(self.elk_client, list_of_docs)
+        if success_response == len(list_of_docs):
+            self.logger.info('All documents were reported successfully')
+        else:
+            self.logger.error(
+                f'Not all documents were reported successfully.\n'
+                f'The documents that were not reported successfully: {failed_response}'
+            )
+
+    def post_list_of_docs(self, list_of_docs: list[dict], request_timeout: int = 600, quick: bool = False) -> None:
         self.logger.info('Start to report the documents to ELK')
 
         if quick:
-            if failed_response := helpers.streaming_bulk(
-                    self.elk_client,
-                    list_of_docs,
-                    raise_on_error=False,
-                    request_timeout=request_timeout,
-                    chunk_size=1000
-            ):
-                for item in failed_response:
-                    if item[1]['index']['status'] != 201 or item[1]['index']['_shards']['failed'] > 0:
-                        self.logger.debug(f'Failed document: \n{item}')
+            self._set_list_of_docs_quick(list_of_docs=list_of_docs, request_timeout=request_timeout)
         else:
-            success_response, failed_response = helpers.bulk(self.elk_client, list_of_docs)
-            if success_response == len(list_of_docs):
-                self.logger.info('All documents were reported successfully')
-            else:
-                self.logger.error(
-                    f'Not all documents were reported successfully.\n'
-                    f'The documents that were not reported successfully: {failed_response}'
-                )
+            self._set_list_of_docs_safe(list_of_docs=list_of_docs)
         self.logger.info('Finish to report the documents to ELK')
 
-    def fill_list_of_docs(  # noqa
-            self,
-            list_of_docs: list[dict],
-            index_name: str,
-            doc: dict[str, Any]
-    ) -> None:
-        list_of_docs.append(
-            {
-                "_index": index_name,
-                "_source": deepcopy(doc)
-            }
-        )
+    @staticmethod
+    def _get_doc_with_basic_info() -> dict:
+        doc = {}
+
+        timestamp = datetime.now(tz=timezone.utc)
+        date_and_time_str = timestamp.isoformat()
+        date = date_and_time_str.split('T')[0]
+        time_part = date_and_time_str.split('T')[1].split('+')[0]
 
-    def _add_basic_info(
-            self,
-            doc: dict,
-            doc_id: str,
-            timestamp: datetime,
-            date_and_time: str,
-            username: str = None
-    ) -> dict:
         doc.update(
             {
-                'doc_id': doc_id,
+                'doc_id': timestamp.strftime('%Y%m%dT%H%M%S%f'),
                 'timestamp': timestamp,
-                'date_str': f'{date_and_time}'.replace('-', '_')
+                'date_and_time_str': date_and_time_str,
+                'date_str': date,
+                'time_str': time_part,
             }
         )
-        if username:
-            doc['username'] = username
         return doc
 
-    def _add_list_values_as_str(self, doc: dict, value: Any, column: str) -> None:
+    @staticmethod
+    def _add_list_values_as_str(doc: dict, column: str, value: Any) -> None:
         """Add list values as both list and string representation to the provided doc"""
         if type(value) is list and all(type(item) is str for item in value):
             filtered_value = list(filter(lambda item: item is not None, value))
             doc[f'{column}'] = filtered_value
             doc[f'{column}_str'] = ', '.join(filtered_value)
 
-    def _build_document(
-            self,
-            row: dict,
-            doc_id: str,
-            timestamp: datetime,
-            date_and_time: str,
-            username: str = None
-    ) -> dict:
-        doc = self._add_basic_info(
-            doc={},
-            doc_id=doc_id,
-            timestamp=timestamp,
-            date_and_time=date_and_time,
-            username=username
-        )
-
-        # Add custom fields
+    def _build_document(self, row: dict) -> dict:
+        doc = self._get_doc_with_basic_info()
         for custom_field, value in row.items():
             doc[custom_field] = value
+
             self._add_list_values_as_str(
                 doc=doc,
+                column=custom_field,
                 value=value,
-                column=custom_field
             )
-
         return doc
 
-    def _prepare_documents_for_bulk(
-            self,
-            data: list[dict],
-            doc_id: str,
-            doc_index_name: str,
-            timestamp: datetime,
-            date_and_time: str,
-            username: str = None
-    ) -> list[dict]:
+    @staticmethod
+    def _fill_list_of_docs(index_name: str, doc: dict[str, Any]) -> dict:
+        return {
+                "_index": index_name,
+                "_source": deepcopy(doc)
+            }
+
+    def _prepare_documents_for_bulk(self, data: list[dict], doc_index_name: str) -> list[dict]:
         list_of_docs = []
         for index, row in enumerate(data):
             self.logger.debug(f'index: {index + 1}/{len(data)}')
-
-            doc = self._build_document(
-                row=row,
-                doc_id=doc_id,
-                timestamp=timestamp,
-                date_and_time=date_and_time,
-                username=username
+            doc = self._build_document(row=row)
+            list_of_docs.append(
+                self._fill_list_of_docs(index_name=doc_index_name, doc=doc)
             )
-
-            self.fill_list_of_docs(
-                list_of_docs=list_of_docs,
-                index_name=doc_index_name,
-                doc=doc
-            )
-
         return list_of_docs
 
-    def fill_elk_index_as_bulk_chunk(self, list_of_docs: list[dict], chunk_size=1000, time_sleep: int = 60) -> None:
+    def post_list_of_docs_as_bulk_chunk(
+        self,
+        list_of_docs: list[dict],
+        chunk_size=1000,
+        time_sleep: int = 60,
+        quick: bool = False
+    ) -> None:
         for i in range(0, len(list_of_docs), chunk_size):
             self.logger.info(f'index: {i} - {i + chunk_size} / {len(list_of_docs)}')
             chunk = list_of_docs[i:i + chunk_size]
-            self.set_list_of_docs(list_of_docs=chunk)
+            self.post_list_of_docs(list_of_docs=chunk, quick=quick)
         time.sleep(time_sleep)
 
-    def fill_elk_index_as_bulk(
-            self,
-            data: list[dict],
-            doc_index_name: str,
-            doc_id: Optional[str] = None,
-            timestamp: datetime = None,
-            date_and_time: Optional[str] = None,
-            username: Optional[str] = None,
-            chunk_size=1000,
-            time_sleep: int = 1
+    def fill_elk_index_as_bulk(  # post_list_of_docs_as_bulk
+        self,
+        data: list[dict],
+        doc_index_name: str,
+        chunk_size=1000,
+        time_sleep: int = 1,
+        quick: bool = True
     ) -> None:  # use as the main function to send data to the elastic search
-        timestamp = timestamp or datetime.now(tz=timezone.utc)
         list_of_docs = self._prepare_documents_for_bulk(
             data=data,
             doc_index_name=doc_index_name,
-            timestamp=timestamp,
-            doc_id=doc_id or timestamp.strftime('%Y_%m_%dT%H_%M_%S_%f'),
-            date_and_time=date_and_time or timestamp.isoformat(),
-            username=username
         )
 
-        self.fill_elk_index_as_bulk_chunk(
+        self.post_list_of_docs_as_bulk_chunk(
             list_of_docs=list_of_docs,
             chunk_size=chunk_size,
-            time_sleep=time_sleep
+            time_sleep=time_sleep,
+            quick=quick,
         )
 
-    def convert_dataframes_to_list_of_docs(self, dataframe: pd.DataFrame) -> list:
-        data = []
-        dataframe_values = dataframe.values.tolist()
-        for index, value in enumerate(dataframe_values):
-            self.logger.debug(f'index: {index}/{len(dataframe_values)}')
-            data.append(value)
-        return data
-
     def check_if_index_exists(self, index: str) -> bool:
         if self.elk_client.indices.exists(index=index):
             self.logger.info(f"The index '{index}' exists.")
@@ -231,7 +190,6 @@ class ElasticSearch(ABC):
             self.delete_index(index=index)
             return True
         else:
-            self.logger.info(f"The index '{index}' does not exist.")
             return False
 
     def get_documents(self, index: str, query: dict) -> list[dict]:
@@ -241,16 +199,24 @@ class ElasticSearch(ABC):
         except Exception as e:
             raise Exception(f"Failed to get documents from index '{index}': {str(e)}")
 
+    def convert_dataframes_to_list_of_docs(self, dataframe: pd.DataFrame) -> list:
+        data = []
+        dataframe_values = dataframe.values.tolist()
+        for index, value in enumerate(dataframe_values):
+            self.logger.debug(f'index: {index}/{len(dataframe_values)}')
+            data.append(value)
+        return data
+
 
 class ElasticSearchOnPrem(ElasticSearch):
     def __init__(
-            self,
-            elk_hostname: str,
-            elasticsearch_port: Optional[int] = 9200,
-            kibana_port: Optional[int] = 5602,
-            protocol: UrlProtocol = UrlProtocol.HTTPS,
-            username: Optional[str] = None,
-            password: Optional[str] = None
+        self,
+        elk_hostname: str,
+        elasticsearch_port: Optional[int] = 9200,
+        kibana_port: Optional[int] = 5602,
+        protocol: UrlProtocol = UrlProtocol.HTTPS,
+        username: Optional[str] = None,
+        password: Optional[str] = None
     ):
         super().__init__(
             elk_hostname=elk_hostname,
@@ -272,10 +238,10 @@ class ElasticSearchOnPrem(ElasticSearch):
                 hosts=[self.elasticsearch_url],
                 http_auth=(self.username, self.password),
                 # ca_certs="/etc/elasticsearch/certs/http_ca.crt",
-                verify_certs=False, timeout=60
+                verify_certs=False
             )
         else:
-            self.elk_client = Elasticsearch(hosts=[f'{self.elasticsearch_url}'], timeout=20)
+            self.elk_client = Elasticsearch(hosts=[f'{self.elasticsearch_url}'])
 
         if self.elk_client.ping():
             self.logger.info("Elasticsearch on-prem Connection Successful")
@@ -287,13 +253,13 @@ class ElasticSearchOnPrem(ElasticSearch):
 
 class ElasticSearchCloud(ElasticSearch):
     def __init__(
-            self,
-            elk_hostname: str,
-            elasticsearch_port: Optional[int] = 9200,
-            kibana_port: Optional[int] = 5602,
-            protocol: UrlProtocol = UrlProtocol.HTTPS,
-            username: Optional[str] = None,
-            password: Optional[str] = None
+        self,
+        elk_hostname: str,
+        elasticsearch_port: Optional[int] = 9200,
+        kibana_port: Optional[int] = 5602,
+        protocol: UrlProtocol = UrlProtocol.HTTPS,
+        username: Optional[str] = None,
+        password: Optional[str] = None
     ):
         super().__init__(
             elk_hostname=elk_hostname,

{python_databases-0.1.4.dist-info → python_databases-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-databases
-Version: 0.1.4
+Version: 0.1.5
 Summary: A Python library for managing and interacting with multiple databases seamlessly.
 Home-page: https://github.com/aviz92/python-databases
 Author: Avi Zaguri

{python_databases-0.1.4.dist-info → python_databases-0.1.5.dist-info}/RECORD

@@ -1,10 +1,10 @@
 python_databases/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-python_databases/elasticsearch_infrastructure/__init__.py,sha256=zLL3GJwgu5a55TgRhyl_mDbjc-qQBWOskRc0y9diqyw,296
-python_databases/elasticsearch_infrastructure/elasticsearch.py,sha256=WWlYi7icu3mJLEFDL98XbrG2bhAyBx1u_Xv8EPEgQt8,11176
+python_databases/elasticsearch_infrastructure/__init__.py,sha256=cjs95C1yBZpc3kEdkcXEamDvQGsO6ob5r5ttFCJfDaA,314
+python_databases/elasticsearch_infrastructure/elasticsearch.py,sha256=A5NwzqMIcAyuMLhVIlbpoIYO18jQFqWmtaZ7XyeLzzw,10259
 python_databases/postgresql_infrastructure/__init__.py,sha256=h1eJx0ogbtcF6UnN2c7uW5iCuadDab3So1VI15y3kWA,150
 python_databases/postgresql_infrastructure/postgresql.py,sha256=zH8y5w2k2djhHLefDJhlGwa6MtoZKONhJ5vMxIxDdlc,2856
-python_databases-0.1.4.dist-info/licenses/LICENSE,sha256=cSikHY6SZFsPZSBizCDAJ0-Bjjzxt-JtX6TVbKxwimo,1067
-python_databases-0.1.4.dist-info/METADATA,sha256=i0_YUTvulCbHXObsil2bPLbHsXv3XDh_IxpHsx6QWrg,1792
-python_databases-0.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-python_databases-0.1.4.dist-info/top_level.txt,sha256=fXcMUntaYJfKeFS7FjQ6DY4WVXYBsgdrDmhOudYs-vg,17
-python_databases-0.1.4.dist-info/RECORD,,
+python_databases-0.1.5.dist-info/licenses/LICENSE,sha256=cSikHY6SZFsPZSBizCDAJ0-Bjjzxt-JtX6TVbKxwimo,1067
+python_databases-0.1.5.dist-info/METADATA,sha256=SccVOpAUU_gPpKNiHgZO2DANGUgjFw7pGcpf59hgUiA,1792
+python_databases-0.1.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+python_databases-0.1.5.dist-info/top_level.txt,sha256=fXcMUntaYJfKeFS7FjQ6DY4WVXYBsgdrDmhOudYs-vg,17
+python_databases-0.1.5.dist-info/RECORD,,