python-databases 0.1.4__py3-none-any.whl → 0.1.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- python_databases/elasticsearch_infrastructure/__init__.py +3 -3
- python_databases/elasticsearch_infrastructure/elasticsearch.py +123 -178
- python_databases/postgresql_infrastructure/__init__.py +1 -1
- python_databases/postgresql_infrastructure/postgresql.py +23 -18
- {python_databases-0.1.4.dist-info → python_databases-0.1.6.dist-info}/METADATA +3 -2
- python_databases-0.1.6.dist-info/RECORD +10 -0
- python_databases-0.1.4.dist-info/RECORD +0 -10
- {python_databases-0.1.4.dist-info → python_databases-0.1.6.dist-info}/WHEEL +0 -0
- {python_databases-0.1.4.dist-info → python_databases-0.1.6.dist-info}/licenses/LICENSE +0 -0
- {python_databases-0.1.4.dist-info → python_databases-0.1.6.dist-info}/top_level.txt +0 -0
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
from dotenv import load_dotenv
|
|
2
2
|
|
|
3
3
|
from python_databases.elasticsearch_infrastructure.elasticsearch import (
|
|
4
|
-
UrlProtocol,
|
|
5
4
|
ElasticSearch,
|
|
5
|
+
ElasticSearchCloud,
|
|
6
6
|
ElasticSearchOnPrem,
|
|
7
|
-
|
|
7
|
+
UrlProtocol,
|
|
8
8
|
)
|
|
9
9
|
|
|
10
10
|
load_dotenv()
|
|
11
11
|
|
|
12
|
-
__all__ = [
|
|
12
|
+
__all__ = ["UrlProtocol", "ElasticSearch", "ElasticSearchOnPrem", "ElasticSearchCloud"]
|
|
@@ -2,9 +2,10 @@ import logging
|
|
|
2
2
|
import time
|
|
3
3
|
from abc import ABC, abstractmethod
|
|
4
4
|
from copy import deepcopy
|
|
5
|
-
from datetime import
|
|
5
|
+
from datetime import UTC, datetime
|
|
6
6
|
from enum import Enum
|
|
7
|
-
from typing import
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
8
9
|
import pandas as pd
|
|
9
10
|
from custom_python_logger import get_logger
|
|
10
11
|
from elasticsearch import Elasticsearch, helpers
|
|
@@ -12,20 +13,20 @@ from retrying import retry
|
|
|
12
13
|
|
|
13
14
|
|
|
14
15
|
class UrlProtocol(Enum):
|
|
15
|
-
HTTP =
|
|
16
|
-
HTTPS =
|
|
16
|
+
HTTP = "http"
|
|
17
|
+
HTTPS = "https"
|
|
17
18
|
|
|
18
19
|
|
|
19
20
|
class ElasticSearch(ABC):
|
|
20
21
|
def __init__(
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
):
|
|
22
|
+
self,
|
|
23
|
+
elk_hostname: str,
|
|
24
|
+
elasticsearch_port: int | None,
|
|
25
|
+
kibana_port: int | None,
|
|
26
|
+
protocol: UrlProtocol,
|
|
27
|
+
username: str | None,
|
|
28
|
+
password: str | None,
|
|
29
|
+
) -> None:
|
|
29
30
|
self.logger = get_logger(self.__class__.__name__)
|
|
30
31
|
self._change_elasticsearch_logger()
|
|
31
32
|
|
|
@@ -36,191 +37,131 @@ class ElasticSearch(ABC):
|
|
|
36
37
|
|
|
37
38
|
self.elasticsearch_port = elasticsearch_port
|
|
38
39
|
if not self.elasticsearch_port:
|
|
39
|
-
self.elasticsearch_url = f
|
|
40
|
+
self.elasticsearch_url = f"{self.protocol.value}://{self.elk_hostname}"
|
|
40
41
|
else:
|
|
41
|
-
self.elasticsearch_url = f
|
|
42
|
+
self.elasticsearch_url = f"{self.protocol.value}://{self.elk_hostname}:{self.elasticsearch_port}"
|
|
42
43
|
self.kibana_port = kibana_port
|
|
43
44
|
|
|
44
45
|
self.elk_client = None
|
|
45
46
|
|
|
46
|
-
def _change_elasticsearch_logger(self):
|
|
47
|
-
tracer = get_logger('elasticsearch')
|
|
48
|
-
tracer.setLevel(logging.CRITICAL) # or desired level
|
|
49
|
-
# tracer.addHandler(logging.FileHandler('indexer.log'))
|
|
50
|
-
|
|
51
47
|
@abstractmethod
|
|
52
|
-
def connect_to_elasticsearch(self):
|
|
48
|
+
def connect_to_elasticsearch(self) -> None:
|
|
53
49
|
pass
|
|
54
50
|
|
|
55
|
-
|
|
56
|
-
|
|
51
|
+
@staticmethod
|
|
52
|
+
def _change_elasticsearch_logger(debug_level: int = logging.CRITICAL) -> None:
|
|
53
|
+
tracer = get_logger("elasticsearch")
|
|
54
|
+
tracer.setLevel(debug_level)
|
|
55
|
+
|
|
56
|
+
def _set_list_of_docs_quick(self, list_of_docs: list[dict], request_timeout: int = 600) -> None:
|
|
57
|
+
if failed_response := helpers.streaming_bulk(
|
|
58
|
+
self.elk_client, list_of_docs, raise_on_error=False, request_timeout=request_timeout, chunk_size=1000
|
|
59
|
+
):
|
|
60
|
+
for item in failed_response:
|
|
61
|
+
if item[1]["index"]["status"] != 201 or item[1]["index"]["_shards"]["failed"] > 0:
|
|
62
|
+
self.logger.debug(f"Failed document: \n{item}")
|
|
63
|
+
|
|
64
|
+
def _set_list_of_docs_safe(self, list_of_docs: list[dict]) -> None:
|
|
65
|
+
success_response, failed_response = helpers.bulk(self.elk_client, list_of_docs)
|
|
66
|
+
if success_response == len(list_of_docs):
|
|
67
|
+
self.logger.info("All documents were reported successfully")
|
|
68
|
+
else:
|
|
69
|
+
self.logger.error(
|
|
70
|
+
f"Not all documents were reported successfully.\n"
|
|
71
|
+
f"The documents that were not reported successfully: {failed_response}"
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
def post_list_of_docs(self, list_of_docs: list[dict], request_timeout: int = 600, quick: bool = False) -> None:
|
|
75
|
+
self.logger.info("Start to report the documents to ELK")
|
|
57
76
|
|
|
58
77
|
if quick:
|
|
59
|
-
|
|
60
|
-
self.elk_client,
|
|
61
|
-
list_of_docs,
|
|
62
|
-
raise_on_error=False,
|
|
63
|
-
request_timeout=request_timeout,
|
|
64
|
-
chunk_size=1000
|
|
65
|
-
):
|
|
66
|
-
for item in failed_response:
|
|
67
|
-
if item[1]['index']['status'] != 201 or item[1]['index']['_shards']['failed'] > 0:
|
|
68
|
-
self.logger.debug(f'Failed document: \n{item}')
|
|
78
|
+
self._set_list_of_docs_quick(list_of_docs=list_of_docs, request_timeout=request_timeout)
|
|
69
79
|
else:
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
self,
|
|
82
|
-
list_of_docs: list[dict],
|
|
83
|
-
index_name: str,
|
|
84
|
-
doc: dict[str, Any]
|
|
85
|
-
) -> None:
|
|
86
|
-
list_of_docs.append(
|
|
87
|
-
{
|
|
88
|
-
"_index": index_name,
|
|
89
|
-
"_source": deepcopy(doc)
|
|
90
|
-
}
|
|
91
|
-
)
|
|
80
|
+
self._set_list_of_docs_safe(list_of_docs=list_of_docs)
|
|
81
|
+
self.logger.info("Finish to report the documents to ELK")
|
|
82
|
+
|
|
83
|
+
@staticmethod
|
|
84
|
+
def _get_doc_with_basic_info() -> dict:
|
|
85
|
+
doc = {}
|
|
86
|
+
|
|
87
|
+
timestamp = datetime.now(tz=UTC)
|
|
88
|
+
date_and_time_str = timestamp.isoformat()
|
|
89
|
+
date = date_and_time_str.split("T")[0]
|
|
90
|
+
time_part = date_and_time_str.split("T")[1].split("+")[0]
|
|
92
91
|
|
|
93
|
-
def _add_basic_info(
|
|
94
|
-
self,
|
|
95
|
-
doc: dict,
|
|
96
|
-
doc_id: str,
|
|
97
|
-
timestamp: datetime,
|
|
98
|
-
date_and_time: str,
|
|
99
|
-
username: str = None
|
|
100
|
-
) -> dict:
|
|
101
92
|
doc.update(
|
|
102
93
|
{
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
94
|
+
"doc_id": timestamp.strftime("%Y%m%dT%H%M%S%f"),
|
|
95
|
+
"timestamp": timestamp,
|
|
96
|
+
"date_and_time_str": date_and_time_str,
|
|
97
|
+
"date_str": date,
|
|
98
|
+
"time_str": time_part,
|
|
106
99
|
}
|
|
107
100
|
)
|
|
108
|
-
if username:
|
|
109
|
-
doc['username'] = username
|
|
110
101
|
return doc
|
|
111
102
|
|
|
112
|
-
|
|
103
|
+
@staticmethod
|
|
104
|
+
def _add_list_values_as_str(doc: dict, column: str, value: Any) -> None:
|
|
113
105
|
"""Add list values as both list and string representation to the provided doc"""
|
|
114
|
-
if
|
|
115
|
-
filtered_value =
|
|
116
|
-
doc[f
|
|
117
|
-
doc[f
|
|
118
|
-
|
|
119
|
-
def _build_document(
|
|
120
|
-
self,
|
|
121
|
-
row: dict,
|
|
122
|
-
doc_id: str,
|
|
123
|
-
timestamp: datetime,
|
|
124
|
-
date_and_time: str,
|
|
125
|
-
username: str = None
|
|
126
|
-
) -> dict:
|
|
127
|
-
doc = self._add_basic_info(
|
|
128
|
-
doc={},
|
|
129
|
-
doc_id=doc_id,
|
|
130
|
-
timestamp=timestamp,
|
|
131
|
-
date_and_time=date_and_time,
|
|
132
|
-
username=username
|
|
133
|
-
)
|
|
106
|
+
if isinstance(value, list) and all(isinstance(item, str) for item in value):
|
|
107
|
+
filtered_value = [item for item in value if item is not None]
|
|
108
|
+
doc[f"{column}"] = filtered_value
|
|
109
|
+
doc[f"{column}_str"] = ", ".join(filtered_value)
|
|
134
110
|
|
|
135
|
-
|
|
111
|
+
def _build_document(self, row: dict) -> dict:
|
|
112
|
+
doc = self._get_doc_with_basic_info()
|
|
136
113
|
for custom_field, value in row.items():
|
|
137
114
|
doc[custom_field] = value
|
|
115
|
+
|
|
138
116
|
self._add_list_values_as_str(
|
|
139
117
|
doc=doc,
|
|
118
|
+
column=custom_field,
|
|
140
119
|
value=value,
|
|
141
|
-
column=custom_field
|
|
142
120
|
)
|
|
143
|
-
|
|
144
121
|
return doc
|
|
145
122
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
timestamp: datetime,
|
|
152
|
-
date_and_time: str,
|
|
153
|
-
username: str = None
|
|
154
|
-
) -> list[dict]:
|
|
123
|
+
@staticmethod
|
|
124
|
+
def _fill_list_of_docs(index_name: str, doc: dict[str, Any]) -> dict:
|
|
125
|
+
return {"_index": index_name, "_source": deepcopy(doc)}
|
|
126
|
+
|
|
127
|
+
def _prepare_documents_for_bulk(self, data: list[dict], doc_index_name: str) -> list[dict]:
|
|
155
128
|
list_of_docs = []
|
|
156
129
|
for index, row in enumerate(data):
|
|
157
|
-
self.logger.debug(f
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
row=row,
|
|
161
|
-
doc_id=doc_id,
|
|
162
|
-
timestamp=timestamp,
|
|
163
|
-
date_and_time=date_and_time,
|
|
164
|
-
username=username
|
|
165
|
-
)
|
|
166
|
-
|
|
167
|
-
self.fill_list_of_docs(
|
|
168
|
-
list_of_docs=list_of_docs,
|
|
169
|
-
index_name=doc_index_name,
|
|
170
|
-
doc=doc
|
|
171
|
-
)
|
|
172
|
-
|
|
130
|
+
self.logger.debug(f"index: {index + 1}/{len(data)}")
|
|
131
|
+
doc = self._build_document(row=row)
|
|
132
|
+
list_of_docs.append(self._fill_list_of_docs(index_name=doc_index_name, doc=doc))
|
|
173
133
|
return list_of_docs
|
|
174
134
|
|
|
175
|
-
def
|
|
135
|
+
def post_list_of_docs_as_bulk_chunk(
|
|
136
|
+
self, list_of_docs: list[dict], chunk_size: int = 1000, time_sleep: int = 60, quick: bool = False
|
|
137
|
+
) -> None:
|
|
176
138
|
for i in range(0, len(list_of_docs), chunk_size):
|
|
177
|
-
self.logger.info(f
|
|
178
|
-
chunk = list_of_docs[i:i + chunk_size]
|
|
179
|
-
self.
|
|
139
|
+
self.logger.info(f"index: {i} - {i + chunk_size} / {len(list_of_docs)}")
|
|
140
|
+
chunk = list_of_docs[i : i + chunk_size]
|
|
141
|
+
self.post_list_of_docs(list_of_docs=chunk, quick=quick)
|
|
180
142
|
time.sleep(time_sleep)
|
|
181
143
|
|
|
182
|
-
def fill_elk_index_as_bulk(
|
|
183
|
-
|
|
184
|
-
data: list[dict],
|
|
185
|
-
doc_index_name: str,
|
|
186
|
-
doc_id: Optional[str] = None,
|
|
187
|
-
timestamp: datetime = None,
|
|
188
|
-
date_and_time: Optional[str] = None,
|
|
189
|
-
username: Optional[str] = None,
|
|
190
|
-
chunk_size=1000,
|
|
191
|
-
time_sleep: int = 1
|
|
144
|
+
def fill_elk_index_as_bulk( # post_list_of_docs_as_bulk
|
|
145
|
+
self, data: list[dict], doc_index_name: str, chunk_size: int = 1000, time_sleep: int = 1, quick: bool = True
|
|
192
146
|
) -> None: # use as the main function to send data to the elastic search
|
|
193
|
-
timestamp = timestamp or datetime.now(tz=timezone.utc)
|
|
194
147
|
list_of_docs = self._prepare_documents_for_bulk(
|
|
195
148
|
data=data,
|
|
196
149
|
doc_index_name=doc_index_name,
|
|
197
|
-
timestamp=timestamp,
|
|
198
|
-
doc_id=doc_id or timestamp.strftime('%Y_%m_%dT%H_%M_%S_%f'),
|
|
199
|
-
date_and_time=date_and_time or timestamp.isoformat(),
|
|
200
|
-
username=username
|
|
201
150
|
)
|
|
202
151
|
|
|
203
|
-
self.
|
|
152
|
+
self.post_list_of_docs_as_bulk_chunk(
|
|
204
153
|
list_of_docs=list_of_docs,
|
|
205
154
|
chunk_size=chunk_size,
|
|
206
|
-
time_sleep=time_sleep
|
|
155
|
+
time_sleep=time_sleep,
|
|
156
|
+
quick=quick,
|
|
207
157
|
)
|
|
208
158
|
|
|
209
|
-
def convert_dataframes_to_list_of_docs(self, dataframe: pd.DataFrame) -> list:
|
|
210
|
-
data = []
|
|
211
|
-
dataframe_values = dataframe.values.tolist()
|
|
212
|
-
for index, value in enumerate(dataframe_values):
|
|
213
|
-
self.logger.debug(f'index: {index}/{len(dataframe_values)}')
|
|
214
|
-
data.append(value)
|
|
215
|
-
return data
|
|
216
|
-
|
|
217
159
|
def check_if_index_exists(self, index: str) -> bool:
|
|
218
160
|
if self.elk_client.indices.exists(index=index):
|
|
219
161
|
self.logger.info(f"The index '{index}' exists.")
|
|
220
162
|
return True
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
return False
|
|
163
|
+
self.logger.info(f"The index '{index}' does not exist.")
|
|
164
|
+
return False
|
|
224
165
|
|
|
225
166
|
def delete_index(self, index: str) -> None:
|
|
226
167
|
self.elk_client.indices.delete(index=index)
|
|
@@ -230,35 +171,41 @@ class ElasticSearch(ABC):
|
|
|
230
171
|
if self.check_if_index_exists(index=index):
|
|
231
172
|
self.delete_index(index=index)
|
|
232
173
|
return True
|
|
233
|
-
|
|
234
|
-
self.logger.info(f"The index '{index}' does not exist.")
|
|
235
|
-
return False
|
|
174
|
+
return False
|
|
236
175
|
|
|
237
176
|
def get_documents(self, index: str, query: dict) -> list[dict]:
|
|
238
177
|
try:
|
|
239
178
|
res = self.elk_client.search(index=index, body=query)["hits"]["hits"]
|
|
240
179
|
return [doc["_source"] for doc in res]
|
|
241
180
|
except Exception as e:
|
|
242
|
-
raise Exception(f"Failed to get documents from index '{index}': {str(e)}")
|
|
181
|
+
raise Exception(f"Failed to get documents from index '{index}': {str(e)}") from e
|
|
182
|
+
|
|
183
|
+
def convert_dataframes_to_list_of_docs(self, dataframe: pd.DataFrame) -> list:
|
|
184
|
+
data = []
|
|
185
|
+
dataframe_values = dataframe.values.tolist()
|
|
186
|
+
for index, value in enumerate(dataframe_values):
|
|
187
|
+
self.logger.debug(f"index: {index}/{len(dataframe_values)}")
|
|
188
|
+
data.append(value)
|
|
189
|
+
return data
|
|
243
190
|
|
|
244
191
|
|
|
245
192
|
class ElasticSearchOnPrem(ElasticSearch):
|
|
246
193
|
def __init__(
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
):
|
|
194
|
+
self,
|
|
195
|
+
elk_hostname: str,
|
|
196
|
+
elasticsearch_port: int | None = 9200,
|
|
197
|
+
kibana_port: int | None = 5602,
|
|
198
|
+
protocol: UrlProtocol = UrlProtocol.HTTPS,
|
|
199
|
+
username: str | None = None,
|
|
200
|
+
password: str | None = None,
|
|
201
|
+
) -> None:
|
|
255
202
|
super().__init__(
|
|
256
203
|
elk_hostname=elk_hostname,
|
|
257
204
|
elasticsearch_port=elasticsearch_port,
|
|
258
205
|
kibana_port=kibana_port,
|
|
259
206
|
protocol=protocol,
|
|
260
207
|
username=username,
|
|
261
|
-
password=password
|
|
208
|
+
password=password,
|
|
262
209
|
)
|
|
263
210
|
|
|
264
211
|
self.logger = get_logger(self.__class__.__name__)
|
|
@@ -266,42 +213,41 @@ class ElasticSearchOnPrem(ElasticSearch):
|
|
|
266
213
|
self.connect_to_elasticsearch()
|
|
267
214
|
|
|
268
215
|
@retry(stop_max_attempt_number=3, wait_fixed=180000)
|
|
269
|
-
def connect_to_elasticsearch(self):
|
|
216
|
+
def connect_to_elasticsearch(self) -> None:
|
|
270
217
|
if self.username and self.password:
|
|
271
218
|
self.elk_client = Elasticsearch(
|
|
272
219
|
hosts=[self.elasticsearch_url],
|
|
273
220
|
http_auth=(self.username, self.password),
|
|
274
221
|
# ca_certs="/etc/elasticsearch/certs/http_ca.crt",
|
|
275
|
-
verify_certs=False,
|
|
222
|
+
verify_certs=False,
|
|
276
223
|
)
|
|
277
224
|
else:
|
|
278
|
-
self.elk_client = Elasticsearch(hosts=[f
|
|
225
|
+
self.elk_client = Elasticsearch(hosts=[f"{self.elasticsearch_url}"])
|
|
279
226
|
|
|
280
227
|
if self.elk_client.ping():
|
|
281
228
|
self.logger.info("Elasticsearch on-prem Connection Successful")
|
|
282
229
|
else:
|
|
283
230
|
self.logger.error("Elasticsearch on-prem Connection Failed")
|
|
284
|
-
raise Exception(
|
|
285
|
-
f'Failed to connect to Elasticsearch on-prem on {self.elasticsearch_url}')
|
|
231
|
+
raise Exception(f"Failed to connect to Elasticsearch on-prem on {self.elasticsearch_url}")
|
|
286
232
|
|
|
287
233
|
|
|
288
234
|
class ElasticSearchCloud(ElasticSearch):
|
|
289
235
|
def __init__(
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
):
|
|
236
|
+
self,
|
|
237
|
+
elk_hostname: str,
|
|
238
|
+
elasticsearch_port: int | None = 9200,
|
|
239
|
+
kibana_port: int | None = 5602,
|
|
240
|
+
protocol: UrlProtocol = UrlProtocol.HTTPS,
|
|
241
|
+
username: str | None = None,
|
|
242
|
+
password: str | None = None,
|
|
243
|
+
) -> None:
|
|
298
244
|
super().__init__(
|
|
299
245
|
elk_hostname=elk_hostname,
|
|
300
246
|
elasticsearch_port=elasticsearch_port,
|
|
301
247
|
kibana_port=kibana_port,
|
|
302
248
|
protocol=protocol,
|
|
303
249
|
username=username,
|
|
304
|
-
password=password
|
|
250
|
+
password=password,
|
|
305
251
|
)
|
|
306
252
|
|
|
307
253
|
self.logger = get_logger(self.__class__.__name__)
|
|
@@ -309,12 +255,11 @@ class ElasticSearchCloud(ElasticSearch):
|
|
|
309
255
|
self.connect_to_elasticsearch()
|
|
310
256
|
|
|
311
257
|
@retry(stop_max_attempt_number=3, wait_fixed=180000)
|
|
312
|
-
def connect_to_elasticsearch(self):
|
|
258
|
+
def connect_to_elasticsearch(self) -> None:
|
|
313
259
|
self.elk_client = Elasticsearch(cloud_id=self.elk_hostname, http_auth=(self.username, self.password))
|
|
314
260
|
|
|
315
261
|
if self.elk_client.ping():
|
|
316
262
|
self.logger.info("Elasticsearch cloud Connection Successful")
|
|
317
263
|
else:
|
|
318
264
|
self.logger.error("Elasticsearch cloud Connection Failed")
|
|
319
|
-
raise Exception(
|
|
320
|
-
f'Failed to connect to Elasticsearch cloud on {self.elk_hostname}')
|
|
265
|
+
raise Exception(f"Failed to connect to Elasticsearch cloud on {self.elk_hostname}")
|
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
import logging
|
|
2
1
|
import os
|
|
3
|
-
from
|
|
2
|
+
from types import TracebackType
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
4
5
|
import psycopg2
|
|
5
6
|
from custom_python_logger import get_logger
|
|
6
7
|
from psycopg2 import extras
|
|
7
|
-
from retrying import retry
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
class PostgreSQL:
|
|
11
11
|
def __init__(
|
|
12
12
|
self,
|
|
13
|
-
name:
|
|
14
|
-
host:
|
|
15
|
-
port:
|
|
16
|
-
username:
|
|
17
|
-
password:
|
|
18
|
-
):
|
|
13
|
+
name: str | None = None,
|
|
14
|
+
host: str | None = None,
|
|
15
|
+
port: str | None = None,
|
|
16
|
+
username: str | None = None,
|
|
17
|
+
password: str | None = None,
|
|
18
|
+
) -> None:
|
|
19
19
|
self.logger = get_logger(self.__class__.__name__)
|
|
20
20
|
|
|
21
21
|
self.name = name or os.getenv("POSTGRESQL_NAME")
|
|
@@ -30,21 +30,21 @@ class PostgreSQL:
|
|
|
30
30
|
self.create_connection()
|
|
31
31
|
|
|
32
32
|
@property
|
|
33
|
-
def client(self):
|
|
33
|
+
def client(self) -> psycopg2.extensions.connection:
|
|
34
34
|
if not self._client or self._client.closed:
|
|
35
35
|
self.logger.warning("Re-establishing PostgreSQL connection")
|
|
36
36
|
self.create_connection()
|
|
37
37
|
return self._client
|
|
38
38
|
|
|
39
39
|
@property
|
|
40
|
-
def cursor(self):
|
|
40
|
+
def cursor(self) -> psycopg2.extensions.cursor:
|
|
41
41
|
if not self._cursor or self._cursor.closed:
|
|
42
42
|
self.logger.warning("Re-creating PostgreSQL cursor")
|
|
43
43
|
self.create_connection()
|
|
44
44
|
return self._cursor
|
|
45
45
|
|
|
46
46
|
# @retry(stop_max_attempt_number=3, wait_fixed=180000)
|
|
47
|
-
def create_connection(self):
|
|
47
|
+
def create_connection(self) -> None:
|
|
48
48
|
self.logger.info(f"Create connection to {self.host}:{self.port}")
|
|
49
49
|
|
|
50
50
|
self._client = psycopg2.connect(
|
|
@@ -56,15 +56,15 @@ class PostgreSQL:
|
|
|
56
56
|
)
|
|
57
57
|
self._cursor = self._client.cursor(cursor_factory=extras.DictCursor)
|
|
58
58
|
|
|
59
|
-
def execute_query(self, query: str, params:
|
|
59
|
+
def execute_query(self, query: str, params: tuple | dict[str, Any] | None = None) -> None:
|
|
60
60
|
self.logger.debug(f"Fetch rows from DB by query {query} with params {params}")
|
|
61
61
|
try:
|
|
62
62
|
self.cursor.execute(query, params)
|
|
63
63
|
except psycopg2.ProgrammingError as e:
|
|
64
64
|
raise psycopg2.ProgrammingError(f"Error executing query: {query}") from e
|
|
65
|
-
self.logger.debug(
|
|
65
|
+
self.logger.debug("Finished to fetch rows by query")
|
|
66
66
|
|
|
67
|
-
def fetch_all(self, query: str, params:
|
|
67
|
+
def fetch_all(self, query: str, params: tuple | dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
|
68
68
|
self.execute_query(query, params)
|
|
69
69
|
try:
|
|
70
70
|
results = self.cursor.fetchall()
|
|
@@ -73,12 +73,17 @@ class PostgreSQL:
|
|
|
73
73
|
self.logger.debug(f"Fetched {len(results)} rows")
|
|
74
74
|
return results
|
|
75
75
|
|
|
76
|
-
def close(self):
|
|
76
|
+
def close(self) -> None:
|
|
77
77
|
self.cursor.close()
|
|
78
78
|
self.client.close()
|
|
79
79
|
|
|
80
|
-
def __enter__(self):
|
|
80
|
+
def __enter__(self) -> "PostgreSQL":
|
|
81
81
|
return self
|
|
82
82
|
|
|
83
|
-
def __exit__(
|
|
83
|
+
def __exit__(
|
|
84
|
+
self,
|
|
85
|
+
exc_type: type[BaseException] | None,
|
|
86
|
+
exc_val: BaseException | None,
|
|
87
|
+
exc_tb: TracebackType | None,
|
|
88
|
+
) -> None:
|
|
84
89
|
self.close()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: python-databases
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.6
|
|
4
4
|
Summary: A Python library for managing and interacting with multiple databases seamlessly.
|
|
5
5
|
Home-page: https://github.com/aviz92/python-databases
|
|
6
6
|
Author: Avi Zaguri
|
|
@@ -11,7 +11,8 @@ Description-Content-Type: text/markdown
|
|
|
11
11
|
License-File: LICENSE
|
|
12
12
|
Requires-Dist: setuptools
|
|
13
13
|
Requires-Dist: wheel
|
|
14
|
-
Requires-Dist: dotenv
|
|
14
|
+
Requires-Dist: python-dotenv
|
|
15
|
+
Requires-Dist: pre-commit
|
|
15
16
|
Requires-Dist: custom-python-logger
|
|
16
17
|
Requires-Dist: retrying
|
|
17
18
|
Requires-Dist: elasticsearch<9
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
python_databases/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
python_databases/elasticsearch_infrastructure/__init__.py,sha256=U1orkPi0Pg-1rtY1CPSF4-eVRfrMr36qpT9VNKjBUw4,297
|
|
3
|
+
python_databases/elasticsearch_infrastructure/elasticsearch.py,sha256=qIimoK4ykU38-pDlfqxgEpzsyXwZBP3ld5qXZE5ewb4,9989
|
|
4
|
+
python_databases/postgresql_infrastructure/__init__.py,sha256=yBjq7SLnNgnxW0DCQHEwCi_FrCNtjsJGJXIq7XtkUZg,150
|
|
5
|
+
python_databases/postgresql_infrastructure/postgresql.py,sha256=z1yBG-80Hj67mjDVlPvdIxfT5vl4IlbQE0rk2UEE4wY,3020
|
|
6
|
+
python_databases-0.1.6.dist-info/licenses/LICENSE,sha256=cSikHY6SZFsPZSBizCDAJ0-Bjjzxt-JtX6TVbKxwimo,1067
|
|
7
|
+
python_databases-0.1.6.dist-info/METADATA,sha256=KHsM3eTM-d9IzB5ISH5-icGU_Fk-A1N5XWVlL07fhfI,1825
|
|
8
|
+
python_databases-0.1.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
9
|
+
python_databases-0.1.6.dist-info/top_level.txt,sha256=fXcMUntaYJfKeFS7FjQ6DY4WVXYBsgdrDmhOudYs-vg,17
|
|
10
|
+
python_databases-0.1.6.dist-info/RECORD,,
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
python_databases/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
python_databases/elasticsearch_infrastructure/__init__.py,sha256=zLL3GJwgu5a55TgRhyl_mDbjc-qQBWOskRc0y9diqyw,296
|
|
3
|
-
python_databases/elasticsearch_infrastructure/elasticsearch.py,sha256=WWlYi7icu3mJLEFDL98XbrG2bhAyBx1u_Xv8EPEgQt8,11176
|
|
4
|
-
python_databases/postgresql_infrastructure/__init__.py,sha256=h1eJx0ogbtcF6UnN2c7uW5iCuadDab3So1VI15y3kWA,150
|
|
5
|
-
python_databases/postgresql_infrastructure/postgresql.py,sha256=zH8y5w2k2djhHLefDJhlGwa6MtoZKONhJ5vMxIxDdlc,2856
|
|
6
|
-
python_databases-0.1.4.dist-info/licenses/LICENSE,sha256=cSikHY6SZFsPZSBizCDAJ0-Bjjzxt-JtX6TVbKxwimo,1067
|
|
7
|
-
python_databases-0.1.4.dist-info/METADATA,sha256=i0_YUTvulCbHXObsil2bPLbHsXv3XDh_IxpHsx6QWrg,1792
|
|
8
|
-
python_databases-0.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
9
|
-
python_databases-0.1.4.dist-info/top_level.txt,sha256=fXcMUntaYJfKeFS7FjQ6DY4WVXYBsgdrDmhOudYs-vg,17
|
|
10
|
-
python_databases-0.1.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|