dbpoint 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbpoint/__init__.py +0 -0
- dbpoint/datadriver.py +328 -0
- dbpoint/dbpoint.py +289 -0
- dbpoint/drivers/__init__.py +0 -0
- dbpoint/drivers/asa.py +80 -0
- dbpoint/drivers/maria.py +54 -0
- dbpoint/drivers/mssql.py +57 -0
- dbpoint/drivers/odbc.py +71 -0
- dbpoint/drivers/oracle.py +82 -0
- dbpoint/drivers/pg.py +170 -0
- dbpoint/perks.py +54 -0
- dbpoint-0.1.1.dist-info/METADATA +120 -0
- dbpoint-0.1.1.dist-info/RECORD +15 -0
- dbpoint-0.1.1.dist-info/WHEEL +4 -0
- dbpoint-0.1.1.dist-info/licenses/LICENSE.txt +10 -0
dbpoint/__init__.py
ADDED
|
File without changes
|
dbpoint/datadriver.py
ADDED
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
'''
|
|
2
|
+
|
|
3
|
+
Ühendus läbi confi (dict), kus võimalikud võtmed järgmistes gruppides:
|
|
4
|
+
a) väljast juhitavad, mida ei tasuks näppida: driver (miks siia jõuti), class (siinne instants ise pärast)
|
|
5
|
+
b) ühenduseteks üldiselt vajaminevad: host, port, engine (ASA jaoks), database, dsn (ODBC DNS), username, password
|
|
6
|
+
c) ühenduse ajal muud kasutatavad: appname (PG jaoks)
|
|
7
|
+
d) metainfos ja muudes üldistustes erisuste juhtimiseks (sh ODBC korral): dialect (ODBC korral nt mysql, ASA korral nt asa5, asa7)
|
|
8
|
+
- võib olla tähenduses metainfo jaoks oluline versioon (ASA tüüpprobleem) kui ka
|
|
9
|
+
- teistele tarbijatele dünaamilsite SQL.ide loomiseks vajalik ver (nt vanemates puudub mingi omadus)
|
|
10
|
+
|
|
11
|
+
SQL käskude jooksutamine (run) koos võimalusega andmete tagastamiseks (do_return), commitiga järel (do_commit_after) ja meta küsimsiega (mõte alles)
|
|
12
|
+
- käskuke jooksutamisel tekkivad vead mäpitakse tootja moduli exceptionitest meie ühtlustatud exceptioniteks (kasutades alamklassis täidetud mäpperit)
|
|
13
|
+
|
|
14
|
+
'''
|
|
15
|
+
from datetime import datetime
|
|
16
|
+
from dbpoint.perks import get_custom_logger, mem_free_now
|
|
17
|
+
from collections import namedtuple
|
|
18
|
+
# two attempts for getting annotations to work (first failed, second is semantically wrong (MySQL Cursor is not Postgre Cursor)):
|
|
19
|
+
#from _typeshed.dbapi import DBAPIConnection, DBAPICursor # too fresh too raw yet (eg. Protocol don't mention rollback())
|
|
20
|
+
#from psycopg2.extensions import connection, cursor
|
|
21
|
+
# third attempt:
|
|
22
|
+
from collections.abc import Sequence, Mapping
|
|
23
|
+
from typing import Any, Protocol
|
|
24
|
+
|
|
25
|
+
logging = get_custom_logger('driver')
|
|
26
|
+
|
|
27
|
+
# next classes (Cursor and Connection) are only for type Annotations for IDEs
|
|
28
|
+
class Cursor(Protocol):
|
|
29
|
+
description: Sequence[Any] | None
|
|
30
|
+
rowcount: int
|
|
31
|
+
arraysize: int
|
|
32
|
+
|
|
33
|
+
def close(self) -> None:
|
|
34
|
+
...
|
|
35
|
+
|
|
36
|
+
def execute(self, operation: Any, *args, **kwargs) -> None:
|
|
37
|
+
...
|
|
38
|
+
|
|
39
|
+
def executemany(
|
|
40
|
+
self,
|
|
41
|
+
operation: Any,
|
|
42
|
+
seq_of_parameters: Sequence[Any] | Mapping[Any, Any],
|
|
43
|
+
*args,
|
|
44
|
+
**kwargs
|
|
45
|
+
) -> None:
|
|
46
|
+
...
|
|
47
|
+
|
|
48
|
+
class Connection(Protocol):
|
|
49
|
+
def close(self) -> None:
|
|
50
|
+
...
|
|
51
|
+
|
|
52
|
+
def commit(self) -> None:
|
|
53
|
+
...
|
|
54
|
+
|
|
55
|
+
def rollback(self) -> None:
|
|
56
|
+
...
|
|
57
|
+
|
|
58
|
+
def cursor(self, *args, **kwargs) -> Cursor:
|
|
59
|
+
...
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class DataDriverGeneric(object):
|
|
63
|
+
|
|
64
|
+
map_of_exceptions = {} # alluv importigu sql_driver_errors ja lihtnime järgi (tootja-draiveri exc klassi viimane osis) vastavus (vt pg teostust)
|
|
65
|
+
conn: Connection = None # klassi instantsil on üks ühendus
|
|
66
|
+
created = None
|
|
67
|
+
last_run = None
|
|
68
|
+
closed_ts = None
|
|
69
|
+
run_count = 0 # tegelikult õnnestunud käskude arv
|
|
70
|
+
run_count_after_last_disconnect = 0
|
|
71
|
+
# escape võidakse ära muuta kui on juba olemas konkreetne draiver ja vb ka lausa konkreetne versioon
|
|
72
|
+
# nt ASA on vana kaldkriips, MySQL on kuni ver 5 (incl) kaldkriips, aga uuemad juba starndadi järgi ülakoma
|
|
73
|
+
apostrophe_escape = r"''"
|
|
74
|
+
# mõelda, kas on vaja
|
|
75
|
+
#error = None ¤ või Exception # ja konkreetne draiver paneb connect sees siia enda oma (endanimi.Error) ja run eristab seda muudest py üldisematest
|
|
76
|
+
#
|
|
77
|
+
type_mapper = {}
|
|
78
|
+
columns_definition: list[dict] = []
|
|
79
|
+
|
|
80
|
+
class_mapper = {
|
|
81
|
+
'CHAR' : 'VARCHAR'
|
|
82
|
+
, 'NCHAR' : 'VARCHAR'
|
|
83
|
+
, 'NVARCHAR' : 'TEXT'
|
|
84
|
+
, 'VARCHAR' : 'TEXT'
|
|
85
|
+
, 'DATETIME' : 'TIMESTAMP'
|
|
86
|
+
, 'TINYINT': 'SMALLINT'
|
|
87
|
+
, 'BLOB' : 'BYTEA'
|
|
88
|
+
, 'LONG VARCHAR' : 'TEXT'
|
|
89
|
+
, 'LONG BINARY' : 'BYTEA'
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
def __init__(self):
|
|
93
|
+
self.created = datetime.now()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def me(self):
|
|
97
|
+
return '' # üldine on nimetu, alluvad suurtähelised (läbivalt suur- või läbivalt väiketähelised) lühendid? PG, MYSQL, MSSQL, ORA, ASA, IQ, ASE, ODBC
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def connect(self, profile: dict) -> int: # usutavasti on alluvatel nati erinevad nõksud/detailid
|
|
101
|
+
...
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def stream(self, sql: str):
|
|
105
|
+
cur: Cursor = self.conn.cursor()
|
|
106
|
+
cur.execute(sql)
|
|
107
|
+
if not self.analyze_column_meta(cur.description):
|
|
108
|
+
return
|
|
109
|
+
#self.last_apos, self.last_structure = self.make_for_create(cur.description)
|
|
110
|
+
#self.last_columns = self.make_for_columns(cur.description)
|
|
111
|
+
while True:
|
|
112
|
+
try:
|
|
113
|
+
row = cur.fetchone()
|
|
114
|
+
except Exception as e1:
|
|
115
|
+
logging.error(e1)
|
|
116
|
+
break
|
|
117
|
+
|
|
118
|
+
if row is None:
|
|
119
|
+
break
|
|
120
|
+
else:
|
|
121
|
+
yield row
|
|
122
|
+
cur.close()
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def get_columns_definition(self) -> list[dict]:
|
|
126
|
+
return self.columns_definition
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def get_mapper(self):
|
|
131
|
+
def the_mapper(dataclass: str) -> str:
|
|
132
|
+
dataclass2 = self.class_mapper.get(dataclass.upper(), dataclass.upper())
|
|
133
|
+
#print(f"{dataclass} -> {dataclass2}")
|
|
134
|
+
return dataclass2
|
|
135
|
+
return the_mapper
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def analyze_column_meta(self, cursor_description: tuple | None) -> bool:
|
|
139
|
+
|
|
140
|
+
self.columns_definition.clear()
|
|
141
|
+
if cursor_description is None:
|
|
142
|
+
return False
|
|
143
|
+
|
|
144
|
+
for desc in cursor_description: # https://peps.python.org/pep-0249/#description
|
|
145
|
+
dataclass = self.type_mapper.get(desc[1], 'TEXT')
|
|
146
|
+
|
|
147
|
+
if dataclass in ('BIGINT', 'INT', 'NUMERIC', 'BOOLEAN'):
|
|
148
|
+
needs_escape = False
|
|
149
|
+
else:
|
|
150
|
+
needs_escape = True
|
|
151
|
+
|
|
152
|
+
if dataclass not in ('TEXT') and desc[3] > 1 and desc[4] is not None: # and desc[4] != 65535:
|
|
153
|
+
details = []
|
|
154
|
+
details.append(f"{desc[4]}")
|
|
155
|
+
if desc[5] is not None and desc[5] != 65535:
|
|
156
|
+
details.append(f"{desc[5]}")
|
|
157
|
+
datatype_details = ",".join(details)
|
|
158
|
+
datatype_details = f"({datatype_details})" # sulud ümber
|
|
159
|
+
else:
|
|
160
|
+
datatype_details = ''
|
|
161
|
+
datatype = dataclass + datatype_details
|
|
162
|
+
temp_name = desc[0] # vaja oleks korduvust ja nime olemasolu kontrollida (aga need võivad ka failida ja las arendaja teeb korda)
|
|
163
|
+
col_def = {'name' : temp_name, 'name_original' : desc[0], "class" : dataclass, "type" : datatype, "needs_escape" : needs_escape}
|
|
164
|
+
self.columns_definition.append(col_def)
|
|
165
|
+
return True
|
|
166
|
+
|
|
167
|
+
def extract_name_string(self, separator: str=', ') -> str:
|
|
168
|
+
if not self.columns_definition:
|
|
169
|
+
return ''
|
|
170
|
+
return separator.join([col['name'] for col in self.columns_definition])
|
|
171
|
+
|
|
172
|
+
def run(self, sql: str, do_return: bool = True, **kwargs):
|
|
173
|
+
# , do_commit_after: bool = True, on_error_rollback = True, do_grab_meta = False
|
|
174
|
+
|
|
175
|
+
if sql is None or len(sql.strip()) < 1:
|
|
176
|
+
raise Exception("Empty command")
|
|
177
|
+
#raise unified.DataDriverEmptyCommand('Empty command') # kas tühi SQL on fataalne või edasi mindav, otsustagu väljakutsuja
|
|
178
|
+
|
|
179
|
+
# olulised muutuvad argumendid ja nende vaikeväärtused
|
|
180
|
+
defaults = {
|
|
181
|
+
'on_success_commit' : True
|
|
182
|
+
, 'new_transaction': False
|
|
183
|
+
, 'on_error_rollback' : True
|
|
184
|
+
, 'on_error_disconnect' : False
|
|
185
|
+
, 'verbose' : False
|
|
186
|
+
, 'style' : 'number' # both on liiga ohtlik, kuna meil metasüsteem ja mitmes kohas on iteratsioon üle veergude
|
|
187
|
+
}
|
|
188
|
+
control = {**defaults, **kwargs}
|
|
189
|
+
|
|
190
|
+
# alternatiiv:
|
|
191
|
+
kwargs.setdefault('verbose', False)
|
|
192
|
+
|
|
193
|
+
#
|
|
194
|
+
extra_on_success_commit = control['on_success_commit']
|
|
195
|
+
extra_on_error_rollback = control['on_error_rollback']
|
|
196
|
+
extra_on_error_disconnect = control['on_error_disconnect']
|
|
197
|
+
extra_new_transaction = control['new_transaction']
|
|
198
|
+
|
|
199
|
+
#' extra_verbose = control['verbose']
|
|
200
|
+
# ja selle alternatiiv:
|
|
201
|
+
extra_verbose = kwargs.get('verbose', False)
|
|
202
|
+
extra_return_style = control['style'] # text, number, both
|
|
203
|
+
if extra_return_style == 'text':
|
|
204
|
+
extra_return_style = 'name'
|
|
205
|
+
if extra_return_style not in ('name', 'number', 'both'):
|
|
206
|
+
extra_return_style = 'number'
|
|
207
|
+
|
|
208
|
+
if extra_new_transaction:
|
|
209
|
+
self.conn.commit()
|
|
210
|
+
|
|
211
|
+
result_set = []
|
|
212
|
+
try:
|
|
213
|
+
#with self.conn.cursor() as cur: // see tuli ära jätta, sest asa cursori __enter__ evib mingit viga (AttributeError)
|
|
214
|
+
cur: Cursor = self.conn.cursor()
|
|
215
|
+
cur.execute(sql)
|
|
216
|
+
#logging.debug("execution instructed")
|
|
217
|
+
|
|
218
|
+
if do_return:
|
|
219
|
+
# cur.description: tuple[str, DBAPITypeCode, int | None, int | None, int | None, int | None, bool | None]
|
|
220
|
+
if not self.analyze_column_meta(cur.description):
|
|
221
|
+
logging.debug("during analyze emptyness happenes")
|
|
222
|
+
return []
|
|
223
|
+
|
|
224
|
+
if extra_return_style == 'number':
|
|
225
|
+
#logging.debug("numbered columns instructed")
|
|
226
|
+
result_set = cur.fetchall()
|
|
227
|
+
|
|
228
|
+
if extra_return_style == 'name':
|
|
229
|
+
#logging.debug("named columns instructed")
|
|
230
|
+
columns_named_spaced = self.extract_name_string(' ')
|
|
231
|
+
if columns_named_spaced is None:
|
|
232
|
+
raise Exception('no metadata grabbed')
|
|
233
|
+
record_type = namedtuple('NamedTupleRecord', columns_named_spaced, rename=True)
|
|
234
|
+
result_set = list(map(record_type._make, cur.fetchall()))
|
|
235
|
+
|
|
236
|
+
if extra_return_style == 'both':
|
|
237
|
+
#logging.debug("dualistic columns instructed")
|
|
238
|
+
columns_named_spaced = self.extract_name_string(' ')
|
|
239
|
+
record_type = namedtuple('DictRecord', columns_named_spaced, rename=True)
|
|
240
|
+
result_set = []
|
|
241
|
+
row: tuple
|
|
242
|
+
for row in cur.fetchall():
|
|
243
|
+
# row on numbrilise indeksiga (list): row[0], row[1] jne
|
|
244
|
+
new_row : dict = {}
|
|
245
|
+
for pos, cell in enumerate(row): # kordame datat
|
|
246
|
+
new_row[record_type._fields[pos]] = cell # str key
|
|
247
|
+
new_row[pos] = cell # int key
|
|
248
|
+
result_set.append(new_row) # list[dict[int|str, any]]
|
|
249
|
+
|
|
250
|
+
except Exception as ex:
|
|
251
|
+
#print('hakkame töötlema ja peitma ja muud tegema, mis pole päris hea vea lokaliseerimiseks, aga hea lõpptarbijale')
|
|
252
|
+
if extra_on_error_rollback:
|
|
253
|
+
self.conn.rollback()
|
|
254
|
+
logging.debug("rollback happenes")
|
|
255
|
+
if extra_on_error_disconnect:
|
|
256
|
+
self.disconnect()
|
|
257
|
+
logging.debug("disconnect happenes")
|
|
258
|
+
raise ex
|
|
259
|
+
finally:
|
|
260
|
+
cur.close() # see juhtub isegi raise ex korral except osas
|
|
261
|
+
|
|
262
|
+
self.last_run = datetime.now()
|
|
263
|
+
self.run_count = self.run_count + 1 # õnnestunud käsk suurendab käskude arvu
|
|
264
|
+
self.run_count_after_last_disconnect = self.run_count_after_last_disconnect + 1
|
|
265
|
+
|
|
266
|
+
#logging.debug(f"commands executed is now {self.run_count} (and {self.run_count_after_last_disconnect} after last connect)")
|
|
267
|
+
|
|
268
|
+
if extra_on_success_commit:
|
|
269
|
+
#logging.debug("commit happenes")
|
|
270
|
+
self.conn.commit()
|
|
271
|
+
|
|
272
|
+
if do_return:
|
|
273
|
+
#logging.debug("return data happenes")
|
|
274
|
+
return result_set
|
|
275
|
+
else:
|
|
276
|
+
#logging.debug("just return happenes")
|
|
277
|
+
return []
|
|
278
|
+
|
|
279
|
+
def escape(self, cell_value, data_class, needs_escape):
|
|
280
|
+
'''
|
|
281
|
+
Postgre jaoks kirjutame üle niikuinii, sest saame kasutada märkhaaval asendamise asemel nn 3nda märgiga ümbritsemist (mis aitab säilitada json ja xml sisusid, peamiselt json)
|
|
282
|
+
Teistel ei tarvitse olla boolean tüüpi, aga sellest ei lähe midagi siin katki.
|
|
283
|
+
Pigem läheb katki, kui on boolean ja see antakse stringina teisiti (ega tõlgita py bool-iks) -- sii ssee fn üle kirjutada
|
|
284
|
+
'''
|
|
285
|
+
if cell_value is None:
|
|
286
|
+
return 'NULL'
|
|
287
|
+
if data_class == 'BOOLEAN' or isinstance(cell_value, bool):
|
|
288
|
+
return 'TRUE' if cell_value == 't' else 'FALSE'
|
|
289
|
+
if needs_escape:
|
|
290
|
+
# vs liht-asendamise meetod
|
|
291
|
+
return (str(cell_value)).replace("'", self.apostrophe_escape)
|
|
292
|
+
else:
|
|
293
|
+
return f"{cell_value}"
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def disconnect(self):
|
|
297
|
+
try:
|
|
298
|
+
self.conn.close()
|
|
299
|
+
except:
|
|
300
|
+
...
|
|
301
|
+
finally:
|
|
302
|
+
self.conn = None
|
|
303
|
+
self.closed_ts = datetime.now()
|
|
304
|
+
self.run_count_after_last_disconnect = 0
|
|
305
|
+
|
|
306
|
+
def commit(self):
|
|
307
|
+
self.conn.commit()
|
|
308
|
+
|
|
309
|
+
def rollback(self):
|
|
310
|
+
self.conn.rollback()
|
|
311
|
+
|
|
312
|
+
def __repr__(self) -> str:
|
|
313
|
+
# profiiliga seotud metainfo
|
|
314
|
+
info = []
|
|
315
|
+
info.append(self.me())
|
|
316
|
+
info.append(f"inited at {self.created:%F %T.%f}") # ISO formatted date, Time, fractions (6 kohta)
|
|
317
|
+
if(self.conn is not None):
|
|
318
|
+
info.append('connected,')
|
|
319
|
+
if self.last_run is None:
|
|
320
|
+
info.append("no runs")
|
|
321
|
+
else:
|
|
322
|
+
info.append(f"last run at {self.last_run:%F %T.%f}")
|
|
323
|
+
else:
|
|
324
|
+
if(self.closed_ts is not None): # kui pole ühendust veel olnud, siis pole ka sulgemise aega
|
|
325
|
+
info.append(f"disconnected at {self.closed_ts:%F %T.%f}")
|
|
326
|
+
info.append(f"run count is {self.run_count} ({self.run_count_after_last_disconnect})")
|
|
327
|
+
return " ".join(info)
|
|
328
|
+
|
dbpoint/dbpoint.py
ADDED
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
#import datacontroller.datacontroller_exceptions as unified
|
|
2
|
+
from typing import Iterable, Callable
|
|
3
|
+
from dbpoint.perks import get_custom_logger, mem_free_now
|
|
4
|
+
|
|
5
|
+
logging = get_custom_logger('dbpoint')
|
|
6
|
+
|
|
7
|
+
class Hub(object):
|
|
8
|
+
|
|
9
|
+
profiles = {}
|
|
10
|
+
known_drivers = {}
|
|
11
|
+
# implemented saab known-iks, kui a) teda esimest korda soovitakse ja b) siinses kaustas on sellenimeline .py fail
|
|
12
|
+
implemented_drivers = {'pg' : ['pg'], 'oracle' : ['oracle']
|
|
13
|
+
, 'asa' : ['asa'], 'maria' : ['maria']
|
|
14
|
+
, 'mssql' : ['mssql'], 'odbc' : ['odbc']}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def __init__(self, list_of_profiles: list[dict]):
|
|
18
|
+
#print(f"Init of {self.__class__.__name__}") # kuna metaclass on Singleton, siis juhtub see ühe korra ainult
|
|
19
|
+
# siin võib ka kulutada aega ja uurida välja, kas kõvakettal on midagi, mis kvalifitseeruks draiveriks -> implemented_drivers
|
|
20
|
+
# aga midagi ei tohi importida, initsialiseerida, ühendada (sest kõiki kindlasti vaja ei lähe)
|
|
21
|
+
|
|
22
|
+
self.last_sql = ''
|
|
23
|
+
self.last_sql_with_error = ''
|
|
24
|
+
|
|
25
|
+
for one_profile in list_of_profiles or []:
|
|
26
|
+
if isinstance(one_profile, dict):
|
|
27
|
+
self.add_profile(one_profile.get('name', ''), one_profile.copy())
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def get_last_error_command(self):
|
|
31
|
+
return self.last_sql_with_error
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def get_last_command(self):
|
|
35
|
+
return self.last_sql
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def run(self, profile_name: str, sql: str, do_return: bool = True, **kwargs):
|
|
39
|
+
# param "sql" on üldjuhul SQL-KÄSK
|
|
40
|
+
# aga see võib olla ka midagi muud, kui driaver pole SQL-draiver (X-tee jm hierahr json/xml, excel/csv)
|
|
41
|
+
# see võib olla path/url
|
|
42
|
+
self.last_sql = sql
|
|
43
|
+
driver = self.get_driver(profile_name)
|
|
44
|
+
|
|
45
|
+
rs = None
|
|
46
|
+
try:
|
|
47
|
+
#logging.debug(f"Command for profile '{profile_name}'")
|
|
48
|
+
rs = driver.run(sql, do_return, **kwargs)
|
|
49
|
+
# vigu tasub püüda, kui otsustame mingeid olukordi üldtasemel ise ära töödelda
|
|
50
|
+
except Exception as e3:
|
|
51
|
+
self.last_sql_with_error = sql
|
|
52
|
+
raise e3
|
|
53
|
+
#self.last_sql_with_error = "" (jätame viimase vea puutumata, isegi kui on väga vana)
|
|
54
|
+
return rs # mis võib olla ka tühi list (kui ei soovitud datat)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def fn_stream(self, profile_name: str):
|
|
58
|
+
def streamer(sql):
|
|
59
|
+
driver = self.get_driver(profile_name)
|
|
60
|
+
return driver.stream(sql)
|
|
61
|
+
return streamer
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def copy_to(self, sql : str, profile_name : str, first_row_grab : Callable, side_effects : Callable | None, prepare_row_command : Callable, save_command : Callable, info_step: int = 1000):
|
|
65
|
+
permanent_info = {}
|
|
66
|
+
pos = 0
|
|
67
|
+
logging.debug(f"copy_to.. {profile_name}")
|
|
68
|
+
flow = self.fn_stream(profile_name)
|
|
69
|
+
for pos, row in enumerate(flow(sql), 1):
|
|
70
|
+
if pos == 1:
|
|
71
|
+
permanent_info : dict = first_row_grab()
|
|
72
|
+
if permanent_info is None:
|
|
73
|
+
logging.error("Problem with GRAB")
|
|
74
|
+
return -1
|
|
75
|
+
#logging.debug(f"pos=1 start side effect")
|
|
76
|
+
side_quest = side_effects() if side_effects is not None else True
|
|
77
|
+
#logging.debug(f"pos=1 end side effect")
|
|
78
|
+
command = prepare_row_command(row, permanent_info)
|
|
79
|
+
if not save_command(command, pos):
|
|
80
|
+
logging.error(f"Problem with SAVE, made {pos}")
|
|
81
|
+
return -pos # if 1st row failes, return -1, if second returns -2
|
|
82
|
+
if pos % info_step == 0:
|
|
83
|
+
mem_free = mem_free_now()
|
|
84
|
+
logging.info(f"Pulled up to here {pos} rows, free memory {mem_free:.2f}")
|
|
85
|
+
if mem_free < 1:
|
|
86
|
+
logging.error(f"Out of memory very soon, so lets quit as we can it do now")
|
|
87
|
+
return -pos
|
|
88
|
+
|
|
89
|
+
logging.info(f"copy_to END {pos} rows")
|
|
90
|
+
return pos
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def generate_command_for_create_table(self, target_table: str, create_as_temp: False, cols_def = None, map_columns: dict = None) -> str:
|
|
94
|
+
as_temp = ' TEMP' if create_as_temp else ''
|
|
95
|
+
if map_columns is None:
|
|
96
|
+
create_columns = ', '.join([col_def['name'] + ' ' + col_def['type'] for col_def in cols_def])
|
|
97
|
+
else:
|
|
98
|
+
create_columns = ', '.join([map_columns[col_def['name']] + ' ' + col_def['type'] for col_def in cols_def if col_def['name'] in map_columns and map_columns[col_def['name']] != ''])
|
|
99
|
+
|
|
100
|
+
create_table = f"CREATE{as_temp} TABLE {target_table} ({create_columns})"
|
|
101
|
+
print(f"{create_table=}")
|
|
102
|
+
return create_table
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def to_file(self, profile_name: str, query: str, file_path) -> int:
|
|
106
|
+
|
|
107
|
+
driver = self.get_driver(profile_name)
|
|
108
|
+
|
|
109
|
+
try:
|
|
110
|
+
number_of_rows = driver.to_file(query, file_path)
|
|
111
|
+
except: # FIXME unified.DataControllerException as e0:
|
|
112
|
+
print('vigusk juhtus: ' + str(e0))
|
|
113
|
+
return -1
|
|
114
|
+
return number_of_rows
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def get_driver(self, profile_name: str | None): # -> (Cursor, None)
|
|
118
|
+
"""
|
|
119
|
+
Returns object capable to make actions and thus having database connection (makes on if no)
|
|
120
|
+
"""
|
|
121
|
+
if profile_name is None:
|
|
122
|
+
logging.error("Missing profile name for driver")
|
|
123
|
+
return None
|
|
124
|
+
|
|
125
|
+
profile_data = self.get_profile(profile_name)
|
|
126
|
+
if profile_data is None:
|
|
127
|
+
return None
|
|
128
|
+
# lugeda profiilist välja, kes tagab teostuse
|
|
129
|
+
driver_name = profile_data['driver']
|
|
130
|
+
if profile_data['class'] is None:
|
|
131
|
+
driver_class = self.known_drivers[driver_name].DataDriver()
|
|
132
|
+
self.set_profile_class(profile_name, driver_class)
|
|
133
|
+
else:
|
|
134
|
+
driver_class = profile_data['class']
|
|
135
|
+
|
|
136
|
+
if driver_class.connect(profile_data) == 1: # 0 => was already connected
|
|
137
|
+
logging.debug(f"Profile {profile_name} connected")
|
|
138
|
+
|
|
139
|
+
return driver_class
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def get_profile(self, profile_name: str | None) -> dict | None:
|
|
143
|
+
if profile_name is None:
|
|
144
|
+
msg = f"Profile name (connection alias) is missing"
|
|
145
|
+
logging.error(msg)
|
|
146
|
+
return None
|
|
147
|
+
#raise Exception(msg)
|
|
148
|
+
if self.is_profile_exists(profile_name):
|
|
149
|
+
return self.profiles[profile_name] # see on py suva, kas see on viide sama obj sisse või uus -- seega alati muuta selfi!
|
|
150
|
+
else:
|
|
151
|
+
msg = f"{profile_name} does not exists"
|
|
152
|
+
logging.error(msg)
|
|
153
|
+
return None
|
|
154
|
+
#raise unified.DataControllerProfileExistenceException(f"Profile '{profile_name}' not existing")
|
|
155
|
+
#raise Exception(msg)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def find_driver(self, named_driver):
|
|
159
|
+
'''
|
|
160
|
+
Find driver by name, and import module containing it
|
|
161
|
+
'''
|
|
162
|
+
import importlib
|
|
163
|
+
import os.path
|
|
164
|
+
logging.debug(f"finding driver for '{named_driver}'")
|
|
165
|
+
# turvameede -- mapper
|
|
166
|
+
if named_driver not in self.implemented_drivers:
|
|
167
|
+
msg = f"Driver {named_driver} is not implemented"
|
|
168
|
+
raise Exception(msg)
|
|
169
|
+
#raise unified.DataControllerProfileUnknowDriverException(f"Driver {named_driver} is not implemented")
|
|
170
|
+
#module_to_import = self.implemented_drivers[named_driver][0] # vana
|
|
171
|
+
|
|
172
|
+
module_name = self.implemented_drivers[named_driver][0]
|
|
173
|
+
logging.debug(f"Module name for {named_driver} is {module_name}")
|
|
174
|
+
long_module_name = '.'.join(['dbpoint', 'drivers', module_name]) # need on built-in moodulid! aga custom?
|
|
175
|
+
logging.debug(f"Long module name is {long_module_name}")
|
|
176
|
+
try: # lets try to import module, starting from root package and long_module_name
|
|
177
|
+
named_driver_module = importlib.import_module(long_module_name, 'dbpoint') # package = self.current_file_dir_name)
|
|
178
|
+
self.known_drivers[named_driver] = named_driver_module
|
|
179
|
+
except Exception as e1:
|
|
180
|
+
print(e1)
|
|
181
|
+
msg = f"Cannot import module {long_module_name} from package dbpoint"
|
|
182
|
+
logging.error(msg)
|
|
183
|
+
return None
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def is_profile_exists(self, profile_name: str) -> bool:
|
|
187
|
+
return profile_name in self.profiles
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def is_driver_known(self, driver_name: str, try_to_find: bool = True) -> bool:
|
|
191
|
+
if driver_name in self.known_drivers:
|
|
192
|
+
return True
|
|
193
|
+
if not try_to_find:
|
|
194
|
+
return False
|
|
195
|
+
self.find_driver(driver_name)
|
|
196
|
+
return self.is_driver_known(driver_name, False) # kutsume välja iseenda, aga ei luba tsüklisse minna
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def add_profile(self, profile_name, profile_data) -> None:
|
|
200
|
+
|
|
201
|
+
if 'driver' not in profile_data:
|
|
202
|
+
msg = f"{profile_name} do not have driver"
|
|
203
|
+
logging.error(msg)
|
|
204
|
+
#raise unified.DataControllerProfileValidationException(f"Profile '{profile_name}' don't specify driver")
|
|
205
|
+
raise Exception(msg)
|
|
206
|
+
driver_name = profile_data['driver']
|
|
207
|
+
if self.is_driver_known(driver_name):
|
|
208
|
+
profile_data['class'] = None
|
|
209
|
+
self.profiles[profile_name] = profile_data
|
|
210
|
+
return
|
|
211
|
+
logging.error(f"{profile_name} uses unknown driver {driver_name}")
|
|
212
|
+
msg = f"Profile '{profile_name}' uses unknown driver '{driver_name}'"
|
|
213
|
+
raise Exception(msg)
|
|
214
|
+
#raise unified.DataControllerProfileUnknowDriverException()
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def is_profile_connected(self, profile_name) -> bool: # kas kasutame nime? või sisu dicti?
|
|
218
|
+
return self.profiles[profile_name]['connected']
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def set_profile_class(self, profile_name: str, profile_class):
|
|
222
|
+
self.profiles[profile_name]['class'] = profile_class
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def do_real_connection_check(self, profile_name):
|
|
226
|
+
'''
|
|
227
|
+
see võiks veenduda, kas ühendus on jätkuvalt olemas ja kui pole, siis markeerida kadunuks
|
|
228
|
+
'''
|
|
229
|
+
...
|
|
230
|
+
|
|
231
|
+
def commit(self, profile_name):
|
|
232
|
+
driver = self.get_driver(profile_name)
|
|
233
|
+
if driver is None:
|
|
234
|
+
error.log("Cannot commit, because of missing profile name")
|
|
235
|
+
return None
|
|
236
|
+
return driver.commit() # == None anyway
|
|
237
|
+
|
|
238
|
+
def rollback(self, profile_name):
|
|
239
|
+
driver = self.get_driver(profile_name)
|
|
240
|
+
if driver is None:
|
|
241
|
+
error.log("Cannot rollback, because of missing profile name")
|
|
242
|
+
return None
|
|
243
|
+
return driver.rollback()
|
|
244
|
+
|
|
245
|
+
def disconnect(self, profile_name):
|
|
246
|
+
if not self.is_profile_exists(profile_name):
|
|
247
|
+
print(f"Profiili nimega '{profile_name}' pole olemas")
|
|
248
|
+
return
|
|
249
|
+
driver_class = self.profiles[profile_name]['class']
|
|
250
|
+
if driver_class is None: # olukord, kus pole veel ühendust vajatudki, st pole loodud
|
|
251
|
+
return
|
|
252
|
+
driver_class.disconnect() # siia ümber pole try vaja, sest driver peaklass juba tegeleb
|
|
253
|
+
self.profiles[profile_name]['connected'] = False
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def disconnect_all(self):
|
|
257
|
+
for profile_name in self.profiles:
|
|
258
|
+
self.disconnect(profile_name)
|
|
259
|
+
logging.debug(f"{profile_name} disconnected")
|
|
260
|
+
|
|
261
|
+
def escape(self, profile_name, cell_value, data_class, needs_escape):
|
|
262
|
+
driver = self.get_driver(profile_name)
|
|
263
|
+
if driver is None:
|
|
264
|
+
error.log("Cannot make escape expression, because of missing profile name")
|
|
265
|
+
return None
|
|
266
|
+
return driver.escape(cell_value, data_class, needs_escape)
|
|
267
|
+
|
|
268
|
+
def get_columns_definition(self, profile_name):
|
|
269
|
+
driver = self.get_driver(profile_name)
|
|
270
|
+
if driver is None:
|
|
271
|
+
error.log("Cannot get columns list, because of missing profile name")
|
|
272
|
+
return None
|
|
273
|
+
return driver.get_columns_definition()
|
|
274
|
+
|
|
275
|
+
def __repr__(self):
|
|
276
|
+
# kõik profiilid ühendusinfoga (parool on varjestatud) ja profiilide metainfoga (millal, palju)
|
|
277
|
+
str_lines = []
|
|
278
|
+
for jrk, (name, profile) in enumerate(self.profiles.items(), 1):
|
|
279
|
+
str_lines.append(f"{jrk}) {name}")
|
|
280
|
+
for key, value in profile.items():
|
|
281
|
+
if key in ['password']:
|
|
282
|
+
out_value = "'" + value[0:1] + '*****' + "'"
|
|
283
|
+
else:
|
|
284
|
+
out_value = repr(value) # nt class
|
|
285
|
+
str_lines.append(f" {key} = {out_value}")
|
|
286
|
+
str_lines.append("")
|
|
287
|
+
str_lines.append("")
|
|
288
|
+
return "\n".join(str_lines)
|
|
289
|
+
|
|
File without changes
|