dbpoint 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbpoint/__init__.py +0 -0
- dbpoint/datadriver.py +328 -0
- dbpoint/dbpoint.py +289 -0
- dbpoint/drivers/__init__.py +0 -0
- dbpoint/drivers/asa.py +80 -0
- dbpoint/drivers/maria.py +54 -0
- dbpoint/drivers/mssql.py +57 -0
- dbpoint/drivers/odbc.py +71 -0
- dbpoint/drivers/oracle.py +82 -0
- dbpoint/drivers/pg.py +170 -0
- dbpoint/perks.py +54 -0
- dbpoint-0.1.1.dist-info/METADATA +120 -0
- dbpoint-0.1.1.dist-info/RECORD +15 -0
- dbpoint-0.1.1.dist-info/WHEEL +4 -0
- dbpoint-0.1.1.dist-info/licenses/LICENSE.txt +10 -0
dbpoint/drivers/asa.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Konkreetse DB Lib 2.0 draiveri wrapper-klass, mis omab tarkust,
|
|
3
|
+
a) kuidas ühendusprofiili andmetest teha ühenduseks vajalik struktuur ja ühendust võtta
|
|
4
|
+
b) kuidas RDBM draiveri spetsiifilised vead teisendada meie universaalseteks vigadeks
|
|
5
|
+
(selleks, et kasutaja ei peaks mooduleid importima ega urgitsema teostaja detaildies)
|
|
6
|
+
|
|
7
|
+
Nõuded: paigaldatud on ASA client (seal on dbcapi.dll ja sõltuvused)
|
|
8
|
+
https://help.sap.com/docs/SUPPORT_CONTENT/sqlany/3362971128.html
|
|
9
|
+
Hüpotees: võib võtta uusima (ver 17), saadaval on ka 16 ja 12.
|
|
10
|
+
|
|
11
|
+
'''
|
|
12
|
+
|
|
13
|
+
from dbpoint.datadriver import DataDriverGeneric # ülemklass (fn-id run, disconnect)
|
|
14
|
+
#import datacontroller.datacontroller_exceptions as unified
|
|
15
|
+
#from dbpoint import logging
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class DataDriver(DataDriverGeneric):
|
|
19
|
+
|
|
20
|
+
map_of_exceptions = {
|
|
21
|
+
'SyntaxError' : Exception # unified.DataDriverSyntaxException
|
|
22
|
+
, 'UndefinedTable' : Exception # unified.DataDriverUndefinedTable
|
|
23
|
+
, 'IntegrityError' : Exception # unified.DataDriverSyntaxException # fixme
|
|
24
|
+
, 'ProgrammingError' : Exception # unified.DataDriverSyntaxException # fixme
|
|
25
|
+
, 'OperationalError' : Exception # unified.DataDriverSyntaxException # fixme
|
|
26
|
+
, 'InternalError' : Exception # unified.DataDriverSyntaxException # fixme
|
|
27
|
+
, 'DataError' : Exception # unified.DataDriverSyntaxException # fixme
|
|
28
|
+
, 'DatabaseError' : Exception # unified.DataDriverSyntaxException # fixme // see on üldine viga, eelmiste üldistus
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
def me(self) -> str:
|
|
32
|
+
return 'ASA'
|
|
33
|
+
|
|
34
|
+
type_mapper = {
|
|
35
|
+
20 : 'BIGINT'
|
|
36
|
+
, 1184 : 'TIMESTAMPTZ'
|
|
37
|
+
, 23 : 'INT'
|
|
38
|
+
, 3802 : 'JSONB' #(jsonb tehakse pythoni json stringiks, kus on ülakomad jutumärkide asemel ja ei saa tagasi kirjutada)
|
|
39
|
+
, 16 : 'BOOLEAN'
|
|
40
|
+
, 1043 : 'TEXT' # varchar tasub niikuinii pikemaks teha, lisaks on tüütu saada universaalselt kätte pikkust
|
|
41
|
+
, 25 :'TEXT'
|
|
42
|
+
, 1700 : 'NUMERIC'
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def connect(self, profile: dict) -> int:
|
|
49
|
+
|
|
50
|
+
if self.conn is not None: # kui ühendus on olemas, siis ei võta ühendust
|
|
51
|
+
return 0
|
|
52
|
+
|
|
53
|
+
try:
|
|
54
|
+
import sqlanydb # https://github.com/sqlanywhere/sqlanydb/blob/master/sqlanydb.py
|
|
55
|
+
except ImportError:
|
|
56
|
+
raise Exception(f"Driver package not installed. Do pip install sqlanydb (and install client-side drivers)")
|
|
57
|
+
#raise unified.DataDriverNotInstalled('tee: pip install sqlanydb (ja paigalda ASA kliendifailid)')
|
|
58
|
+
|
|
59
|
+
conn_dict = self.profile_to_dict(profile)
|
|
60
|
+
self.conn = sqlanydb.connect(**conn_dict)
|
|
61
|
+
#print('now ASA is connected')
|
|
62
|
+
self.apostrophe_escape = '\\'
|
|
63
|
+
# vb tõsta ka versiooniklassid profile -> driver/self
|
|
64
|
+
#self.error = sqlanydb.Error
|
|
65
|
+
return 1
|
|
66
|
+
|
|
67
|
+
def profile_to_dict(self, profile: dict) -> dict:
|
|
68
|
+
|
|
69
|
+
conn_args = {}
|
|
70
|
+
conn_args['uid'] = profile.get('username', 'xxxxx')
|
|
71
|
+
conn_args['pwd'] = profile.get('password', 'xxxxx')
|
|
72
|
+
conn_args['eng'] = profile.get('engine', 'xxxxx')
|
|
73
|
+
conn_args['dbn'] = profile.get('database', 'xxxxx')
|
|
74
|
+
|
|
75
|
+
# ja eraldi võti TCP ühenduste spetsiifika jaoks
|
|
76
|
+
port = profile.get('port', 2638)
|
|
77
|
+
host = profile.get('host', 'localhost')
|
|
78
|
+
conn_args['links'] = f"tcpip(host={host};port={port})"
|
|
79
|
+
|
|
80
|
+
return conn_args
|
dbpoint/drivers/maria.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Konkreetse DB Lib 2.0 draiveri wrapper-klass, mis omab tarkust,
|
|
3
|
+
a) kuidas ühendusprofiili andmetest teha ühenduseks vajalik struktuur ja ühendust võtta
|
|
4
|
+
b) kuidas RDBM draiveri spetsiifilised vead teisendada meie universaalseteks vigadeks
|
|
5
|
+
(selleks, et kasutaja ei peaks mooduleid importima ega urgitsema teostaja detaildies)
|
|
6
|
+
'''
|
|
7
|
+
|
|
8
|
+
from dbpoint.datadriver import DataDriverGeneric # ülemklass (fn-id run, disconnect)
|
|
9
|
+
#import datacontroller.datacontroller_exceptions as unified
|
|
10
|
+
from dbpoint import logging
|
|
11
|
+
|
|
12
|
+
class DataDriver(DataDriverGeneric):
|
|
13
|
+
|
|
14
|
+
map_of_exceptions = {
|
|
15
|
+
'SyntaxError' : Exception # unified.DataDriverSyntaxException
|
|
16
|
+
, 'UndefinedTable' : Exception # unified.DataDriverUndefinedTable
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
def me(self) -> str:
|
|
20
|
+
return 'MARIA'
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def connect(self, profile: dict) -> int:
|
|
24
|
+
|
|
25
|
+
if self.conn is not None: # kui ühendus on olemas, siis ei võta ühendust
|
|
26
|
+
return 0
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
import mariadb # https://github.com/sqlanywhere/sqlanydb/blob/master/sqlanydb.py
|
|
30
|
+
except ImportError:
|
|
31
|
+
raise Exception(f"Driver package not installed. Do pip install mariadb")
|
|
32
|
+
#raise unified.DataDriverNotInstalled('tee: pip install mariadb')
|
|
33
|
+
|
|
34
|
+
conn_dict = self.profile_to_dict(profile)
|
|
35
|
+
self.conn = mariadb.connect(**conn_dict)
|
|
36
|
+
self.conn.autocommit = False # mysql/mariadb iseärasus (default on true)
|
|
37
|
+
#print('now MariaDB is connected')
|
|
38
|
+
if profile.get('version', 10) < 10: # vaikeversioonina eeldame uuemat, nt 8. aga kuni mysql ver 5-ni oli varjestaja kaldkriips (mitte ülakoma)
|
|
39
|
+
self.apostrophe_escape = '\\'
|
|
40
|
+
#self.error = mariadb.Error
|
|
41
|
+
return 1
|
|
42
|
+
|
|
43
|
+
def profile_to_dict(self, profile: dict) -> dict:
|
|
44
|
+
|
|
45
|
+
conn_args = {}
|
|
46
|
+
|
|
47
|
+
conn_args['user'] = profile.get('username', 'xxxxx')
|
|
48
|
+
conn_args['password'] = profile.get('password', 'xxxxx')
|
|
49
|
+
conn_args['database'] = profile.get('database', 'xxxxx')
|
|
50
|
+
conn_args['host'] = profile.get('host', 'localhost')
|
|
51
|
+
conn_args['port'] = profile.get('port', 3306)
|
|
52
|
+
|
|
53
|
+
return conn_args
|
|
54
|
+
|
dbpoint/drivers/mssql.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Konkreetse DB Lib 2.0 draiveri wrapper-klass, mis omab tarkust,
|
|
3
|
+
a) kuidas ühendusprofiili andmetest teha ühenduseks vajalik struktuur ja ühendust võtta
|
|
4
|
+
b) kuidas RDBM draiveri spetsiifilised vead teisendada meie universaalseteks vigadeks
|
|
5
|
+
(selleks, et kasutaja ei peaks mooduleid importima ega urgitsema teostaja detaildies)
|
|
6
|
+
'''
|
|
7
|
+
|
|
8
|
+
from dbpoint.datadriver import DataDriverGeneric # ülemklass (fn-id run, disconnect)
|
|
9
|
+
#import datacontroller.datacontroller_exceptions as unified
|
|
10
|
+
from dbpoint import logging
|
|
11
|
+
|
|
12
|
+
class DataDriver(DataDriverGeneric):
|
|
13
|
+
|
|
14
|
+
map_of_exceptions = {
|
|
15
|
+
'SyntaxError' : Exception # unified.DataDriverSyntaxException
|
|
16
|
+
, 'UndefinedTable' : Exception # unified.DataDriverUndefinedTable
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
def me(self) -> str:
|
|
20
|
+
return 'MSSQL'
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def connect(self, profile: dict) -> int:
|
|
24
|
+
|
|
25
|
+
if self.conn is not None: # kui ühendus on olemas, siis ei võta ühendust
|
|
26
|
+
return 0
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
import pymssql # NB! pidav olema ühilduv kuni python 3.8 (seega kasutamisel olla ettevaatlik, vajadusel läbi ODBC)
|
|
30
|
+
except ImportError:
|
|
31
|
+
#raise unified.DataDriverNotInstalled('tee: pip install pymssql')
|
|
32
|
+
raise Exception(f"Driver package not installed. Do pip install pymssql")
|
|
33
|
+
|
|
34
|
+
conn_dict = self.profile_to_dict(profile)
|
|
35
|
+
self.conn = pymssql.connect(**conn_dict)
|
|
36
|
+
#self.error = pymssql.Error
|
|
37
|
+
return 1
|
|
38
|
+
|
|
39
|
+
def profile_to_dict(self, profile: dict) -> dict:
|
|
40
|
+
|
|
41
|
+
conn_args = {}
|
|
42
|
+
|
|
43
|
+
conn_args['database'] = profile.get('database', 'xxxxx').strip("/") # kaldkriipsude eemaldamise vajadus tulenes vist ühest teisest sisendist ja nüüd üle?
|
|
44
|
+
|
|
45
|
+
# MS SQL host on host+port
|
|
46
|
+
host = profile.get('host', 'localhost')
|
|
47
|
+
port = profile.get('port', 1433)
|
|
48
|
+
conn_args['host'] = f"{host}:{port}" # koos pordiga!
|
|
49
|
+
|
|
50
|
+
# Trusted_Connection=yes jaoks anname ilma kasutajanimeta-paroolita (ei toimi linuxis)
|
|
51
|
+
# trusted connection tähendab, et baasi kasutajaks on windowsi kasutaja
|
|
52
|
+
if profile.get('password', '') != '': # ja lepime endaga kokku, et kui konfis pole parooli antud, siis ongi soov trusted connection järele
|
|
53
|
+
conn_args['user'] = profile.get('username', 'xxxxx')
|
|
54
|
+
conn_args['password'] = profile.get('password', 'xxxxx')
|
|
55
|
+
|
|
56
|
+
return conn_args
|
|
57
|
+
|
dbpoint/drivers/odbc.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Konkreetse DB Lib 2.0 draiveri wrapper-klass, mis omab tarkust,
|
|
3
|
+
a) kuidas ühendusprofiili andmetest teha ühenduseks vajalik struktuur ja ühendust võtta
|
|
4
|
+
b) kuidas RDBM draiveri spetsiifilised vead teisendada meie universaalseteks vigadeks
|
|
5
|
+
(selleks, et kasutaja ei peaks mooduleid importima ega urgitsema teostaja detaildies)
|
|
6
|
+
'''
|
|
7
|
+
|
|
8
|
+
from dbpoint.datadriver import DataDriverGeneric # ülemklass (fn-id run, disconnect)
|
|
9
|
+
#import datacontroller.datacontroller_exceptions as unified
|
|
10
|
+
from dbpoint import logging
|
|
11
|
+
|
|
12
|
+
class DataDriver(DataDriverGeneric):
|
|
13
|
+
|
|
14
|
+
map_of_exceptions = {
|
|
15
|
+
'SyntaxError' : Exception # unified.DataDriverSyntaxException
|
|
16
|
+
, 'UndefinedTable' : Exception # unified.DataDriverUndefinedTable
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
def me(self) -> str:
|
|
20
|
+
return 'ODBC'
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def connect(self, profile: dict) -> int:
|
|
24
|
+
|
|
25
|
+
if self.conn is not None: # kui ühendus on olemas, siis ei võta ühendust
|
|
26
|
+
return 0
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
import pyodbc
|
|
30
|
+
except ImportError:
|
|
31
|
+
#raise unified.DataDriverNotInstalled('tee: pip install pyodbc')
|
|
32
|
+
raise Exception(f"Driver package not installed. Do pip install pyodbc")
|
|
33
|
+
|
|
34
|
+
conn_string = self.profile_to_string(profile) # https://www.connectionstrings.com/
|
|
35
|
+
self.conn = pyodbc.connect(**conn_string)
|
|
36
|
+
#self.error = pyodbc.Error
|
|
37
|
+
return 1
|
|
38
|
+
|
|
39
|
+
def profile_to_string(self, profile: dict) -> str:
|
|
40
|
+
|
|
41
|
+
# anname eelistuse valmis tehtud (draiverile sobivale) connect-stringile, kui see on olemas.
|
|
42
|
+
# PS. Connectstring peab vastama pigem ODBC draiverile (kui tootele), sest see suhtleb edasi tootega
|
|
43
|
+
|
|
44
|
+
if 'conn_string' in profile:
|
|
45
|
+
return profile['conn_string']
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
conn_str = ''
|
|
49
|
+
|
|
50
|
+
# kui polnud antud tervet, siis proovime juhutarkusega stringi kokku saada (ja see on üsna hiromantia, kuna kõigil tootjatel on omad kiiksud)
|
|
51
|
+
product = profile.get('dialect', 'general')
|
|
52
|
+
if product == 'general':
|
|
53
|
+
if 'driver' in profile: # nn keerukam lähenemine, mis võtab ainult ODBCINST draveri info ja lisame baasi, hosti, pordi ja kasutaja
|
|
54
|
+
conn_str = f"Driver={{{driver}}};Host={profile['host']};Database={profile['database']};Port={profile['port']};UID={profile['username']};PWD={profile['password']}"
|
|
55
|
+
else: # lühike variant, kus ODBC-s on DSN kirjeldatud ja lisame ainult kasutajanime ja parooli
|
|
56
|
+
conn_str = f"DSN={profile['database']};UID={profile['username']};PWD={profile['password']}"
|
|
57
|
+
if product in ('mssql', 'progress'):
|
|
58
|
+
if profile['password'] == '':
|
|
59
|
+
creds = "Trusted_Connection=yes;Uid=auth_window"
|
|
60
|
+
else:
|
|
61
|
+
creds = f"UID={profile['username']};PWD={profile['password']}"
|
|
62
|
+
|
|
63
|
+
if 'driver' in profile: # nn keerukam lähenemine, mis võtab ainult ODBCINST draveri info ja lisame baasi, hosti, pordi ja kasutaja
|
|
64
|
+
conn_str = f"Driver={{{driver}}};Host={conn_conf['host']};Database={conn_conf['database']};Port={conn_conf['port']};{creds}"
|
|
65
|
+
else: # lühike variant, kus ODBC-s on DSN kirjeldatud ja lisame ainult kasutajanime ja parooli
|
|
66
|
+
conn_str = f"DSN={conn_conf['database']};{creds}"
|
|
67
|
+
# võimalus lülitada Progressi fataalsena mõjuv hoiatus välja (parem oleks algandmed korda teha -- SQL ja APL veeru defid pole süngis)
|
|
68
|
+
# conn_str = conn_str + ';truncateTooLarge=output'
|
|
69
|
+
|
|
70
|
+
return conn_str
|
|
71
|
+
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Konkreetse DB Lib 2.0 draiveri wrapper-klass, mis omab tarkust,
|
|
3
|
+
a) kuidas ühendusprofiili andmetest teha ühenduseks vajalik struktuur ja ühendust võtta
|
|
4
|
+
b) kuidas RDBM draiveri spetsiifilised vead teisendada meie universaalseteks vigadeks
|
|
5
|
+
(selleks, et kasutaja ei peaks mooduleid importima ega urgitsema teostaja detaildies)
|
|
6
|
+
'''
|
|
7
|
+
|
|
8
|
+
from dbpoint.datadriver import DataDriverGeneric # ülemklass (fn-id run, disconnect)
|
|
9
|
+
#import datacontroller.datacontroller_exceptions as unified
|
|
10
|
+
from dbpoint import logging
|
|
11
|
+
|
|
12
|
+
class DataDriver(DataDriverGeneric):
|
|
13
|
+
|
|
14
|
+
map_of_exceptions = {
|
|
15
|
+
'SyntaxError': Exception # unified.DataDriverSyntaxException
|
|
16
|
+
, 'UndefinedTable' : Exception # unified.DataDriverUndefinedTable
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def me(self) -> str:
|
|
21
|
+
return 'ORA'
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def connect(self, profile: dict) -> int:
|
|
25
|
+
|
|
26
|
+
if self.conn is not None: # kui ühendus on olemas, siis ei võta ühendust
|
|
27
|
+
return 0
|
|
28
|
+
|
|
29
|
+
try:
|
|
30
|
+
import oracledb
|
|
31
|
+
except:
|
|
32
|
+
raise Exception(f"Driver package not installed. Do pip install oracledb")
|
|
33
|
+
#raise unified.DataDriverNotInstalled('tee: pip install oracledb')
|
|
34
|
+
|
|
35
|
+
conn_args = self.profile_to_dict(profile)
|
|
36
|
+
if profile.get('mode', 'thin') == 'thick':
|
|
37
|
+
import platform, os
|
|
38
|
+
oracle_instantclient_subdir = 'instantclient_11_2' # näib, et see võib olla ka suvaline uuem?!
|
|
39
|
+
# On Linux and related platforms, enable Thick mode by calling init_oracle_client() without passing a lib_dir parameter.
|
|
40
|
+
# oracledb.init_oracle_client() # samaväärne lib_dir=None, järgmine kood tegeleb (tervikluse mõttes) kõigega
|
|
41
|
+
oracle_instanclient_dir = None # default suitable for Linux
|
|
42
|
+
print(platform.system())
|
|
43
|
+
print(platform.machine())
|
|
44
|
+
if platform.system() == "Darwin" and platform.machine() == "x86_64": # macOS
|
|
45
|
+
oracle_instanclient_dir = "/".join([os.environ.get('HOME'), 'Downloads', oracle_instantclient_subdir]) # mac home ja Downloads (testimata loomulikult, ideena lihtsalt)
|
|
46
|
+
elif platform.system() == "Windows":
|
|
47
|
+
oracle_instanclient_dir = "C:\\" + "\\".join(['oracle', oracle_instantclient_subdir])
|
|
48
|
+
print(oracle_instantclient_subdir)
|
|
49
|
+
if not os.path.exists(oracle_instantclient_subdir):
|
|
50
|
+
print(f"kausta {oracle_instanclient_dir} ei ole olemas")
|
|
51
|
+
raise unified.DataDriverNotInstalledAdditional(f"Paksu Oracle jaoks pole InstanceClient asukohas {oracle_instanclient_dir}, midagi on vaja muuta")
|
|
52
|
+
oracledb.init_oracle_client(lib_dir=oracle_instanclient_dir) # paksu kliendi jaoks vaja initsialiseerida
|
|
53
|
+
|
|
54
|
+
self.conn = oracledb.connect(**conn_args) # nimelised argumendid ehk kwargs on dict ja viidata vaja kahe tärniga
|
|
55
|
+
#self.error = oracledb.Error
|
|
56
|
+
return 1
|
|
57
|
+
|
|
58
|
+
def profile_to_dict(self, profile: dict) -> dict:
|
|
59
|
+
|
|
60
|
+
conn_args = {}
|
|
61
|
+
conn_args['user'] = profile.get('username', 'xxxxx')
|
|
62
|
+
conn_args['password'] = profile.get('password', 'xxxxx')
|
|
63
|
+
# aga vb anda host port ja service eraldi?
|
|
64
|
+
#conn_args['dsn'] = profile.get('host', 'localhost') + "/" + profile.get('database', 'xxxxx') # nt 192.168.0.54/xe
|
|
65
|
+
conn_args['port'] = profile.get('port', 1521)
|
|
66
|
+
conn_args['host'] = profile.get('host', 'localhost')
|
|
67
|
+
conn_args['service_name'] = profile.get('database', 'xe')
|
|
68
|
+
conn_args['tcp_connect_timeout'] = 8 # default on 20 sek
|
|
69
|
+
return conn_args
|
|
70
|
+
|
|
71
|
+
'''
|
|
72
|
+
https://python-oracledb.readthedocs.io/en/latest/api_manual/module.html
|
|
73
|
+
|
|
74
|
+
oracledb.connect(dsn=None, pool=None, conn_class=None, params=None, user=None, proxy_user=None, password=None, newpassword=None, wallet_password=None
|
|
75
|
+
, access_token=None, host=None, port=1521, protocol='tcp', https_proxy=None, https_proxy_port=0
|
|
76
|
+
, service_name=None, sid=None, server_type=None, cclass=None, purity=oracledb.PURITY_DEFAULT, expire_time=0, retry_count=0, retry_delay=1
|
|
77
|
+
, tcp_connect_timeout=20.0, ssl_server_dn_match=True, ssl_server_cert_dn=None, wallet_location=None, events=False, externalauth=False
|
|
78
|
+
, mode=oracledb.AUTH_MODE_DEFAULT, disable_oob=False, stmtcachesize=oracledb.defaults.stmtcachesize, edition=None, tag=None, matchanytag=False
|
|
79
|
+
, config_dir=oracledb.defaults.config_dir, appcontext=[], shardingkey=[], supershardingkey=[], debug_jdwp=None, connection_id_prefix=None
|
|
80
|
+
, ssl_context=None, sdu=8192, pool_boundary=None, use_tcp_fast_open=False, ssl_version=None, handle=0)
|
|
81
|
+
|
|
82
|
+
'''
|
dbpoint/drivers/pg.py
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Konkreetse DB Lib 2.0 draiveri wrapper-klass, mis omab tarkust,
|
|
3
|
+
a) kuidas ühendusprofiili andmetest teha ühenduseks vajalik struktuur ja ühendust võtta
|
|
4
|
+
b) kuidas RDBM draiveri spetsiifilised vead teisendada meie universaalseteks vigadeks
|
|
5
|
+
(selleks, et kasutaja ei peaks mooduleid importima ega urgitsema teostaja detaildies)
|
|
6
|
+
'''
|
|
7
|
+
|
|
8
|
+
from dbpoint.datadriver import DataDriverGeneric # ülemklass (fn-id run, disconnect)
|
|
9
|
+
#import datacontroller.datacontroller_exceptions as unified
|
|
10
|
+
#from dbpoint import logging
|
|
11
|
+
|
|
12
|
+
class DataDriver(DataDriverGeneric):
|
|
13
|
+
|
|
14
|
+
map_of_exceptions = {
|
|
15
|
+
'SyntaxError': Exception # unified.DataDriverSyntaxException
|
|
16
|
+
, 'UndefinedTable' : Exception # unified.DataDriverUndefinedTable
|
|
17
|
+
, 'InvalidSchemaName' : Exception # unified.DataDriverInvalidSchemaName
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
type_mapper = {
|
|
21
|
+
20 : 'BIGINT'
|
|
22
|
+
, 1184 : 'TIMESTAMPTZ'
|
|
23
|
+
, 23 : 'INT'
|
|
24
|
+
, 3802 : 'JSONB' #(jsonb tehakse pythoni json stringiks, kus on ülakomad jutumärkide asemel ja ei saa tagasi kirjutada)
|
|
25
|
+
, 16 : 'BOOLEAN'
|
|
26
|
+
, 1043 : 'TEXT' # varchar tasub niikuinii pikemaks teha, lisaks on tüütu saada universaalselt kätte pikkust
|
|
27
|
+
, 25 :'TEXT'
|
|
28
|
+
, 1700 : 'NUMERIC'
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def me(self) -> str:
|
|
33
|
+
return 'PG'
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def connect(self, profile: dict) -> int:
|
|
37
|
+
|
|
38
|
+
if self.conn is not None: # kui ühendus on olemas, siis ei võta ühendust
|
|
39
|
+
return 0
|
|
40
|
+
|
|
41
|
+
try:
|
|
42
|
+
import psycopg2
|
|
43
|
+
except:
|
|
44
|
+
# raise -1
|
|
45
|
+
#raise unified.DataDriverNotInstalled('tee: pip install psycopg2')
|
|
46
|
+
raise Exception(f"Driver package not installed. Do pip install psycopg2")
|
|
47
|
+
|
|
48
|
+
# receive everything in the postgres string representation
|
|
49
|
+
#psycopg2.extensions.string_types.clear() # SEE POLE HEA MÕTE! JSONB tüübiga on suur jama, ja vb on ka date/time omadega
|
|
50
|
+
bad_keys = [199, 114, 3802, 3807, 16, 1114, 1184, 1083, 1266, 1082, 704, 1186]
|
|
51
|
+
for bad_key in bad_keys:
|
|
52
|
+
if bad_key in psycopg2.extensions.string_types: # teisel sama draiveriga ühendusel
|
|
53
|
+
del psycopg2.extensions.string_types[bad_key]
|
|
54
|
+
#for k in psycopg2.extensions.string_types.keys():
|
|
55
|
+
# print(f"{k} = {psycopg2.extensions.string_types[k]}")
|
|
56
|
+
|
|
57
|
+
conn_url = self.profile_to_url(profile)
|
|
58
|
+
self.conn: psycopg2.extensions.connection = psycopg2.connect(conn_url) # on error return -1?
|
|
59
|
+
#self.error = psycopg2.Error
|
|
60
|
+
return 1
|
|
61
|
+
|
|
62
|
+
def profile_to_url(self, profile: dict) -> str:
|
|
63
|
+
params = [] # küsimärgi-tagune (omavahel eraldatud ampersandiga)
|
|
64
|
+
#if 'appname' in profile:
|
|
65
|
+
# params.append(f"application_name={profile['appname']}")
|
|
66
|
+
|
|
67
|
+
if 'extra' in profile:
|
|
68
|
+
for key, value in profile['extra'].items(): # FIXME -- make it more secure!!!
|
|
69
|
+
params.append(f"{key}={value}")
|
|
70
|
+
|
|
71
|
+
parts = [] # maksimaalselt 2 osa
|
|
72
|
+
parts.append(f"postgresql://{profile['username']}:{profile['password']}@{profile['host']}:{profile['port']}/{profile['database']}")
|
|
73
|
+
if(params):
|
|
74
|
+
parts.append('&'.join(params))
|
|
75
|
+
|
|
76
|
+
connection_url = '?'.join(parts)
|
|
77
|
+
if profile.get('debug', False):
|
|
78
|
+
print(connection_url) # ohtlik!
|
|
79
|
+
return connection_url
|
|
80
|
+
|
|
81
|
+
def escape(self, cell_value, data_class, needs_escape):
|
|
82
|
+
'''
|
|
83
|
+
Postgres jaoks üle kirjutatud funktsioon, et saaks topelt-dollar ümbritsemise meetodiga lahendada paljud murekohad
|
|
84
|
+
Ja juhusk, kui algväärtuses on topelt-dollar sees, siis proovime midagi muud
|
|
85
|
+
(paha on ainult see, et praegune algoritm proovib kõik läbi selmet esimese mitteleidumise korral rõõmustada ja katkestada)
|
|
86
|
+
'''
|
|
87
|
+
if cell_value is None:
|
|
88
|
+
return 'NULL'
|
|
89
|
+
if data_class == 'BOOLEAN':
|
|
90
|
+
return 'TRUE' if cell_value == 't' else 'FALSE'
|
|
91
|
+
if needs_escape:
|
|
92
|
+
# uue märgistuse abil ümbritsemise meetod (võtame kandidaatidest esimese, mida ei leidu väärtuses)
|
|
93
|
+
surround = [surround_candidate for surround_candidate in ['$xx$', '$xiyaz$', '$sUvaLinE$'] if surround_candidate not in str(cell_value)] [0]
|
|
94
|
+
return f"{surround}{cell_value}{surround}"
|
|
95
|
+
# vs liht-asendamise meetod
|
|
96
|
+
# return (str(cell)).replace("'", self.apostrophe_escape)
|
|
97
|
+
else:
|
|
98
|
+
return f"{cell_value}"
|
|
99
|
+
|
|
100
|
+
def from_file(self, file_path: str, **kwargs) -> int:
|
|
101
|
+
import os
|
|
102
|
+
if not os.path.file_exists(file_path):
|
|
103
|
+
raise Exception # fixme, parem valida
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
#curTrg = self.getCursor(idx2, True) # kui ühendus on aegunud, siis loob uue ühenduse
|
|
107
|
+
#curTrg.copy_expert(trgSql, file_handler, size=self.bufferSize)
|
|
108
|
+
#file_handler.close()
|
|
109
|
+
#curTrg.close()
|
|
110
|
+
#self.doCommit(idx2, True) # self.conns[idx2]['conn'].commit()
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def to_file(self, query: str, file_path: str, **kwargs) -> int:
|
|
115
|
+
|
|
116
|
+
universal = False
|
|
117
|
+
|
|
118
|
+
# kaks lahendusteed: universaalne (teeme päringu ja kirjutame faili tehes ise kõik varjestused) ja postgre pakutav copy_expert
|
|
119
|
+
if universal:
|
|
120
|
+
try:
|
|
121
|
+
file_handler = open(file_path, 'w') # avada lugemiseks ('w' kirjutab üle, kui fail olemas)
|
|
122
|
+
except Exception as e1:
|
|
123
|
+
# siia logimine?
|
|
124
|
+
number_of_rows = -1 # ??? raise ju parem?!
|
|
125
|
+
...
|
|
126
|
+
raise e1
|
|
127
|
+
|
|
128
|
+
rs = self.run(query, True) # this loads the entire table into memory - which will crash your machine for a big table!
|
|
129
|
+
number_of_rows = 0
|
|
130
|
+
for row in rs:
|
|
131
|
+
file_handler.write("\t".join([str(cell) for cell in row]) + "\n")
|
|
132
|
+
number_of_rows += 1
|
|
133
|
+
|
|
134
|
+
file_handler.close()
|
|
135
|
+
|
|
136
|
+
else:
|
|
137
|
+
number_of_rows = self.to_file_postgres(query, file_path)
|
|
138
|
+
|
|
139
|
+
return number_of_rows
|
|
140
|
+
|
|
141
|
+
def to_file_postgres(self, query: str, file_path: str, **kwargs) -> int:
|
|
142
|
+
buffer_size = 2**14 # 16384 (vaikimisi on 8192)
|
|
143
|
+
control = """WITH (FORMAT 'csv', DELIMITER E'\t', NULL '(NULL)', QUOTE '\"', ESCAPE '\\', ENCODING 'utf8')"""
|
|
144
|
+
|
|
145
|
+
try:
|
|
146
|
+
file_handler = open(file_path, 'w') # avada lugemiseks ('w' kirjutab üle, kui fail olemas)
|
|
147
|
+
except Exception as e1:
|
|
148
|
+
...
|
|
149
|
+
raise e1
|
|
150
|
+
|
|
151
|
+
cur = self.conn.cursor()
|
|
152
|
+
copy_to_expression = f"COPY ({query}\n) TO STDOUT {control}"
|
|
153
|
+
try:
|
|
154
|
+
cur.copy_expert(copy_to_expression, file_handler, size=buffer_size)
|
|
155
|
+
number_of_rows = cur.rowcount
|
|
156
|
+
self.conn.commit()
|
|
157
|
+
except Exception as e1:
|
|
158
|
+
print(e1)
|
|
159
|
+
self.conn.rollback()
|
|
160
|
+
number_of_rows = -1
|
|
161
|
+
finally:
|
|
162
|
+
cur.close()
|
|
163
|
+
|
|
164
|
+
file_handler.close()
|
|
165
|
+
# väike mure: kuidas me teame, mitu kirjet kirjutati
|
|
166
|
+
return number_of_rows
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
|
dbpoint/perks.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
import sys
|
|
4
|
+
import psutil
|
|
5
|
+
|
|
6
|
+
def reconf_logging(solution_name, level=logging.DEBUG):
|
|
7
|
+
"""
|
|
8
|
+
Once you can redefine logging globally -- before first use!
|
|
9
|
+
Let's add timestamp in begginning (if no time is displayed by default (you cannot tell the bees))
|
|
10
|
+
And let's print out everything, incl debug-level
|
|
11
|
+
FIXME: make these adaptions controlled by some env.var (not by Conf class, which may itself already use logging)
|
|
12
|
+
|
|
13
|
+
https://docs.python.org/3/library/logging.html#logrecord-attributes
|
|
14
|
+
%(message)s
|
|
15
|
+
%(asctime)s
|
|
16
|
+
%(lineno)d
|
|
17
|
+
%(module)s
|
|
18
|
+
%(name)s
|
|
19
|
+
%(levelname)s
|
|
20
|
+
"""
|
|
21
|
+
frm = "%(asctime)s:%(levelname)8s:%(lineno)5d:%(module)-15s:%(name)-6s:%(message)s"
|
|
22
|
+
|
|
23
|
+
if '%(module)' not in frm:
|
|
24
|
+
frm = ' %(module)-12s:' + frm
|
|
25
|
+
if '%(lineno)' not in frm:
|
|
26
|
+
frm = '%(lineno)4d:' + frm
|
|
27
|
+
if '%(asctime)' not in frm:
|
|
28
|
+
frm = '%(asctime)s:' + frm
|
|
29
|
+
logging.basicConfig(format=frm, level=level, force=True)
|
|
30
|
+
|
|
31
|
+
def get_custom_logger(solution_name):
|
|
32
|
+
reconf_logging(solution_name, logging.INFO)
|
|
33
|
+
return logging.getLogger(solution_name)
|
|
34
|
+
|
|
35
|
+
### PURE FUNCTIONS ###
|
|
36
|
+
|
|
37
|
+
def halt(code: int, message: str):
|
|
38
|
+
"""
|
|
39
|
+
Shortcut (for serious error situations) for logging message and do quick exit
|
|
40
|
+
Be aware: for Airflow any return code (incl 0) means that task is failed (dont use halt on normal flow!)
|
|
41
|
+
For other running systems (bash script etc) You can control flow using exit code (regular end is code 0 automatically)
|
|
42
|
+
Exit code must by between 0 and 255. Any other cases we will map to 255
|
|
43
|
+
"""
|
|
44
|
+
code = code if code >= 0 and code <= 255 else 255
|
|
45
|
+
logging.error(f"{code} {message}")
|
|
46
|
+
sys.exit(code)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def mem_free_now() -> float:
|
|
50
|
+
"""
|
|
51
|
+
Returns free memory in megabytes as float
|
|
52
|
+
"""
|
|
53
|
+
return psutil.virtual_memory().available / (1024 * 1024)
|
|
54
|
+
|