SQLDbWrpr 5.0.1__py3-none-any.whl → 5.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sqldbwrpr/sqldbwrpr.py CHANGED
@@ -1,8 +1,8 @@
1
- """Wrapper for importing CSV and Text files into MySQL and Postgresql.
1
+ """Database wrapper utilities for MySQL and PostgreSQL.
2
2
 
3
- Only the MySQL implementation is working. The MSSQL ODBC implementation
4
- is giving me a lot of shit with the connection and I gave up for now to
5
- get it working.
3
+ SqlDbWrpr creates databases and tables from either the legacy dictionary
4
+ structure or SQLAlchemy metadata, then imports and exports CSV data through a
5
+ small DB-API based wrapper.
6
6
  """
7
7
 
8
8
  import datetime
@@ -14,21 +14,18 @@ import csvwrpr
14
14
  import displayfx
15
15
  import fixdate
16
16
  import mysql.connector
17
- import pyodbc
17
+ import psycopg
18
18
  from beetools import msg as bm
19
- from mysql.connector import Error
20
19
  from mysql.connector import errorcode
20
+ from psycopg import sql as psycopg_sql
21
21
 
22
- # from pathlib import Path
23
22
 
24
-
25
- # _PROJ_DESC = __doc__.split("\n")[0]
26
- # _PROJ_PATH = Path(__file__)
27
- # _PROJ_NAME = _PROJ_PATH.stem
23
+ class SchemaSourceError(ValueError):
24
+ """Raised when no supported database schema source is supplied."""
28
25
 
29
26
 
30
27
  class SQLDbWrpr:
31
- """This module creates a wrapper for the MySql database."""
28
+ """Base wrapper for schema creation and CSV import/export operations."""
32
29
 
33
30
  def __init__(
34
31
  self,
@@ -38,6 +35,8 @@ class SQLDbWrpr:
38
35
  p_recreate_db=False,
39
36
  p_db_name="",
40
37
  p_db_structure=None,
38
+ p_sqlalchemy_base=None,
39
+ p_sqlalchemy_metadata=None,
41
40
  p_batch_size=10000,
42
41
  p_bar_len=50,
43
42
  p_msg_width=50,
@@ -47,18 +46,25 @@ class SQLDbWrpr:
47
46
  p_ssl_key=None,
48
47
  p_ssl_cert=None,
49
48
  ):
50
- """Create database with supplied structure and return a connector to the database
49
+ """Initialise a common wrapper state and resolve the database schema.
50
+
51
+ A legacy `p_db_structure` dictionary takes precedence. If it is not
52
+ supplied, SQLAlchemy metadata is read from `p_sqlalchemy_metadata` or
53
+ from `p_sqlalchemy_base.metadata`. A `SchemaSourceError` is raised when
54
+ none of those schema sources are available.
51
55
 
52
56
  Parameters
53
- - p_host_name = Host to connect to
54
- - p_user_name = User name for connection
55
- - p_password = paswword of user
56
- - ReCreate = Recresate the database or connect to existing database
57
- - db_name =
58
- - table_details: Details of the tables to be created
59
- - batch_size: Bulk data will be managed by batch_size to commit
60
- - p_bar_len: Length for the progress bar
61
- - p_msg_width: Width of message before progress bar
57
+ - p_host_name: Host to connect to.
58
+ - p_user_name: User name for the connection.
59
+ - p_password: Password for the connection.
60
+ - p_recreate_db: Recreate the database when the backend supports it.
61
+ - p_db_name: Database name.
62
+ - p_db_structure: Legacy SqlDbWrpr table/field dictionary.
63
+ - p_sqlalchemy_base: Declarative base that exposes SQLAlchemy metadata.
64
+ - p_sqlalchemy_metadata: SQLAlchemy MetaData instance.
65
+ - p_batch_size: Number of rows committed per import batch.
66
+ - p_bar_len: Length of progress bars.
67
+ - p_msg_width: Width of progress messages.
62
68
  """
63
69
  self.logger_name = __name__
64
70
  self.logger = logging.getLogger(self.logger_name)
@@ -70,10 +76,12 @@ class SQLDbWrpr:
70
76
  self.conn = None
71
77
  self.cur = None
72
78
  self.db_name = p_db_name
73
- if p_db_structure:
74
- self.db_structure = p_db_structure
75
- else:
76
- self.db_structure = {}
79
+ self.db_structure = self.resolve_db_structure(
80
+ p_db_structure=p_db_structure,
81
+ p_sqlalchemy_base=p_sqlalchemy_base,
82
+ p_sqlalchemy_metadata=p_sqlalchemy_metadata,
83
+ )
84
+ self.db_error = mysql.connector.Error
77
85
  self.delimiter = ","
78
86
  self.fkey_ref_act = {
79
87
  "C": "CASCADE",
@@ -82,7 +90,9 @@ class SQLDbWrpr:
82
90
  "N": "SET NULL",
83
91
  }
84
92
  self.host_name = p_host_name
93
+ self.identifier_quote = ""
85
94
  self.idx_type = {"U": "UNIQUE", "F": "FULLTEXT", "S": "SPATIAL"}
95
+ self.inline_indexes = True
86
96
  self.msg_width = p_msg_width
87
97
  self.non_char_fields = {}
88
98
  self._password = p_password
@@ -95,10 +105,75 @@ class SQLDbWrpr:
95
105
  self.db_port = p_db_port
96
106
 
97
107
  def close(self):
98
- """Close the connention"""
108
+ """Close the active database connection."""
99
109
  if self.conn:
100
110
  self.conn.close()
101
111
 
112
+ def build_column_sql(self, p_field_name, p_field_type, p_field_params, p_field_comment):
113
+ """Build SQL for one column in a CREATE TABLE statement."""
114
+ sql_str = f"{self.quote_identifier(p_field_name)} {self.render_field_type(p_field_type, p_field_params)}"
115
+ if p_field_params["AI"] == "Y":
116
+ sql_str += " AUTO_INCREMENT"
117
+ if p_field_params["UN"] == "Y" and p_field_params["AI"] != "Y":
118
+ sql_str += " UNSIGNED"
119
+ if p_field_params["NN"] == "Y":
120
+ sql_str += " NOT NULL"
121
+ if p_field_params["ZF"] == "Y":
122
+ sql_str += " ZEROFILL"
123
+ if p_field_params["DEF"]:
124
+ sql_str += self.render_default_sql(p_field_type, p_field_params["DEF"])
125
+ if p_field_comment:
126
+ sql_str += f' COMMENT "{p_field_comment}"'
127
+ return sql_str
128
+
129
+ def build_index_sql(self, p_table_name, p_idx_name, p_idx_fields, p_unique=False):
130
+ """Build an index clause for the active SQL dialect."""
131
+ idx_type = "UNIQUE INDEX" if p_unique else "INDEX"
132
+ index_fields = []
133
+ for field_det in p_idx_fields:
134
+ index_fields.append(f"{self.quote_identifier(field_det[0])} {self.sort_order[field_det[2]]}")
135
+ return f"{idx_type} {self.quote_identifier(p_idx_name)} ({','.join(index_fields)}) VISIBLE, "
136
+
137
+ def build_insert_sql(self, p_table_name, p_header, p_replace=False):
138
+ """Build SQL for inserting rows into a table."""
139
+ insert_command = "REPLACE" if p_replace else "INSERT"
140
+ return "{} INTO {} ({}) VALUES ({})".format(
141
+ insert_command,
142
+ self.quote_identifier(p_table_name),
143
+ ",".join([self.quote_identifier(str(x)) for x in p_header]),
144
+ ",".join([self.param_placeholder() for x in range(len(p_header))]),
145
+ )
146
+
147
+ def param_placeholder(self):
148
+ """Return the DB-API placeholder used by this backend."""
149
+ return "%s"
150
+
151
+ def quote_identifier(self, p_identifier):
152
+ """Quote an SQL identifier for this backend."""
153
+ if not self.identifier_quote:
154
+ return str(p_identifier)
155
+ escaped_identifier = str(p_identifier).replace(self.identifier_quote, self.identifier_quote * 2)
156
+ return f"{self.identifier_quote}{escaped_identifier}{self.identifier_quote}"
157
+
158
+ def quote_identifier_list(self, p_identifiers):
159
+ """Quote and join SQL identifiers."""
160
+ return ",".join([self.quote_identifier(p_identifier) for p_identifier in p_identifiers])
161
+
162
+ def render_default_sql(self, p_field_type, p_default_value):
163
+ """Render a column default expression."""
164
+ if p_field_type[0] == "varchar" or p_field_type[0] == "char":
165
+ return f' DEFAULT "{p_default_value}"'
166
+ return f" DEFAULT {p_default_value}"
167
+
168
+ def render_field_type(self, p_field_type, p_field_params):
169
+ """Render a field type for this backend."""
170
+ field_type = p_field_type[0]
171
+ if field_type == "varchar" or field_type == "char":
172
+ return f"{field_type} ({str(p_field_type[1])})"
173
+ if field_type == "decimal":
174
+ return f"{field_type}({str(p_field_type[1])}, {str(p_field_type[2])})"
175
+ return field_type
176
+
102
177
  def create_db(self):
103
178
  """Create the database according to self.db_structure."""
104
179
  self.cur.execute("SHOW DATABASES")
@@ -108,7 +183,7 @@ class SQLDbWrpr:
108
183
  try:
109
184
  self.cur.execute(f"DROP DATABASE {self.db_name}")
110
185
  self.conn.commit()
111
- except mysql.connector.Error as err:
186
+ except self.db_error as err:
112
187
  self._print_err_msg(err, "Could not drop the database")
113
188
  self.close()
114
189
  sys.exit()
@@ -117,23 +192,23 @@ class SQLDbWrpr:
117
192
  self.conn.commit()
118
193
  self.cur.execute(f"USE {self.db_name}")
119
194
  self.conn.commit()
120
- except mysql.connector.Error as err:
195
+ except self.db_error as err:
121
196
  self._print_err_msg(err, "Could not create the database")
122
197
  self.close()
123
198
  sys.exit()
124
199
  return True
125
200
 
126
201
  def create_tables(self):
127
- """Create db tables from MySQL.table_details dict"""
202
+ """Create database tables, indexes, and constraints from the resolved schema."""
128
203
 
129
204
  def build_db(p_db_sql_str_set):
130
- """Description"""
205
+ """Execute the generated table or index SQL statements."""
131
206
  for sql_set in p_db_sql_str_set:
132
207
  try:
133
208
  self.cur.execute(sql_set[1])
134
209
  if self.silent:
135
210
  print(f"Created table = {sql_set[0]}")
136
- except mysql.connector.Error as err:
211
+ except self.db_error as err:
137
212
  print(f"Failed creating table = {sql_set[0]}: {err}\nForced termination of program")
138
213
  print(f"{sql_set[1]}")
139
214
  sys.exit()
@@ -147,7 +222,7 @@ class SQLDbWrpr:
147
222
  p_idx_set_up_list,
148
223
  p_constraint_set_up_list,
149
224
  ):
150
- """Description"""
225
+ """Combine column, primary-key, index, and constraint SQL."""
151
226
  table_set_up_str = p_table_set_up_str
152
227
  table_set_up_str += p_primary_key_str
153
228
  for idx_str in p_idx_set_up_list:
@@ -161,7 +236,7 @@ class SQLDbWrpr:
161
236
 
162
237
  def build_constraints(p_table_name):
163
238
  # noinspection PySingleQuotedDocstring
164
- """Description"""
239
+ """Build foreign-key constraint SQL for one table."""
165
240
  constraint_list = []
166
241
  fkey_nr_list = []
167
242
  for field_name in self.db_structure[p_table_name]:
@@ -173,9 +248,9 @@ class SQLDbWrpr:
173
248
  "CONSTRAINT fk_{}_{} FOREIGN KEY ({}) REFERENCES {} ({}) ON DELETE {} ON UPDATE {}, ".format(
174
249
  fkey["FKeyTable"],
175
250
  fkey["RefTable"],
176
- ".".join(fkey["FKeyFlds"]),
177
- fkey["RefTable"],
178
- ".".join(fkey["RefFields"]),
251
+ self.quote_identifier_list(fkey["FKeyFlds"]),
252
+ self.quote_identifier(fkey["RefTable"]),
253
+ self.quote_identifier_list(fkey["RefFields"]),
179
254
  self.fkey_ref_act[fkey["OnDelete"]],
180
255
  self.fkey_ref_act[fkey["OnUpdate"]],
181
256
  )
@@ -187,10 +262,10 @@ class SQLDbWrpr:
187
262
  # def build_constraints
188
263
 
189
264
  def build_all_indexes(p_table_name):
190
- """Description"""
265
+ """Build inline or post-create index SQL for one table."""
191
266
 
192
267
  # def build_primary_key_idx(p_table_name):
193
- # '''Description'''
268
+ # '''Build primary-key index SQL.'''
194
269
  # idx_name_list = []
195
270
  # idx_str_list = []
196
271
  # pkey = get_primary_key(p_table_name)
@@ -205,7 +280,7 @@ class SQLDbWrpr:
205
280
  # # end build_primary_key_idx
206
281
 
207
282
  def build_unique_key_idx(p_table_name, p_dx_name_list, p_idx_str_list):
208
- """Description"""
283
+ """Build grouped index definitions from legacy field metadata."""
209
284
  idx_list = {}
210
285
  idx_name_list = p_dx_name_list
211
286
  idx_str_list = p_idx_str_list
@@ -229,14 +304,16 @@ class SQLDbWrpr:
229
304
  idx_name = f"idx_{idx_name[:-1]}"
230
305
  if idx_name not in idx_name_list:
231
306
  idx_name_list.append(idx_name)
232
- if field_det[3] == "U":
233
- idx_str = f"{self.idx_type[field_det[3]]} INDEX {idx_name} ("
307
+ idx_str = self.build_index_sql(
308
+ p_table_name,
309
+ idx_name,
310
+ idx_instance_order,
311
+ field_det[3] == "U",
312
+ )
313
+ if self.inline_indexes:
314
+ idx_str_list.append(idx_str)
234
315
  else:
235
- idx_str = f"INDEX {idx_name} ("
236
- for field_det in idx_instance_order:
237
- idx_str += f"{field_det[0]} {self.sort_order[field_det[2]]}, "
238
- idx_str = idx_str[:-2] + ") VISIBLE, "
239
- idx_str_list.append(idx_str)
316
+ post_create_sql_set.append([idx_name, idx_str])
240
317
  return idx_str_list, idx_name_list
241
318
 
242
319
  # end build_unique_key_idx
@@ -249,47 +326,33 @@ class SQLDbWrpr:
249
326
  # def build_all_indexes
250
327
 
251
328
  def build_primary_key_sql_str(p_table_name):
252
- """Description"""
329
+ """Build the primary-key clause for one table."""
253
330
  primary_key_det = get_primary_key(p_table_name)
254
- sql_str = "PRIMARY KEY ({}), ".format(",".join(primary_key_det["Flds"]))
331
+ sql_str = "PRIMARY KEY ({}), ".format(self.quote_identifier_list(primary_key_det["Flds"]))
255
332
  return sql_str
256
333
 
257
334
  # def build_primary_key_sql_str
258
335
 
259
336
  def build_table_sql_str(p_table_name):
260
- """Description"""
261
- sql_str = f"CREATE TABLE {p_table_name} ("
337
+ """Build the CREATE TABLE prefix and column definitions."""
338
+ sql_str = f"CREATE TABLE {self.quote_identifier(p_table_name)} ("
262
339
  for field_name in self.db_structure[p_table_name]:
263
340
  field_type_st_ref = self.db_structure[p_table_name][field_name]["Type"]
264
341
  field_param_st_ref = self.db_structure[p_table_name][field_name]["Params"]
265
342
  field_comment_st_ref = self.db_structure[p_table_name][field_name]["Comment"]
266
- sql_str += f"{field_name} {field_type_st_ref[0]}"
267
- if field_type_st_ref[0] == "varchar" or field_type_st_ref[0] == "char":
268
- sql_str += f" ({str(field_type_st_ref[1])})"
269
- elif field_type_st_ref[0] == "decimal":
270
- sql_str += f"({str(field_type_st_ref[1])}, {str(field_type_st_ref[2])})"
271
- if field_param_st_ref["AI"] == "Y":
272
- sql_str += " AUTO_INCREMENT"
273
- if field_param_st_ref["UN"] == "Y" and field_param_st_ref["AI"] != "Y":
274
- sql_str += " UNSIGNED"
275
- if field_param_st_ref["NN"] == "Y":
276
- sql_str += " NOT NULL"
277
- if field_param_st_ref["ZF"] == "Y":
278
- sql_str += " ZEROFILL"
279
- if field_param_st_ref["DEF"]:
280
- if field_type_st_ref[0] == "varchar" or field_type_st_ref[0] == "char":
281
- sql_str += ' DEFAULT "{}"'.format(field_param_st_ref["DEF"])
282
- else:
283
- sql_str += " DEFAULT {}".format(field_param_st_ref["DEF"])
284
- if field_comment_st_ref:
285
- sql_str += f' COMMENT "{field_comment_st_ref}"'
343
+ sql_str += self.build_column_sql(
344
+ field_name,
345
+ field_type_st_ref,
346
+ field_param_st_ref,
347
+ field_comment_st_ref,
348
+ )
286
349
  sql_str += ", "
287
350
  return sql_str
288
351
 
289
352
  # end build_table_sql_str
290
353
 
291
354
  def get_foreign_key(p_table_name, p_field_name):
292
- """Description"""
355
+ """Return normalized foreign-key metadata for a field."""
293
356
  fkey = {
294
357
  "Present": False,
295
358
  "FKeyFlds": [],
@@ -327,7 +390,7 @@ class SQLDbWrpr:
327
390
  # end get_foreign_key
328
391
 
329
392
  def get_primary_key(p_table_name):
330
- """Description"""
393
+ """Return normalized primary-key metadata for a table."""
331
394
  pkey = {"Present": False, "Flds": (), "SortPairList": [], "SortPairStr": []}
332
395
  for field_name in self.db_structure[p_table_name]:
333
396
  pkey_field_det = self.db_structure[p_table_name][field_name]
@@ -351,7 +414,7 @@ class SQLDbWrpr:
351
414
  # end get_primary_key
352
415
 
353
416
  def order_table_build_list(p_db_sql_str_set, p_constraint_set_up_list):
354
- """Description"""
417
+ """Order table creation so referenced tables are created first."""
355
418
  db_sql_str_set = p_db_sql_str_set
356
419
  ordered = False
357
420
  while not ordered:
@@ -385,13 +448,13 @@ class SQLDbWrpr:
385
448
  # end order_table_build_list
386
449
 
387
450
  def structure_validation():
388
- """Description"""
451
+ """Validate schema relationships before SQL generation."""
389
452
 
390
453
  def check_pkey_fkey_overlap(p_remove_fkey_pkey__overlap=True):
391
- """Description"""
454
+ """Detect and optionally remove primary-key/foreign-key overlap."""
392
455
 
393
456
  def partial_overlap(p_fkey, p_pkey):
394
- """Description"""
457
+ """Return whether a foreign key partially overlaps a primary key."""
395
458
  is_overlap = False
396
459
  for field_name in p_fkey["FKeyFlds"]:
397
460
  if field_name in p_pkey["Flds"]:
@@ -401,7 +464,7 @@ class SQLDbWrpr:
401
464
  # end partial_overlap
402
465
 
403
466
  def remove_fkey(p_fkey):
404
- """Description"""
467
+ """Remove a foreign-key definition from all participating fields."""
405
468
  for field_name in self.db_structure[p_fkey["FKeyTable"]]:
406
469
  if self.db_structure[p_fkey["FKeyTable"]][field_name]["Params"]["FKey"]:
407
470
  if (
@@ -448,6 +511,7 @@ class SQLDbWrpr:
448
511
  # end structure_validation()
449
512
 
450
513
  success = True
514
+ post_create_sql_set = []
451
515
  structure_validation()
452
516
  table_set_up_list = ""
453
517
  idx_set_up_list = []
@@ -474,42 +538,32 @@ class SQLDbWrpr:
474
538
  pass
475
539
  db_sql_str_set = order_table_build_list(db_sql_str_set, constraint_set_up_list)
476
540
  build_db(db_sql_str_set)
541
+ build_db(post_create_sql_set)
477
542
  return success
478
543
 
479
544
  def create_users(self, p_admin_user, p_new_users):
480
- c_user_name = 0
481
- self.cur.execute("SELECT User, Host FROM mysql.user")
482
- curr_users = self.cur.fetchall()
483
- for user in p_new_users:
484
- if not user[c_user_name] in curr_users:
485
- try:
486
- self.cur.execute(
487
- "CREATE USER IF NOT EXISTS '{}'@'{}' IDENTIFIED BY '{}'".format(
488
- user[0], self.host_name, user[1]
489
- )
490
- )
491
- except mysql.connector.Error as err:
492
- self._print_err_msg(err, "Could not create user")
493
- self.close()
494
- sys.exit()
495
- self.conn.commit()
496
- self.success = True
545
+ """Create database users.
546
+
547
+ Concrete database backends must implement this because user management
548
+ SQL differs between databases.
549
+ """
550
+ raise NotImplementedError(f"{self.__class__.__name__} must implement create_users()")
497
551
 
498
552
  def delete_users(self, p_admin_user, p_del_users):
499
- c_user_name = 0
500
- # c_password = 1
501
- c_host = 2
502
- self.cur.execute("SELECT User FROM mysql.user")
503
- curr_users = [x[0] for x in self.cur.fetchall()]
504
- for user in p_del_users:
505
- if user[c_user_name] in curr_users:
506
- try:
507
- self.cur.execute(f"DROP USER '{user[c_user_name]}'@'{user[c_host]}'")
508
- except mysql.connector.Error as err:
509
- self._print_err_msg(err, "Could not delete user")
510
- self.close()
511
- sys.exit()
512
- self.success = True
553
+ """Delete database users.
554
+
555
+ Concrete database backends must implement this because user management
556
+ SQL differs between databases.
557
+ """
558
+ raise NotImplementedError(f"{self.__class__.__name__} must implement delete_users()")
559
+
560
+ def grant_rights(self, p_admin_user, p_user_rights):
561
+ """Grant database rights.
562
+
563
+ Concrete database backends must implement this because privilege syntax
564
+ differs between databases.
565
+ """
566
+ raise NotImplementedError(f"{self.__class__.__name__} must implement grant_rights()")
513
567
 
514
568
  def _err_broken_rec(self, p_sql_str, p_csv_db_slice):
515
569
  """Write broken record to logger"""
@@ -537,11 +591,11 @@ class SQLDbWrpr:
537
591
  """Export a table to a csv file
538
592
 
539
593
  Parameters
540
- - p_csv_path - Path name of the file to be exported
541
- - p_table_name = '' - Table name to export
542
- - p_delimeter = '|' - Field delimiter to use
594
+ - p_csv_path - Path name of the file to be exported
595
+ - p_table_name = '' - Table name to export
596
+ - p_delimeter = '|' - Field delimiter to use
543
597
  - p_strip_chars = '' - characters to strip from text
544
- - p__vol_size = 0 - Create a multiple volume export. p__vol_size is
598
+ - p__vol_size = 0 - Create a multiple-volume export. p__vol_size is
545
599
  the number of records per file. 0 will create
546
600
  only one volume.
547
601
  """
@@ -558,12 +612,13 @@ class SQLDbWrpr:
558
612
  file_name_list = []
559
613
  header = p_delimeter.join(self.db_structure[p_table_name])
560
614
  prim_key_sql_str = "SELECT "
561
- all_sql_str = "SELECT " + header.replace(p_delimeter, ",") + " FROM " + p_table_name + " WHERE "
615
+ select_header = self.quote_identifier_list(self.db_structure[p_table_name])
616
+ all_sql_str = "SELECT " + select_header + " FROM " + self.quote_identifier(p_table_name) + " WHERE "
562
617
  for i, field in enumerate(self.db_structure[p_table_name]):
563
618
  if self.db_structure[p_table_name][field]["Params"]["PrimaryKey"][0] == "Y":
564
- prim_key_sql_str += field + ", "
565
- all_sql_str += field + " = %s and "
566
- prim_key_sql_str = prim_key_sql_str[:-2] + " FROM " + p_table_name
619
+ prim_key_sql_str += self.quote_identifier(field) + ", "
620
+ all_sql_str += self.quote_identifier(field) + " = " + self.param_placeholder() + " and "
621
+ prim_key_sql_str = prim_key_sql_str[:-2] + " FROM " + self.quote_identifier(p_table_name)
567
622
  all_sql_str = all_sql_str[:-5]
568
623
  print(f"Collecting {p_table_name} table records")
569
624
  self.cur.execute(prim_key_sql_str)
@@ -626,7 +681,12 @@ class SQLDbWrpr:
626
681
  file_name_list.append(os.path.split(p_csv_path))
627
682
  if not p_sql_query:
628
683
  header = p_delimeter.join(self.db_structure[p_table_name])
629
- sql_str = "SELECT " + header.replace(p_delimeter, ",") + " FROM " + p_table_name
684
+ sql_str = (
685
+ "SELECT "
686
+ + self.quote_identifier_list(self.db_structure[p_table_name])
687
+ + " FROM "
688
+ + self.quote_identifier(p_table_name)
689
+ )
630
690
  else:
631
691
  header = p_delimeter.join(p_sql_query[0])
632
692
  sql_str = p_sql_query[1]
@@ -663,8 +723,8 @@ class SQLDbWrpr:
663
723
 
664
724
  file_name_list = None
665
725
  try:
666
- self.cur.execute("SELECT COUNT(*) FROM " + p_table_name)
667
- except mysql.connector.Error as err:
726
+ self.cur.execute("SELECT COUNT(*) FROM " + self.quote_identifier(p_table_name))
727
+ except self.db_error as err:
668
728
  print(f"Err mesg: {err.msg}")
669
729
  print(err.msg)
670
730
  else:
@@ -677,7 +737,7 @@ class SQLDbWrpr:
677
737
  return file_name_list
678
738
 
679
739
  def get_db_field_types(self):
680
- """Description"""
740
+ """Populate field type lookup lists used during import/export."""
681
741
  for p_table_name in self.db_structure:
682
742
  self.char_fields[p_table_name] = []
683
743
  self.non_char_fields[p_table_name] = []
@@ -690,40 +750,6 @@ class SQLDbWrpr:
690
750
  else:
691
751
  self.non_char_fields[p_table_name].append(field)
692
752
 
693
- def grant_rights(self, p_admin_user, p_user_rights):
694
- c_user_name = 0
695
- # c_password = 1
696
- c_host = 1
697
- c_db = 2
698
- c_table = 3
699
- c_rights = 4
700
- # success = True
701
- for right in p_user_rights:
702
- try:
703
- sql_str = "GRANT {} ON {}.{} TO '{}'@'{}'".format(
704
- ",".join(right[c_rights:]),
705
- right[c_db],
706
- right[c_table],
707
- right[c_user_name],
708
- right[c_host],
709
- )
710
- self.cur.execute(sql_str)
711
- self.conn.commit()
712
- sql_str = "GRANT {} ON {}.{} TO '{}'@'{}' WITH GRANT OPTION".format(
713
- ",".join(right[c_rights:]),
714
- right[c_db],
715
- right[c_table],
716
- right[c_user_name],
717
- right[c_host],
718
- )
719
- self.cur.execute(sql_str)
720
- self.conn.commit()
721
- except mysql.connector.Error as err:
722
- self._print_err_msg(err)
723
- self.close()
724
- sys.exit()
725
- self.success = True
726
-
727
753
  def import_csv(
728
754
  self,
729
755
  p_table_name,
@@ -737,41 +763,40 @@ class SQLDbWrpr:
737
763
  p_verbose=False,
738
764
  p_replace=False,
739
765
  ):
740
- """Import a csv file into a database table.
766
+ """Import CSV rows into a database table.
741
767
 
742
768
  Parameters
743
769
  - p_table_name
744
- Table name to import the csv data into
770
+ Table name to import the CSV data into
745
771
  - p_csv_file_name = ''
746
- Csv file name. Empty if structure contained in p_csv_db
772
+ CSV file name. Empty when rows are supplied in p_csv_db
747
773
  - p_key = ''
748
774
  Key used to insert in table
749
775
  - p_header = ''
750
- - Header of csv files
776
+ - Header of CSV files
751
777
  - p_del_head = ''
752
778
  - Delete the header
753
779
  - p_csv_db = ''
754
- - Contains the csv table in a structure and makes p_csv_file_name obsolete.
780
+ - Contains the CSV rows directly and makes p_csv_file_name obsolete.
755
781
  - p_csv_corr_str_file_name = ''
756
- - String that contains any strings that should be replace in the csv
757
- file before parsing
782
+ - File containing string replacements to apply before parsing
758
783
  - p_vol_type = 'Multi'
759
784
  - Multi - Read multiple volume
760
785
  - Single - Read single file
761
786
  - p_verbose = False
762
- - Determine if there are any output to screen
787
+ - Determine whether progress output is written to screen
763
788
  - debug = False
764
- - Switch debug on
789
+ - Switch debug output on
765
790
  - p_replace = False
766
791
  - False - INSERT into database
767
792
  - True - REPLACE into database
768
793
  """
769
794
 
770
795
  def import_volume(p_csv_db, p_header, p_verbose):
771
- """Description"""
796
+ """Prepare and write one in-memory CSV volume to the active table."""
772
797
 
773
798
  def convert_str_to_none(p_non_char_fields_idx, p_csv_db):
774
- """Description"""
799
+ """Convert blank non-character values to None before insert."""
775
800
  rows_to_del = []
776
801
  csv_db = p_csv_db
777
802
  list_len = len(csv_db)
@@ -813,7 +838,7 @@ class SQLDbWrpr:
813
838
  # end convert_str_to_none
814
839
 
815
840
  def find_non_char_field_idx(p_csv_db):
816
- """Find the index of the fields that could potentially contain mepty strings."""
841
+ """Find non-character field indexes that may contain empty strings."""
817
842
  non_char_fields_idx = []
818
843
  for header_field_name in self.non_char_fields[p_table_name]:
819
844
  for row_idx, data_field_name in enumerate(p_csv_db[0]):
@@ -825,7 +850,7 @@ class SQLDbWrpr:
825
850
  # end find_non_char_field_idx
826
851
 
827
852
  def fix_dates(p_csv_db, p_table_name, p_header):
828
- """Ensure date and datetime fields in the database is valid."""
853
+ """Normalize date and datetime values before insert."""
829
854
  c_field_idx = 0
830
855
  c_field_type = 1
831
856
  csv_db = p_csv_db
@@ -879,7 +904,7 @@ class SQLDbWrpr:
879
904
  # end fix_dates
880
905
 
881
906
  def write_to_table(p_csv_db):
882
- """Write the data to a table"""
907
+ """Write prepared rows to the destination table."""
883
908
  i = 1
884
909
  j = 0 # In case batch size is more than all records
885
910
  list_len = len(p_csv_db)
@@ -893,21 +918,11 @@ class SQLDbWrpr:
893
918
  p_verbose=p_verbose,
894
919
  p_bar_len=self.bar_len,
895
920
  )
896
- # sql_str = 'REPLACE'
897
- if p_replace:
898
- sql_str = "REPLACE"
899
- else:
900
- sql_str = "INSERT"
901
- sql_str = "{} INTO {} ({}) VALUES ({})".format(
902
- sql_str,
903
- p_table_name,
904
- ",".join([str(x) for x in header]),
905
- ",".join(["%s" for x in range(len(header))]),
906
- )
921
+ sql_str = self.build_insert_sql(p_table_name, header, p_replace=p_replace)
907
922
  for j in range(self.batch_size, list_len, self.batch_size):
908
923
  try:
909
924
  self.cur.executemany(sql_str, p_csv_db[i : j + 1])
910
- except Error as err:
925
+ except self.db_error as err:
911
926
  self.logger.error(err)
912
927
  self.conn.rollback()
913
928
  self._err_broken_rec(sql_str, p_csv_db[i : j + 1])
@@ -939,7 +954,7 @@ class SQLDbWrpr:
939
954
  # end import_volume
940
955
 
941
956
  def import_single_volume(p_csv_db, p_header, p_verbose):
942
- """Description"""
957
+ """Import one supplied in-memory CSV volume."""
943
958
  success = False
944
959
  # if not p_csv_db:
945
960
  # if os.path.isfile(p_csv_file_name):
@@ -965,7 +980,7 @@ class SQLDbWrpr:
965
980
  # end import_single_volume
966
981
 
967
982
  def import_multi_volume(p_verbose, p_header):
968
- """Description"""
983
+ """Import numbered CSV files until the next volume is missing."""
969
984
  vol_cntr = 1
970
985
  success = False
971
986
  vol_csv_file_name = p_csv_file_name
@@ -1018,7 +1033,7 @@ class SQLDbWrpr:
1018
1033
  p_verbose=False,
1019
1034
  p_debug=False,
1020
1035
  ):
1021
- """Import a csv file into a database table.
1036
+ """Split CSV rows into one or more destination table imports.
1022
1037
 
1023
1038
  Parameters
1024
1039
  - p_split_struct - { 'Seq01': { 'TableName': Desttable_name1, 'Key': TableKey, 'Replace': False, 'Flds': [[ OrgField1, DestField1, [ Command, Parm1, Parm2, Parm3 ]],
@@ -1036,8 +1051,8 @@ class SQLDbWrpr:
1036
1051
  - Replace (boolean): Either use REPLACE or INSERT SQL statement to add records to the table. INSERT will cause
1037
1052
  a failure when the record to be added is a duplicate.
1038
1053
  - Fields (str): Mandatory key word (in the python dict structure) to list the fields in the table
1039
- - OrgFieldN (str): Field name from the csv file top copy to the database table
1040
- - DestFieldN (str): Destination filed where OrgFieldN will be copied into
1054
+ - OrgFieldN (str): Field name from the CSV file to copy to the database table
1055
+ - DestFieldN (str): Destination field where OrgFieldN will be copied into
1041
1056
  - Command (int): 0 = Copy OrgFieldN to DestFieldN as is
1042
1057
  Parm1 = Truncate OrgFieldN at Parm1 if it is a string and insert into DestFieldN. 0 for no truncation. Non 'str' will not be truncated
1043
1058
  Parm2 = True if you do not want to add the row if the result is empty, else False
@@ -1181,8 +1196,158 @@ class SQLDbWrpr:
1181
1196
  p_replace=p_split_struct[seq]["Replace"],
1182
1197
  )
1183
1198
 
1199
+ @classmethod
1200
+ def from_sqlalchemy_metadata(cls, p_sqlalchemy_metadata):
1201
+ """Convert SQLAlchemy metadata into the legacy SqlDbWrpr schema structure."""
1202
+ if not getattr(p_sqlalchemy_metadata, "tables", None):
1203
+ raise SchemaSourceError("SQLAlchemy metadata does not define any tables")
1204
+ db_structure = {}
1205
+ for table in p_sqlalchemy_metadata.sorted_tables:
1206
+ db_structure[table.name] = cls._table_to_legacy_structure(table)
1207
+ return db_structure
1208
+
1209
+ @staticmethod
1210
+ def resolve_db_structure(p_db_structure=None, p_sqlalchemy_base=None, p_sqlalchemy_metadata=None):
1211
+ """Resolve schema from explicit config, SQLAlchemy metadata, or declarative base."""
1212
+ if p_db_structure:
1213
+ return p_db_structure
1214
+ if p_sqlalchemy_metadata is None and p_sqlalchemy_base is not None:
1215
+ p_sqlalchemy_metadata = getattr(p_sqlalchemy_base, "metadata", None)
1216
+ if p_sqlalchemy_metadata is not None:
1217
+ return SQLDbWrpr.from_sqlalchemy_metadata(p_sqlalchemy_metadata)
1218
+ raise SchemaSourceError("Supply p_db_structure, p_sqlalchemy_metadata, or p_sqlalchemy_base")
1219
+
1220
+ @staticmethod
1221
+ def _action_to_legacy_code(p_action):
1222
+ """Convert SQLAlchemy foreign-key actions to legacy codes."""
1223
+ if p_action is None:
1224
+ return "N"
1225
+ action = p_action.upper().replace(" ", "_")
1226
+ action_map = {
1227
+ "CASCADE": "C",
1228
+ "RESTRICT": "R",
1229
+ "SET_DEFAULT": "D",
1230
+ "SET_NULL": "N",
1231
+ }
1232
+ return action_map[action]
1233
+
1234
+ @staticmethod
1235
+ def _build_default_field_params():
1236
+ """Build the legacy default parameter block for a field."""
1237
+ return {
1238
+ "PrimaryKey": ["", ""],
1239
+ "FKey": [],
1240
+ "Index": [],
1241
+ "NN": "",
1242
+ "B": "",
1243
+ "UN": "",
1244
+ "ZF": "",
1245
+ "AI": "",
1246
+ "G": "",
1247
+ "DEF": "",
1248
+ }
1249
+
1250
+ @staticmethod
1251
+ def _column_type_to_legacy(p_column):
1252
+ """Convert a SQLAlchemy column type to the legacy type list."""
1253
+ column_type = p_column.type
1254
+ type_name = column_type.__class__.__name__.lower()
1255
+ if "biginteger" in type_name:
1256
+ return ["bigint"]
1257
+ if "boolean" in type_name:
1258
+ return ["boolean"]
1259
+ if "char" in type_name or "string" in type_name or "varchar" in type_name:
1260
+ if getattr(column_type, "length", None) is None:
1261
+ return ["varchar"]
1262
+ if type_name == "char":
1263
+ return ["char", column_type.length]
1264
+ return ["varchar", column_type.length]
1265
+ if type_name == "date":
1266
+ return ["date"]
1267
+ if "datetime" in type_name:
1268
+ return ["datetime"]
1269
+ if "integer" in type_name:
1270
+ return ["int"]
1271
+ if "largebinary" in type_name or "blob" in type_name or "binary" in type_name:
1272
+ return ["blob"]
1273
+ if "numeric" in type_name or "decimal" in type_name:
1274
+ precision = getattr(column_type, "precision", None)
1275
+ scale = getattr(column_type, "scale", None)
1276
+ if precision is not None and scale is not None:
1277
+ return ["decimal", precision, scale]
1278
+ return ["decimal"]
1279
+ if type_name == "time":
1280
+ return ["time"]
1281
+ return [type_name]
1282
+
1283
+ @staticmethod
1284
+ def _column_to_legacy_field(p_column):
1285
+ """Convert a SQLAlchemy column to a legacy field definition."""
1286
+ field_type = SQLDbWrpr._column_type_to_legacy(p_column)
1287
+ params = SQLDbWrpr._build_default_field_params()
1288
+ if not p_column.nullable or p_column.primary_key:
1289
+ params["NN"] = "Y"
1290
+ if p_column.autoincrement is True:
1291
+ params["AI"] = "Y"
1292
+ if p_column.default is not None and getattr(p_column.default, "is_scalar", False):
1293
+ params["DEF"] = str(p_column.default.arg)
1294
+ if field_type[0] == "blob":
1295
+ params["B"] = "Y"
1296
+ return {
1297
+ "Type": field_type,
1298
+ "Params": params,
1299
+ "Possible Values": "",
1300
+ "Comment": p_column.comment or "",
1301
+ }
1302
+
1303
+ @staticmethod
1304
+ def _set_foreign_keys(p_table, p_table_structure):
1305
+ """Set legacy foreign-key metadata on converted table fields."""
1306
+ for fkey_nr, constraint in enumerate(p_table.foreign_key_constraints, start=1):
1307
+ elements = list(constraint.elements)
1308
+ for field_order, element in enumerate(elements, start=1):
1309
+ p_table_structure[element.parent.name]["Params"]["FKey"] = [
1310
+ fkey_nr,
1311
+ field_order,
1312
+ element.column.table.name,
1313
+ element.column.name,
1314
+ SQLDbWrpr._action_to_legacy_code(constraint.ondelete),
1315
+ SQLDbWrpr._action_to_legacy_code(constraint.onupdate),
1316
+ ]
1317
+
1318
+ @staticmethod
1319
+ def _set_indexes(p_table, p_table_structure):
1320
+ """Set legacy index metadata on converted table fields."""
1321
+ indexes = sorted(p_table.indexes, key=lambda p_index: p_index.name or "")
1322
+ for idx_nr, index in enumerate(indexes, start=1):
1323
+ for field_order, column in enumerate(index.columns, start=1):
1324
+ p_table_structure[column.name]["Params"]["Index"] = [
1325
+ idx_nr,
1326
+ field_order,
1327
+ "A",
1328
+ "U" if index.unique else "",
1329
+ ]
1330
+
1331
+ @staticmethod
1332
+ def _set_primary_key(p_table, p_table_structure):
1333
+ """Set legacy primary-key metadata on converted table fields."""
1334
+ for column in p_table.primary_key.columns:
1335
+ p_table_structure[column.name]["Params"]["PrimaryKey"] = ["Y", "A"]
1336
+
1337
+ @staticmethod
1338
+ def _table_to_legacy_structure(p_table):
1339
+ """Convert a SQLAlchemy table to the legacy table structure."""
1340
+ table_structure = {}
1341
+ for column in p_table.columns:
1342
+ table_structure[column.name] = SQLDbWrpr._column_to_legacy_field(column)
1343
+ SQLDbWrpr._set_foreign_keys(p_table, table_structure)
1344
+ SQLDbWrpr._set_indexes(p_table, table_structure)
1345
+ SQLDbWrpr._set_primary_key(p_table, table_structure)
1346
+ return table_structure
1347
+
1184
1348
  @staticmethod
1185
1349
  def _print_err_msg(p_err, p_msg=""):
1350
+ """Print a formatted database error message."""
1186
1351
  msg = p_msg
1187
1352
  if p_msg:
1188
1353
  msg = f"{p_msg}\n"
@@ -1197,7 +1362,7 @@ class SQLDbWrpr:
1197
1362
 
1198
1363
 
1199
1364
  class MySQL(SQLDbWrpr):
1200
- """This module creates a wrapper for the MySql database."""
1365
+ """Wrapper for MySQL databases."""
1201
1366
 
1202
1367
  def __init__(
1203
1368
  self,
@@ -1208,6 +1373,8 @@ class MySQL(SQLDbWrpr):
1208
1373
  p_recreate_db=False,
1209
1374
  p_db_name=None,
1210
1375
  p_db_structure=None,
1376
+ p_sqlalchemy_base=None,
1377
+ p_sqlalchemy_metadata=None,
1211
1378
  p_batch_size=10000,
1212
1379
  p_bar_len=50,
1213
1380
  p_msg_width=50,
@@ -1220,7 +1387,7 @@ class MySQL(SQLDbWrpr):
1220
1387
  # p_ssl_cert=None
1221
1388
  **kwargs,
1222
1389
  ):
1223
- """Description"""
1390
+ """Connect to MySQL and optionally recreate or select the target database."""
1224
1391
  super().__init__(
1225
1392
  p_host_name=p_host_name,
1226
1393
  p_user_name=p_user_name,
@@ -1228,6 +1395,8 @@ class MySQL(SQLDbWrpr):
1228
1395
  p_db_name=p_db_name,
1229
1396
  p_recreate_db=p_recreate_db,
1230
1397
  p_db_structure=p_db_structure,
1398
+ p_sqlalchemy_base=p_sqlalchemy_base,
1399
+ p_sqlalchemy_metadata=p_sqlalchemy_metadata,
1231
1400
  p_batch_size=p_batch_size,
1232
1401
  p_bar_len=p_bar_len,
1233
1402
  p_msg_width=p_msg_width,
@@ -1252,7 +1421,7 @@ class MySQL(SQLDbWrpr):
1252
1421
  **kwargs,
1253
1422
  )
1254
1423
  self.cur = self.conn.cursor()
1255
- except mysql.connector.Error as err:
1424
+ except self.db_error as err:
1256
1425
  print(
1257
1426
  bm.error(
1258
1427
  f"Error {err}:'({self.user_name}'@'{self.host_name}')",
@@ -1270,7 +1439,7 @@ class MySQL(SQLDbWrpr):
1270
1439
  auth_plugin="mysql_native_password",
1271
1440
  port=self.db_port,
1272
1441
  )
1273
- except mysql.connector.Error as err:
1442
+ except self.db_error as err:
1274
1443
  self._print_err_msg(
1275
1444
  err,
1276
1445
  "Admin user name and/or password not supplied or incorrect",
@@ -1310,9 +1479,80 @@ class MySQL(SQLDbWrpr):
1310
1479
  self.success = True
1311
1480
  pass
1312
1481
 
1482
+ def create_users(self, p_admin_user, p_new_users):
1483
+ """Create MySQL users that do not already exist."""
1484
+ c_user_name = 0
1485
+ self.cur.execute("SELECT User, Host FROM mysql.user")
1486
+ curr_users = self.cur.fetchall()
1487
+ for user in p_new_users:
1488
+ user_key = (user[c_user_name], self.host_name)
1489
+ if user_key not in curr_users:
1490
+ try:
1491
+ self.cur.execute(
1492
+ "CREATE USER IF NOT EXISTS '{}'@'{}' IDENTIFIED BY '{}'".format(
1493
+ user[0], self.host_name, user[1]
1494
+ )
1495
+ )
1496
+ except self.db_error as err:
1497
+ self._print_err_msg(err, "Could not create user")
1498
+ self.close()
1499
+ sys.exit()
1500
+ self.conn.commit()
1501
+ self.success = True
1502
+
1503
+ def delete_users(self, p_admin_user, p_del_users):
1504
+ """Delete MySQL users that exist."""
1505
+ c_user_name = 0
1506
+ c_host = 2
1507
+ self.cur.execute("SELECT CAST(User AS CHAR) FROM mysql.user")
1508
+ curr_users = [x[0] for x in self.cur.fetchall()]
1509
+ for user in p_del_users:
1510
+ if user[c_user_name] in curr_users:
1511
+ try:
1512
+ self.cur.execute(f"DROP USER '{user[c_user_name]}'@'{user[c_host]}'")
1513
+ except self.db_error as err:
1514
+ self._print_err_msg(err, "Could not delete user")
1515
+ self.close()
1516
+ sys.exit()
1517
+ self.conn.commit()
1518
+ self.success = True
1519
+
1520
+ def grant_rights(self, p_admin_user, p_user_rights):
1521
+ """Grant configured MySQL rights to users."""
1522
+ c_user_name = 0
1523
+ c_host = 1
1524
+ c_db = 2
1525
+ c_table = 3
1526
+ c_rights = 4
1527
+ for right in p_user_rights:
1528
+ try:
1529
+ sql_str = "GRANT {} ON {}.{} TO '{}'@'{}'".format(
1530
+ ",".join(right[c_rights:]),
1531
+ right[c_db],
1532
+ right[c_table],
1533
+ right[c_user_name],
1534
+ right[c_host],
1535
+ )
1536
+ self.cur.execute(sql_str)
1537
+ self.conn.commit()
1538
+ sql_str = "GRANT {} ON {}.{} TO '{}'@'{}' WITH GRANT OPTION".format(
1539
+ ",".join(right[c_rights:]),
1540
+ right[c_db],
1541
+ right[c_table],
1542
+ right[c_user_name],
1543
+ right[c_host],
1544
+ )
1545
+ self.cur.execute(sql_str)
1546
+ self.conn.commit()
1547
+ except self.db_error as err:
1548
+ self._print_err_msg(err)
1549
+ self.close()
1550
+ sys.exit()
1551
+ self.success = True
1313
1552
 
1314
- class MSSQL(SQLDbWrpr):
1315
- """This module creates a wrapper for the MySql database."""
1553
+
1554
+ class PostgreSQL(SQLDbWrpr):
1555
+ """Wrapper for PostgreSQL databases."""
1316
1556
 
1317
1557
  def __init__(
1318
1558
  self,
@@ -1322,12 +1562,17 @@ class MSSQL(SQLDbWrpr):
1322
1562
  p_recreate_db=False,
1323
1563
  p_db_name=None,
1324
1564
  p_db_structure=None,
1565
+ p_sqlalchemy_base=None,
1566
+ p_sqlalchemy_metadata=None,
1325
1567
  p_batch_size=10000,
1326
1568
  p_bar_len=50,
1327
1569
  p_msg_width=50,
1328
1570
  p_verbose=False,
1571
+ p_db_port="5432",
1572
+ p_maintenance_db="postgres",
1573
+ **kwargs,
1329
1574
  ):
1330
- """Description"""
1575
+ """Create a PostgreSQL wrapper and optionally recreate the target database."""
1331
1576
  super().__init__(
1332
1577
  p_host_name=p_host_name,
1333
1578
  p_user_name=p_user_name,
@@ -1335,36 +1580,214 @@ class MSSQL(SQLDbWrpr):
1335
1580
  p_db_name=p_db_name,
1336
1581
  p_recreate_db=p_recreate_db,
1337
1582
  p_db_structure=p_db_structure,
1583
+ p_sqlalchemy_base=p_sqlalchemy_base,
1584
+ p_sqlalchemy_metadata=p_sqlalchemy_metadata,
1338
1585
  p_batch_size=p_batch_size,
1339
1586
  p_bar_len=p_bar_len,
1340
1587
  p_msg_width=p_msg_width,
1341
1588
  p_verbose=p_verbose,
1589
+ p_db_port=p_db_port,
1342
1590
  )
1591
+ self.db_error = psycopg.Error
1592
+ self.identifier_quote = '"'
1593
+ self.inline_indexes = False
1594
+ self.maintenance_db = p_maintenance_db
1595
+ self._connection_kwargs = kwargs
1596
+ connect_db = self.maintenance_db if self.re_create_db else self.db_name
1597
+ self.conn = self._connect(connect_db)
1598
+ self.conn.autocommit = True
1599
+ self.cur = self.conn.cursor()
1600
+ if self.re_create_db:
1601
+ if self.create_db():
1602
+ self.create_tables()
1603
+ self.success = True
1604
+
1605
+ def build_column_sql(self, p_field_name, p_field_type, p_field_params, p_field_comment):
1606
+ """Build PostgreSQL SQL for one column in a CREATE TABLE statement."""
1607
+ sql_str = f"{self.quote_identifier(p_field_name)} {self.render_field_type(p_field_type, p_field_params)}"
1608
+ if p_field_params["NN"] == "Y" and p_field_params["AI"] != "Y":
1609
+ sql_str += " NOT NULL"
1610
+ if p_field_params["DEF"]:
1611
+ sql_str += self.render_default_sql(p_field_type, p_field_params["DEF"])
1612
+ return sql_str
1613
+
1614
+ def build_index_sql(self, p_table_name, p_idx_name, p_idx_fields, p_unique=False):
1615
+ """Build a PostgreSQL CREATE INDEX statement."""
1616
+ unique_sql = "UNIQUE " if p_unique else ""
1617
+ index_fields = []
1618
+ for field_det in p_idx_fields:
1619
+ index_fields.append(f"{self.quote_identifier(field_det[0])} {self.sort_order[field_det[2]]}")
1620
+ return "CREATE {}INDEX {} ON {} ({})".format(
1621
+ unique_sql,
1622
+ self.quote_identifier(p_idx_name),
1623
+ self.quote_identifier(p_table_name),
1624
+ ",".join(index_fields),
1625
+ )
1626
+
1627
+ def build_insert_sql(self, p_table_name, p_header, p_replace=False):
1628
+ """Build a PostgreSQL INSERT or upsert statement."""
1629
+ insert_sql = "INSERT INTO {} ({}) VALUES ({})".format(
1630
+ self.quote_identifier(p_table_name),
1631
+ ",".join([self.quote_identifier(str(x)) for x in p_header]),
1632
+ ",".join([self.param_placeholder() for x in range(len(p_header))]),
1633
+ )
1634
+ if not p_replace:
1635
+ return insert_sql
1636
+ primary_key_fields = [
1637
+ field_name
1638
+ for field_name in self.db_structure[p_table_name]
1639
+ if self.db_structure[p_table_name][field_name]["Params"]["PrimaryKey"][0] == "Y"
1640
+ ]
1641
+ if not primary_key_fields:
1642
+ return f"{insert_sql} ON CONFLICT DO NOTHING"
1643
+ update_fields = [field_name for field_name in p_header if field_name not in primary_key_fields]
1644
+ if not update_fields:
1645
+ return f"{insert_sql} ON CONFLICT ({self.quote_identifier_list(primary_key_fields)}) DO NOTHING"
1646
+ update_sql = ",".join(
1647
+ [
1648
+ f"{self.quote_identifier(field_name)} = EXCLUDED.{self.quote_identifier(field_name)}"
1649
+ for field_name in update_fields
1650
+ ]
1651
+ )
1652
+ return f"{insert_sql} ON CONFLICT ({self.quote_identifier_list(primary_key_fields)}) DO UPDATE SET {update_sql}"
1653
+
1654
+ def create_db(self):
1655
+ """Create the PostgreSQL database according to self.db_structure."""
1656
+ self.cur.execute("SELECT 1 FROM pg_database WHERE datname = %s", (self.db_name,))
1657
+ if self.cur.fetchone():
1658
+ try:
1659
+ self.cur.execute(f"DROP DATABASE {self.quote_identifier(self.db_name)} WITH (FORCE)")
1660
+ except self.db_error as err:
1661
+ self._print_err_msg(err, "Could not drop the database")
1662
+ self.close()
1663
+ sys.exit()
1343
1664
  try:
1344
- self.host_name = "156.38.224.15,1433"
1345
- self.user_name = "chessaco_chessanew"
1346
- self._password = "@Jv&F77%"
1347
- self.db_name = "chessaco_analytics"
1348
- self.driver = "{ODBC Driver 17 for SQL Server}"
1349
- # driver = pyodbc.drivers()
1350
- # con_str_1 = 'DRIVER={};SERVER={};DATABASE={};UID={};PWD={}'.format( self.driver, self.host_name, self.db_name, self.user_name, self._password )
1351
- con_str_2 = (
1352
- "DRIVER={ODBC Driver 17 for SQL Server};SERVER="
1353
- + self.host_name
1354
- + ";DATABASE="
1355
- + self.db_name
1356
- + ";UID="
1357
- + self.user_name
1358
- + ";PWD="
1359
- + self._password
1360
- )
1361
- self.conn = pyodbc.connect(con_str_2)
1362
- pass
1363
- except Error as err:
1364
- self.logger.error(err)
1365
- self.success = self.conn.is_connected()
1366
- if self.conn.is_connected():
1665
+ self.cur.execute(f"CREATE DATABASE {self.quote_identifier(self.db_name)}")
1666
+ self.close()
1667
+ self.conn = self._connect(self.db_name)
1668
+ self.conn.autocommit = True
1367
1669
  self.cur = self.conn.cursor()
1368
- if self.re_create_db:
1369
- self.create_db()
1370
- self.success = self.create_tables()
1670
+ except self.db_error as err:
1671
+ self._print_err_msg(err, "Could not create the database")
1672
+ self.close()
1673
+ sys.exit()
1674
+ return True
1675
+
1676
+ def render_default_sql(self, p_field_type, p_default_value):
1677
+ """Render a PostgreSQL column default expression."""
1678
+ if p_field_type[0] == "varchar" or p_field_type[0] == "char":
1679
+ escaped_value = str(p_default_value).replace("'", "''")
1680
+ return f" DEFAULT '{escaped_value}'"
1681
+ return f" DEFAULT {p_default_value}"
1682
+
1683
+ def render_field_type(self, p_field_type, p_field_params):
1684
+ """Render a legacy field type as a PostgreSQL type."""
1685
+ field_type = p_field_type[0]
1686
+ if p_field_params["AI"] == "Y":
1687
+ if field_type == "bigint":
1688
+ return "BIGSERIAL"
1689
+ return "SERIAL"
1690
+ type_map = {
1691
+ "bigint": "BIGINT",
1692
+ "blob": "BYTEA",
1693
+ "boolean": "BOOLEAN",
1694
+ "date": "DATE",
1695
+ "datetime": "TIMESTAMP",
1696
+ "int": "INTEGER",
1697
+ "tinyint": "SMALLINT",
1698
+ "time": "TIME",
1699
+ }
1700
+ if field_type == "varchar" or field_type == "char":
1701
+ return f"{field_type.upper()}({str(p_field_type[1])})"
1702
+ if field_type == "decimal":
1703
+ return f"DECIMAL({str(p_field_type[1])}, {str(p_field_type[2])})"
1704
+ return type_map.get(field_type, field_type.upper())
1705
+
1706
+ def _connect(self, p_db_name):
1707
+ """Open a PostgreSQL connection."""
1708
+ return psycopg.connect(
1709
+ host=self.host_name,
1710
+ user=self.user_name,
1711
+ password=self._password,
1712
+ dbname=p_db_name,
1713
+ port=self.db_port,
1714
+ **self._connection_kwargs,
1715
+ )
1716
+
1717
+ def create_users(self, p_admin_user, p_new_users):
1718
+ """Create PostgreSQL login roles that do not already exist."""
1719
+ c_user_name = 0
1720
+ c_password = 1
1721
+
1722
+ self.cur.execute("SELECT rolname FROM pg_roles WHERE rolcanlogin = true")
1723
+ curr_users = [row[0] for row in self.cur.fetchall()]
1724
+
1725
+ for user in p_new_users:
1726
+ user_name = user[c_user_name]
1727
+ password = user[c_password]
1728
+
1729
+ if user_name not in curr_users:
1730
+ try:
1731
+ self.cur.execute(
1732
+ psycopg_sql.SQL("CREATE USER {} WITH PASSWORD {}").format(
1733
+ psycopg_sql.Identifier(user_name),
1734
+ psycopg_sql.Literal(password),
1735
+ )
1736
+ )
1737
+ except self.db_error as err:
1738
+ self._print_err_msg(err, "Could not create user")
1739
+ self.close()
1740
+ sys.exit()
1741
+ self.success = True
1742
+
1743
+ def delete_users(self, p_admin_user, p_del_users):
1744
+ """Delete PostgreSQL users that exist."""
1745
+ c_user_name = 0
1746
+
1747
+ self.cur.execute("SELECT rolname FROM pg_roles")
1748
+ curr_users = [row[0] for row in self.cur.fetchall()]
1749
+
1750
+ for user in p_del_users:
1751
+ user_name = user[c_user_name]
1752
+
1753
+ if user_name in curr_users:
1754
+ try:
1755
+ self.cur.execute(f"DROP USER {self.quote_identifier(user_name)}")
1756
+ except self.db_error as err:
1757
+ self._print_err_msg(err, "Could not delete user")
1758
+ self.close()
1759
+ sys.exit()
1760
+ self.success = True
1761
+
1762
+ def grant_rights(self, p_admin_user, p_user_rights):
1763
+ """Grant configured PostgreSQL rights to users.
1764
+
1765
+ Expected right format:
1766
+ [user_name, host, database, table, right1, right2, ...]
1767
+ The host value is ignored because PostgreSQL does not grant privileges
1768
+ by user/host pair like MySQL.
1769
+ """
1770
+ c_user_name = 0
1771
+ c_db = 2
1772
+ c_table = 3
1773
+ c_rights = 4
1774
+
1775
+ for right in p_user_rights:
1776
+ try:
1777
+ rights_sql = ",".join(right[c_rights:])
1778
+ user_sql = self.quote_identifier(right[c_user_name])
1779
+ db_sql = self.quote_identifier(right[c_db])
1780
+ table_sql = self.quote_identifier(right[c_table])
1781
+
1782
+ if right[c_table] == "*":
1783
+ sql_str = f"GRANT {rights_sql} ON DATABASE {db_sql} TO {user_sql}"
1784
+ else:
1785
+ sql_str = f"GRANT {rights_sql} ON TABLE {table_sql} TO {user_sql}"
1786
+
1787
+ self.cur.execute(sql_str)
1788
+ self.conn.commit()
1789
+ except self.db_error as err:
1790
+ self._print_err_msg(err)
1791
+ self.close()
1792
+ sys.exit()
1793
+ self.success = True