SQLDbWrpr 5.1.0__tar.gz → 5.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sqldbwrpr-5.1.0 → sqldbwrpr-5.1.1}/PKG-INFO +122 -2
- {sqldbwrpr-5.1.0 → sqldbwrpr-5.1.1}/README.md +121 -1
- {sqldbwrpr-5.1.0 → sqldbwrpr-5.1.1}/pyproject.toml +1 -1
- {sqldbwrpr-5.1.0 → sqldbwrpr-5.1.1}/src/sqldbwrpr/sqldbwrpr.py +175 -79
- {sqldbwrpr-5.1.0 → sqldbwrpr-5.1.1}/LICENSE.txt +0 -0
- {sqldbwrpr-5.1.0 → sqldbwrpr-5.1.1}/src/sqldbwrpr/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SQLDbWrpr
|
|
3
|
-
Version: 5.1.
|
|
3
|
+
Version: 5.1.1
|
|
4
4
|
Summary: Wrapper for importing CSV and Text files into MySQL and Postgress
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE.txt
|
|
@@ -65,7 +65,7 @@ When both are supplied, `p_db_structure` takes precedence for backward compatibi
|
|
|
65
65
|
- Import CSV data from files or in-memory rows, including single-volume and numbered multi-volume files.
|
|
66
66
|
- Export full tables or custom SQL query results to CSV, including optional multi-volume exports.
|
|
67
67
|
- **Database Support**: Includes MySQL and PostgreSQL wrappers with dialect-specific SQL rendering.
|
|
68
|
-
- **User and Permission Management**: Create MySQL users and grant database rights.
|
|
68
|
+
- **User and Permission Management**: Create and delete MySQL users or PostgreSQL login roles, and grant backend-specific database or table rights.
|
|
69
69
|
- **Batch Processing**: Configure import batch sizes for larger CSV loads.
|
|
70
70
|
|
|
71
71
|
### Project Structure
|
|
@@ -83,6 +83,17 @@ When both are supplied, `p_db_structure` takes precedence for backward compatibi
|
|
|
83
83
|
pip install SqlDbWrpr
|
|
84
84
|
```
|
|
85
85
|
|
|
86
|
+
### Running the Tests
|
|
87
|
+
|
|
88
|
+
The focused database tests use temporary MySQL 8.0 and PostgreSQL 16 Docker containers. Ensure Docker is running, then install the development dependencies and run the test suite:
|
|
89
|
+
|
|
90
|
+
```powershell
|
|
91
|
+
poetry install --with dev
|
|
92
|
+
poetry run pytest
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
The test fixtures wait for each database to become ready and remove the temporary containers after the test session.
|
|
96
|
+
|
|
86
97
|
### Quick Start With A Legacy Structure
|
|
87
98
|
|
|
88
99
|
```python
|
|
@@ -167,6 +178,115 @@ db = PostgreSQL(
|
|
|
167
178
|
db.import_csv("Users", p_csv_db=[("Username",), ("alice",)], p_vol_type="Single")
|
|
168
179
|
```
|
|
169
180
|
|
|
181
|
+
## Function Reference
|
|
182
|
+
|
|
183
|
+
This section documents the callable API in `src/sqldbwrpr/sqldbwrpr.py`.
|
|
184
|
+
|
|
185
|
+
### `SchemaSourceError`
|
|
186
|
+
|
|
187
|
+
- `SchemaSourceError(ValueError)`: Raised when no usable schema source is provided to initialize a wrapper (`p_db_structure`, `p_sqlalchemy_metadata`, or `p_sqlalchemy_base.metadata`).
|
|
188
|
+
|
|
189
|
+
### `SQLDbWrpr` (base wrapper)
|
|
190
|
+
|
|
191
|
+
- `__init__(p_host_name, p_user_name, p_password, p_recreate_db, p_db_name, p_db_structure, p_sqlalchemy_base, p_sqlalchemy_metadata, p_batch_size, p_bar_len, p_msg_width, p_verbose, p_db_port, p_ssl_ca, p_ssl_key, p_ssl_cert)`
|
|
192
|
+
- Initializes shared wrapper state, resolves schema source, configures import/export behavior, and precomputes char vs non-char field maps.
|
|
193
|
+
- Schema resolution order: explicit `p_db_structure` first, then SQLAlchemy metadata, else raises `SchemaSourceError`.
|
|
194
|
+
- `close()`
|
|
195
|
+
- Closes the active DB connection if present.
|
|
196
|
+
- `build_column_sql(p_field_name, p_field_type, p_field_params, p_field_comment)`
|
|
197
|
+
- Builds the column fragment for `CREATE TABLE` in the active dialect (base implementation is MySQL-oriented).
|
|
198
|
+
- Applies `AUTO_INCREMENT`, `UNSIGNED`, `NOT NULL`, `ZEROFILL`, defaults, and comments based on field parameters.
|
|
199
|
+
- `build_index_sql(p_table_name, p_idx_name, p_idx_fields, p_unique=False)`
|
|
200
|
+
- Builds an index clause with sort directions from legacy index metadata.
|
|
201
|
+
- `build_insert_sql(p_table_name, p_header, p_replace=False)`
|
|
202
|
+
- Builds an `INSERT` or `REPLACE` statement with backend parameter placeholders.
|
|
203
|
+
- `param_placeholder()`
|
|
204
|
+
- Returns the DB-API parameter token used by execution (`%s`).
|
|
205
|
+
- `quote_identifier(p_identifier)`
|
|
206
|
+
- Quotes SQL identifiers when a backend quote character is configured; otherwise returns plain identifier text.
|
|
207
|
+
- `quote_identifier_list(p_identifiers)`
|
|
208
|
+
- Quotes and joins a list of identifiers using commas.
|
|
209
|
+
- `render_default_sql(p_field_type, p_default_value)`
|
|
210
|
+
- Renders a `DEFAULT` clause, quoting string-like defaults for char/varchar types.
|
|
211
|
+
- `render_field_type(p_field_type, p_field_params)`
|
|
212
|
+
- Renders legacy type arrays into SQL type declarations (e.g., `varchar(n)`, `decimal(p,s)`).
|
|
213
|
+
- `create_db()`
|
|
214
|
+
- Base database recreation flow: drops an existing DB and creates/selects a fresh one.
|
|
215
|
+
- `create_tables()`
|
|
216
|
+
- Validates legacy schema metadata and creates tables, keys, indexes, and constraints in dependency order.
|
|
217
|
+
- Splits table creation and post-create operations where backend rules require deferred statements.
|
|
218
|
+
- `create_users(p_admin_user, p_new_users)`
|
|
219
|
+
- Creates MySQL users if they do not already exist.
|
|
220
|
+
- `delete_users(p_admin_user, p_del_users)`
|
|
221
|
+
- Drops MySQL users that exist in `mysql.user`.
|
|
222
|
+
- `_err_broken_rec(p_sql_str, p_csv_db_slice)`
|
|
223
|
+
- Retry helper used during imports to isolate/log failing rows and stop on first unrecoverable record.
|
|
224
|
+
- `export_to_csv(p_csv_path, p_table_name, p_delimiter="|", p_strip_chars="", p__vol_size=0, p_sql_query="")`
|
|
225
|
+
- Exports table/query results to CSV in single-file or multi-volume mode.
|
|
226
|
+
- Multi-volume mode chunks output by record count and generates numbered files.
|
|
227
|
+
- `get_db_field_types()`
|
|
228
|
+
- Populates per-table lists of character vs non-character columns for type-sensitive import handling.
|
|
229
|
+
- `grant_rights(p_admin_user, p_user_rights)`
|
|
230
|
+
- Grants MySQL rights and corresponding grant-option privileges according to supplied rights tuples.
|
|
231
|
+
- `import_csv(p_table_name, p_csv_file_name="", p_key="", p_header="", p_del_head=False, p_csv_db="", p_csv_corr_str_file_name="", p_vol_type="Multi", p_verbose=False, p_replace=False)`
|
|
232
|
+
- Imports CSV data from file(s) or in-memory rows.
|
|
233
|
+
- Supports single- and multi-volume input, optional header override/removal, correction-string preprocessing, type conversion, date normalization, and batched insert/replace.
|
|
234
|
+
- `import_and_split_csv(p_split_struct, p_data, p_header="", p_insert_header=False, p_verbose=False, p_debug=False)`
|
|
235
|
+
- Splits an input dataset into multiple destination table payloads using declarative field mappings and transform commands, then imports each generated dataset.
|
|
236
|
+
- `from_sqlalchemy_metadata(p_sqlalchemy_metadata)` (`@classmethod`)
|
|
237
|
+
- Converts SQLAlchemy `MetaData` (sorted tables) to the legacy `db_structure` dictionary.
|
|
238
|
+
- `resolve_db_structure(p_db_structure=None, p_sqlalchemy_base=None, p_sqlalchemy_metadata=None)` (`@staticmethod`)
|
|
239
|
+
- Central schema source resolver used by constructors; raises `SchemaSourceError` when none are provided.
|
|
240
|
+
- `_action_to_legacy_code(p_action)` (`@staticmethod`)
|
|
241
|
+
- Maps SQLAlchemy FK action text (e.g., `CASCADE`, `SET NULL`) to legacy action codes (`C`, `N`, etc.).
|
|
242
|
+
- `_build_default_field_params()` (`@staticmethod`)
|
|
243
|
+
- Produces the baseline legacy field-parameter block used during SQLAlchemy conversion.
|
|
244
|
+
- `_column_type_to_legacy(p_column)` (`@staticmethod`)
|
|
245
|
+
- Converts a SQLAlchemy column type object into the legacy type-array format.
|
|
246
|
+
- `_column_to_legacy_field(p_column)` (`@staticmethod`)
|
|
247
|
+
- Converts one SQLAlchemy column into a legacy field definition, including nullability, autoincrement, defaults, and comments.
|
|
248
|
+
- `_set_foreign_keys(p_table, p_table_structure)` (`@staticmethod`)
|
|
249
|
+
- Writes legacy foreign-key metadata onto converted table fields.
|
|
250
|
+
- `_set_indexes(p_table, p_table_structure)` (`@staticmethod`)
|
|
251
|
+
- Writes legacy index metadata for converted table fields.
|
|
252
|
+
- `_set_primary_key(p_table, p_table_structure)` (`@staticmethod`)
|
|
253
|
+
- Marks primary-key columns in converted legacy table definitions.
|
|
254
|
+
- `_table_to_legacy_structure(p_table)` (`@staticmethod`)
|
|
255
|
+
- Converts an entire SQLAlchemy table into a legacy table structure and enriches it with keys/indexes.
|
|
256
|
+
- `_print_err_msg(p_err, p_msg="")` (`@staticmethod`)
|
|
257
|
+
- Formats and prints database error details before termination paths.
|
|
258
|
+
|
|
259
|
+
### `MySQL(SQLDbWrpr)`
|
|
260
|
+
|
|
261
|
+
- `__init__(p_host_name, p_user_name, p_password, p_user_rights, p_recreate_db, p_db_name, p_db_structure, p_sqlalchemy_base, p_sqlalchemy_metadata, p_batch_size, p_bar_len, p_msg_width, p_verbose, p_admin_username, p_admin_user_password, p_db_port, **kwargs)`
|
|
262
|
+
- Opens a MySQL connection and cursor, optionally recreates database/tables, or selects the configured DB.
|
|
263
|
+
- Includes fallback logic to create and grant rights to missing users when valid admin credentials and rights data are supplied.
|
|
264
|
+
|
|
265
|
+
### `PostgreSQL(SQLDbWrpr)`
|
|
266
|
+
|
|
267
|
+
- `__init__(p_host_name, p_user_name, p_password, p_recreate_db, p_db_name, p_db_structure, p_sqlalchemy_base, p_sqlalchemy_metadata, p_batch_size, p_bar_len, p_msg_width, p_verbose, p_db_port, p_maintenance_db, **kwargs)`
|
|
268
|
+
- Configures PostgreSQL-specific behavior (quoted identifiers, non-inline indexes), opens connection, and optionally recreates DB/tables.
|
|
269
|
+
- `build_column_sql(p_field_name, p_field_type, p_field_params, p_field_comment)`
|
|
270
|
+
- PostgreSQL-specific column SQL renderer; handles non-AI `NOT NULL` and defaults.
|
|
271
|
+
- `build_index_sql(p_table_name, p_idx_name, p_idx_fields, p_unique=False)`
|
|
272
|
+
- Builds PostgreSQL `CREATE INDEX`/`CREATE UNIQUE INDEX` statements.
|
|
273
|
+
- `build_insert_sql(p_table_name, p_header, p_replace=False)`
|
|
274
|
+
- Builds PostgreSQL `INSERT`; when `p_replace=True`, builds `ON CONFLICT` upsert/no-op behavior from primary-key metadata.
|
|
275
|
+
- `create_db()`
|
|
276
|
+
- PostgreSQL DB recreation flow via `pg_database` lookup, optional forced drop, create, and reconnect.
|
|
277
|
+
- `create_users(p_admin_user, p_new_users)`
|
|
278
|
+
- Creates PostgreSQL login roles that do not already exist, using the supplied passwords.
|
|
279
|
+
- `delete_users(p_admin_user, p_del_users)`
|
|
280
|
+
- Drops PostgreSQL roles that currently exist.
|
|
281
|
+
- `grant_rights(p_admin_user, p_user_rights)`
|
|
282
|
+
- Grants database-level rights when the table target is `*`; otherwise grants table-level rights. PostgreSQL ignores the host field in each rights entry.
|
|
283
|
+
- `render_default_sql(p_field_type, p_default_value)`
|
|
284
|
+
- Renders PostgreSQL defaults with proper escaping for string values.
|
|
285
|
+
- `render_field_type(p_field_type, p_field_params)`
|
|
286
|
+
- Maps legacy types to PostgreSQL types, including `SERIAL`/`BIGSERIAL` for autoincrement fields.
|
|
287
|
+
- `_connect(p_db_name)`
|
|
288
|
+
- Internal connector helper returning a `psycopg` connection for the requested database.
|
|
289
|
+
|
|
170
290
|
______________________________________________________________________
|
|
171
291
|
|
|
172
292
|
## Updating ReleaseNotes Instructions
|
|
@@ -29,7 +29,7 @@ When both are supplied, `p_db_structure` takes precedence for backward compatibi
|
|
|
29
29
|
- Import CSV data from files or in-memory rows, including single-volume and numbered multi-volume files.
|
|
30
30
|
- Export full tables or custom SQL query results to CSV, including optional multi-volume exports.
|
|
31
31
|
- **Database Support**: Includes MySQL and PostgreSQL wrappers with dialect-specific SQL rendering.
|
|
32
|
-
- **User and Permission Management**: Create MySQL users and grant database rights.
|
|
32
|
+
- **User and Permission Management**: Create and delete MySQL users or PostgreSQL login roles, and grant backend-specific database or table rights.
|
|
33
33
|
- **Batch Processing**: Configure import batch sizes for larger CSV loads.
|
|
34
34
|
|
|
35
35
|
### Project Structure
|
|
@@ -47,6 +47,17 @@ When both are supplied, `p_db_structure` takes precedence for backward compatibi
|
|
|
47
47
|
pip install SqlDbWrpr
|
|
48
48
|
```
|
|
49
49
|
|
|
50
|
+
### Running the Tests
|
|
51
|
+
|
|
52
|
+
The focused database tests use temporary MySQL 8.0 and PostgreSQL 16 Docker containers. Ensure Docker is running, then install the development dependencies and run the test suite:
|
|
53
|
+
|
|
54
|
+
```powershell
|
|
55
|
+
poetry install --with dev
|
|
56
|
+
poetry run pytest
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
The test fixtures wait for each database to become ready and remove the temporary containers after the test session.
|
|
60
|
+
|
|
50
61
|
### Quick Start With A Legacy Structure
|
|
51
62
|
|
|
52
63
|
```python
|
|
@@ -131,6 +142,115 @@ db = PostgreSQL(
|
|
|
131
142
|
db.import_csv("Users", p_csv_db=[("Username",), ("alice",)], p_vol_type="Single")
|
|
132
143
|
```
|
|
133
144
|
|
|
145
|
+
## Function Reference
|
|
146
|
+
|
|
147
|
+
This section documents the callable API in `src/sqldbwrpr/sqldbwrpr.py`.
|
|
148
|
+
|
|
149
|
+
### `SchemaSourceError`
|
|
150
|
+
|
|
151
|
+
- `SchemaSourceError(ValueError)`: Raised when no usable schema source is provided to initialize a wrapper (`p_db_structure`, `p_sqlalchemy_metadata`, or `p_sqlalchemy_base.metadata`).
|
|
152
|
+
|
|
153
|
+
### `SQLDbWrpr` (base wrapper)
|
|
154
|
+
|
|
155
|
+
- `__init__(p_host_name, p_user_name, p_password, p_recreate_db, p_db_name, p_db_structure, p_sqlalchemy_base, p_sqlalchemy_metadata, p_batch_size, p_bar_len, p_msg_width, p_verbose, p_db_port, p_ssl_ca, p_ssl_key, p_ssl_cert)`
|
|
156
|
+
- Initializes shared wrapper state, resolves schema source, configures import/export behavior, and precomputes char vs non-char field maps.
|
|
157
|
+
- Schema resolution order: explicit `p_db_structure` first, then SQLAlchemy metadata, else raises `SchemaSourceError`.
|
|
158
|
+
- `close()`
|
|
159
|
+
- Closes the active DB connection if present.
|
|
160
|
+
- `build_column_sql(p_field_name, p_field_type, p_field_params, p_field_comment)`
|
|
161
|
+
- Builds the column fragment for `CREATE TABLE` in the active dialect (base implementation is MySQL-oriented).
|
|
162
|
+
- Applies `AUTO_INCREMENT`, `UNSIGNED`, `NOT NULL`, `ZEROFILL`, defaults, and comments based on field parameters.
|
|
163
|
+
- `build_index_sql(p_table_name, p_idx_name, p_idx_fields, p_unique=False)`
|
|
164
|
+
- Builds an index clause with sort directions from legacy index metadata.
|
|
165
|
+
- `build_insert_sql(p_table_name, p_header, p_replace=False)`
|
|
166
|
+
- Builds an `INSERT` or `REPLACE` statement with backend parameter placeholders.
|
|
167
|
+
- `param_placeholder()`
|
|
168
|
+
- Returns the DB-API parameter token used by execution (`%s`).
|
|
169
|
+
- `quote_identifier(p_identifier)`
|
|
170
|
+
- Quotes SQL identifiers when a backend quote character is configured; otherwise returns plain identifier text.
|
|
171
|
+
- `quote_identifier_list(p_identifiers)`
|
|
172
|
+
- Quotes and joins a list of identifiers using commas.
|
|
173
|
+
- `render_default_sql(p_field_type, p_default_value)`
|
|
174
|
+
- Renders a `DEFAULT` clause, quoting string-like defaults for char/varchar types.
|
|
175
|
+
- `render_field_type(p_field_type, p_field_params)`
|
|
176
|
+
- Renders legacy type arrays into SQL type declarations (e.g., `varchar(n)`, `decimal(p,s)`).
|
|
177
|
+
- `create_db()`
|
|
178
|
+
- Base database recreation flow: drops an existing DB and creates/selects a fresh one.
|
|
179
|
+
- `create_tables()`
|
|
180
|
+
- Validates legacy schema metadata and creates tables, keys, indexes, and constraints in dependency order.
|
|
181
|
+
- Splits table creation and post-create operations where backend rules require deferred statements.
|
|
182
|
+
- `create_users(p_admin_user, p_new_users)`
|
|
183
|
+
- Creates MySQL users if they do not already exist.
|
|
184
|
+
- `delete_users(p_admin_user, p_del_users)`
|
|
185
|
+
- Drops MySQL users that exist in `mysql.user`.
|
|
186
|
+
- `_err_broken_rec(p_sql_str, p_csv_db_slice)`
|
|
187
|
+
- Retry helper used during imports to isolate/log failing rows and stop on first unrecoverable record.
|
|
188
|
+
- `export_to_csv(p_csv_path, p_table_name, p_delimiter="|", p_strip_chars="", p__vol_size=0, p_sql_query="")`
|
|
189
|
+
- Exports table/query results to CSV in single-file or multi-volume mode.
|
|
190
|
+
- Multi-volume mode chunks output by record count and generates numbered files.
|
|
191
|
+
- `get_db_field_types()`
|
|
192
|
+
- Populates per-table lists of character vs non-character columns for type-sensitive import handling.
|
|
193
|
+
- `grant_rights(p_admin_user, p_user_rights)`
|
|
194
|
+
- Grants MySQL rights and corresponding grant-option privileges according to supplied rights tuples.
|
|
195
|
+
- `import_csv(p_table_name, p_csv_file_name="", p_key="", p_header="", p_del_head=False, p_csv_db="", p_csv_corr_str_file_name="", p_vol_type="Multi", p_verbose=False, p_replace=False)`
|
|
196
|
+
- Imports CSV data from file(s) or in-memory rows.
|
|
197
|
+
- Supports single- and multi-volume input, optional header override/removal, correction-string preprocessing, type conversion, date normalization, and batched insert/replace.
|
|
198
|
+
- `import_and_split_csv(p_split_struct, p_data, p_header="", p_insert_header=False, p_verbose=False, p_debug=False)`
|
|
199
|
+
- Splits an input dataset into multiple destination table payloads using declarative field mappings and transform commands, then imports each generated dataset.
|
|
200
|
+
- `from_sqlalchemy_metadata(p_sqlalchemy_metadata)` (`@classmethod`)
|
|
201
|
+
- Converts SQLAlchemy `MetaData` (sorted tables) to the legacy `db_structure` dictionary.
|
|
202
|
+
- `resolve_db_structure(p_db_structure=None, p_sqlalchemy_base=None, p_sqlalchemy_metadata=None)` (`@staticmethod`)
|
|
203
|
+
- Central schema source resolver used by constructors; raises `SchemaSourceError` when none are provided.
|
|
204
|
+
- `_action_to_legacy_code(p_action)` (`@staticmethod`)
|
|
205
|
+
- Maps SQLAlchemy FK action text (e.g., `CASCADE`, `SET NULL`) to legacy action codes (`C`, `N`, etc.).
|
|
206
|
+
- `_build_default_field_params()` (`@staticmethod`)
|
|
207
|
+
- Produces the baseline legacy field-parameter block used during SQLAlchemy conversion.
|
|
208
|
+
- `_column_type_to_legacy(p_column)` (`@staticmethod`)
|
|
209
|
+
- Converts a SQLAlchemy column type object into the legacy type-array format.
|
|
210
|
+
- `_column_to_legacy_field(p_column)` (`@staticmethod`)
|
|
211
|
+
- Converts one SQLAlchemy column into a legacy field definition, including nullability, autoincrement, defaults, and comments.
|
|
212
|
+
- `_set_foreign_keys(p_table, p_table_structure)` (`@staticmethod`)
|
|
213
|
+
- Writes legacy foreign-key metadata onto converted table fields.
|
|
214
|
+
- `_set_indexes(p_table, p_table_structure)` (`@staticmethod`)
|
|
215
|
+
- Writes legacy index metadata for converted table fields.
|
|
216
|
+
- `_set_primary_key(p_table, p_table_structure)` (`@staticmethod`)
|
|
217
|
+
- Marks primary-key columns in converted legacy table definitions.
|
|
218
|
+
- `_table_to_legacy_structure(p_table)` (`@staticmethod`)
|
|
219
|
+
- Converts an entire SQLAlchemy table into a legacy table structure and enriches it with keys/indexes.
|
|
220
|
+
- `_print_err_msg(p_err, p_msg="")` (`@staticmethod`)
|
|
221
|
+
- Formats and prints database error details before termination paths.
|
|
222
|
+
|
|
223
|
+
### `MySQL(SQLDbWrpr)`
|
|
224
|
+
|
|
225
|
+
- `__init__(p_host_name, p_user_name, p_password, p_user_rights, p_recreate_db, p_db_name, p_db_structure, p_sqlalchemy_base, p_sqlalchemy_metadata, p_batch_size, p_bar_len, p_msg_width, p_verbose, p_admin_username, p_admin_user_password, p_db_port, **kwargs)`
|
|
226
|
+
- Opens a MySQL connection and cursor, optionally recreates database/tables, or selects the configured DB.
|
|
227
|
+
- Includes fallback logic to create and grant rights to missing users when valid admin credentials and rights data are supplied.
|
|
228
|
+
|
|
229
|
+
### `PostgreSQL(SQLDbWrpr)`
|
|
230
|
+
|
|
231
|
+
- `__init__(p_host_name, p_user_name, p_password, p_recreate_db, p_db_name, p_db_structure, p_sqlalchemy_base, p_sqlalchemy_metadata, p_batch_size, p_bar_len, p_msg_width, p_verbose, p_db_port, p_maintenance_db, **kwargs)`
|
|
232
|
+
- Configures PostgreSQL-specific behavior (quoted identifiers, non-inline indexes), opens connection, and optionally recreates DB/tables.
|
|
233
|
+
- `build_column_sql(p_field_name, p_field_type, p_field_params, p_field_comment)`
|
|
234
|
+
- PostgreSQL-specific column SQL renderer; handles non-AI `NOT NULL` and defaults.
|
|
235
|
+
- `build_index_sql(p_table_name, p_idx_name, p_idx_fields, p_unique=False)`
|
|
236
|
+
- Builds PostgreSQL `CREATE INDEX`/`CREATE UNIQUE INDEX` statements.
|
|
237
|
+
- `build_insert_sql(p_table_name, p_header, p_replace=False)`
|
|
238
|
+
- Builds PostgreSQL `INSERT`; when `p_replace=True`, builds `ON CONFLICT` upsert/no-op behavior from primary-key metadata.
|
|
239
|
+
- `create_db()`
|
|
240
|
+
- PostgreSQL DB recreation flow via `pg_database` lookup, optional forced drop, create, and reconnect.
|
|
241
|
+
- `create_users(p_admin_user, p_new_users)`
|
|
242
|
+
- Creates PostgreSQL login roles that do not already exist, using the supplied passwords.
|
|
243
|
+
- `delete_users(p_admin_user, p_del_users)`
|
|
244
|
+
- Drops PostgreSQL roles that currently exist.
|
|
245
|
+
- `grant_rights(p_admin_user, p_user_rights)`
|
|
246
|
+
- Grants database-level rights when the table target is `*`; otherwise grants table-level rights. PostgreSQL ignores the host field in each rights entry.
|
|
247
|
+
- `render_default_sql(p_field_type, p_default_value)`
|
|
248
|
+
- Renders PostgreSQL defaults with proper escaping for string values.
|
|
249
|
+
- `render_field_type(p_field_type, p_field_params)`
|
|
250
|
+
- Maps legacy types to PostgreSQL types, including `SERIAL`/`BIGSERIAL` for autoincrement fields.
|
|
251
|
+
- `_connect(p_db_name)`
|
|
252
|
+
- Internal connector helper returning a `psycopg` connection for the requested database.
|
|
253
|
+
|
|
134
254
|
______________________________________________________________________
|
|
135
255
|
|
|
136
256
|
## Updating ReleaseNotes Instructions
|
|
@@ -17,13 +17,7 @@ import mysql.connector
|
|
|
17
17
|
import psycopg
|
|
18
18
|
from beetools import msg as bm
|
|
19
19
|
from mysql.connector import errorcode
|
|
20
|
-
|
|
21
|
-
# from pathlib import Path
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
# _PROJ_DESC = __doc__.split("\n")[0]
|
|
25
|
-
# _PROJ_PATH = Path(__file__)
|
|
26
|
-
# _PROJ_NAME = _PROJ_PATH.stem
|
|
20
|
+
from psycopg import sql as psycopg_sql
|
|
27
21
|
|
|
28
22
|
|
|
29
23
|
class SchemaSourceError(ValueError):
|
|
@@ -52,7 +46,7 @@ class SQLDbWrpr:
|
|
|
52
46
|
p_ssl_key=None,
|
|
53
47
|
p_ssl_cert=None,
|
|
54
48
|
):
|
|
55
|
-
"""
|
|
49
|
+
"""Initialise a common wrapper state and resolve the database schema.
|
|
56
50
|
|
|
57
51
|
A legacy `p_db_structure` dictionary takes precedence. If it is not
|
|
58
52
|
supplied, SQLAlchemy metadata is read from `p_sqlalchemy_metadata` or
|
|
@@ -548,40 +542,28 @@ class SQLDbWrpr:
|
|
|
548
542
|
return success
|
|
549
543
|
|
|
550
544
|
def create_users(self, p_admin_user, p_new_users):
|
|
551
|
-
"""Create
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
try:
|
|
558
|
-
self.cur.execute(
|
|
559
|
-
"CREATE USER IF NOT EXISTS '{}'@'{}' IDENTIFIED BY '{}'".format(
|
|
560
|
-
user[0], self.host_name, user[1]
|
|
561
|
-
)
|
|
562
|
-
)
|
|
563
|
-
except self.db_error as err:
|
|
564
|
-
self._print_err_msg(err, "Could not create user")
|
|
565
|
-
self.close()
|
|
566
|
-
sys.exit()
|
|
567
|
-
self.conn.commit()
|
|
568
|
-
self.success = True
|
|
545
|
+
"""Create database users.
|
|
546
|
+
|
|
547
|
+
Concrete database backends must implement this because user management
|
|
548
|
+
SQL differs between databases.
|
|
549
|
+
"""
|
|
550
|
+
raise NotImplementedError(f"{self.__class__.__name__} must implement create_users()")
|
|
569
551
|
|
|
570
552
|
def delete_users(self, p_admin_user, p_del_users):
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
self.
|
|
553
|
+
"""Delete database users.
|
|
554
|
+
|
|
555
|
+
Concrete database backends must implement this because user management
|
|
556
|
+
SQL differs between databases.
|
|
557
|
+
"""
|
|
558
|
+
raise NotImplementedError(f"{self.__class__.__name__} must implement delete_users()")
|
|
559
|
+
|
|
560
|
+
def grant_rights(self, p_admin_user, p_user_rights):
|
|
561
|
+
"""Grant database rights.
|
|
562
|
+
|
|
563
|
+
Concrete database backends must implement this because privilege syntax
|
|
564
|
+
differs between databases.
|
|
565
|
+
"""
|
|
566
|
+
raise NotImplementedError(f"{self.__class__.__name__} must implement grant_rights()")
|
|
585
567
|
|
|
586
568
|
def _err_broken_rec(self, p_sql_str, p_csv_db_slice):
|
|
587
569
|
"""Write broken record to logger"""
|
|
@@ -609,11 +591,11 @@ class SQLDbWrpr:
|
|
|
609
591
|
"""Export a table to a csv file
|
|
610
592
|
|
|
611
593
|
Parameters
|
|
612
|
-
- p_csv_path
|
|
613
|
-
- p_table_name = ''
|
|
614
|
-
- p_delimeter = '|'
|
|
594
|
+
- p_csv_path - Path name of the file to be exported
|
|
595
|
+
- p_table_name = '' - Table name to export
|
|
596
|
+
- p_delimeter = '|' - Field delimiter to use
|
|
615
597
|
- p_strip_chars = '' - characters to strip from text
|
|
616
|
-
- p__vol_size = 0
|
|
598
|
+
- p__vol_size = 0 - Create a multiple-volume export. p__vol_size is
|
|
617
599
|
the number of records per file. 0 will create
|
|
618
600
|
only one volume.
|
|
619
601
|
"""
|
|
@@ -768,41 +750,6 @@ class SQLDbWrpr:
|
|
|
768
750
|
else:
|
|
769
751
|
self.non_char_fields[p_table_name].append(field)
|
|
770
752
|
|
|
771
|
-
def grant_rights(self, p_admin_user, p_user_rights):
|
|
772
|
-
"""Grant configured MySQL rights to users."""
|
|
773
|
-
c_user_name = 0
|
|
774
|
-
# c_password = 1
|
|
775
|
-
c_host = 1
|
|
776
|
-
c_db = 2
|
|
777
|
-
c_table = 3
|
|
778
|
-
c_rights = 4
|
|
779
|
-
# success = True
|
|
780
|
-
for right in p_user_rights:
|
|
781
|
-
try:
|
|
782
|
-
sql_str = "GRANT {} ON {}.{} TO '{}'@'{}'".format(
|
|
783
|
-
",".join(right[c_rights:]),
|
|
784
|
-
right[c_db],
|
|
785
|
-
right[c_table],
|
|
786
|
-
right[c_user_name],
|
|
787
|
-
right[c_host],
|
|
788
|
-
)
|
|
789
|
-
self.cur.execute(sql_str)
|
|
790
|
-
self.conn.commit()
|
|
791
|
-
sql_str = "GRANT {} ON {}.{} TO '{}'@'{}' WITH GRANT OPTION".format(
|
|
792
|
-
",".join(right[c_rights:]),
|
|
793
|
-
right[c_db],
|
|
794
|
-
right[c_table],
|
|
795
|
-
right[c_user_name],
|
|
796
|
-
right[c_host],
|
|
797
|
-
)
|
|
798
|
-
self.cur.execute(sql_str)
|
|
799
|
-
self.conn.commit()
|
|
800
|
-
except self.db_error as err:
|
|
801
|
-
self._print_err_msg(err)
|
|
802
|
-
self.close()
|
|
803
|
-
sys.exit()
|
|
804
|
-
self.success = True
|
|
805
|
-
|
|
806
753
|
def import_csv(
|
|
807
754
|
self,
|
|
808
755
|
p_table_name,
|
|
@@ -1532,6 +1479,77 @@ class MySQL(SQLDbWrpr):
|
|
|
1532
1479
|
self.success = True
|
|
1533
1480
|
pass
|
|
1534
1481
|
|
|
1482
|
+
def create_users(self, p_admin_user, p_new_users):
|
|
1483
|
+
"""Create MySQL users that do not already exist."""
|
|
1484
|
+
c_user_name = 0
|
|
1485
|
+
self.cur.execute("SELECT User, Host FROM mysql.user")
|
|
1486
|
+
curr_users = self.cur.fetchall()
|
|
1487
|
+
for user in p_new_users:
|
|
1488
|
+
user_key = (user[c_user_name], self.host_name)
|
|
1489
|
+
if user_key not in curr_users:
|
|
1490
|
+
try:
|
|
1491
|
+
self.cur.execute(
|
|
1492
|
+
"CREATE USER IF NOT EXISTS '{}'@'{}' IDENTIFIED BY '{}'".format(
|
|
1493
|
+
user[0], self.host_name, user[1]
|
|
1494
|
+
)
|
|
1495
|
+
)
|
|
1496
|
+
except self.db_error as err:
|
|
1497
|
+
self._print_err_msg(err, "Could not create user")
|
|
1498
|
+
self.close()
|
|
1499
|
+
sys.exit()
|
|
1500
|
+
self.conn.commit()
|
|
1501
|
+
self.success = True
|
|
1502
|
+
|
|
1503
|
+
def delete_users(self, p_admin_user, p_del_users):
|
|
1504
|
+
"""Delete MySQL users that exist."""
|
|
1505
|
+
c_user_name = 0
|
|
1506
|
+
c_host = 2
|
|
1507
|
+
self.cur.execute("SELECT CAST(User AS CHAR) FROM mysql.user")
|
|
1508
|
+
curr_users = [x[0] for x in self.cur.fetchall()]
|
|
1509
|
+
for user in p_del_users:
|
|
1510
|
+
if user[c_user_name] in curr_users:
|
|
1511
|
+
try:
|
|
1512
|
+
self.cur.execute(f"DROP USER '{user[c_user_name]}'@'{user[c_host]}'")
|
|
1513
|
+
except self.db_error as err:
|
|
1514
|
+
self._print_err_msg(err, "Could not delete user")
|
|
1515
|
+
self.close()
|
|
1516
|
+
sys.exit()
|
|
1517
|
+
self.conn.commit()
|
|
1518
|
+
self.success = True
|
|
1519
|
+
|
|
1520
|
+
def grant_rights(self, p_admin_user, p_user_rights):
|
|
1521
|
+
"""Grant configured MySQL rights to users."""
|
|
1522
|
+
c_user_name = 0
|
|
1523
|
+
c_host = 1
|
|
1524
|
+
c_db = 2
|
|
1525
|
+
c_table = 3
|
|
1526
|
+
c_rights = 4
|
|
1527
|
+
for right in p_user_rights:
|
|
1528
|
+
try:
|
|
1529
|
+
sql_str = "GRANT {} ON {}.{} TO '{}'@'{}'".format(
|
|
1530
|
+
",".join(right[c_rights:]),
|
|
1531
|
+
right[c_db],
|
|
1532
|
+
right[c_table],
|
|
1533
|
+
right[c_user_name],
|
|
1534
|
+
right[c_host],
|
|
1535
|
+
)
|
|
1536
|
+
self.cur.execute(sql_str)
|
|
1537
|
+
self.conn.commit()
|
|
1538
|
+
sql_str = "GRANT {} ON {}.{} TO '{}'@'{}' WITH GRANT OPTION".format(
|
|
1539
|
+
",".join(right[c_rights:]),
|
|
1540
|
+
right[c_db],
|
|
1541
|
+
right[c_table],
|
|
1542
|
+
right[c_user_name],
|
|
1543
|
+
right[c_host],
|
|
1544
|
+
)
|
|
1545
|
+
self.cur.execute(sql_str)
|
|
1546
|
+
self.conn.commit()
|
|
1547
|
+
except self.db_error as err:
|
|
1548
|
+
self._print_err_msg(err)
|
|
1549
|
+
self.close()
|
|
1550
|
+
sys.exit()
|
|
1551
|
+
self.success = True
|
|
1552
|
+
|
|
1535
1553
|
|
|
1536
1554
|
class PostgreSQL(SQLDbWrpr):
|
|
1537
1555
|
"""Wrapper for PostgreSQL databases."""
|
|
@@ -1695,3 +1713,81 @@ class PostgreSQL(SQLDbWrpr):
|
|
|
1695
1713
|
port=self.db_port,
|
|
1696
1714
|
**self._connection_kwargs,
|
|
1697
1715
|
)
|
|
1716
|
+
|
|
1717
|
+
def create_users(self, p_admin_user, p_new_users):
|
|
1718
|
+
"""Create PostgreSQL login roles that do not already exist."""
|
|
1719
|
+
c_user_name = 0
|
|
1720
|
+
c_password = 1
|
|
1721
|
+
|
|
1722
|
+
self.cur.execute("SELECT rolname FROM pg_roles WHERE rolcanlogin = true")
|
|
1723
|
+
curr_users = [row[0] for row in self.cur.fetchall()]
|
|
1724
|
+
|
|
1725
|
+
for user in p_new_users:
|
|
1726
|
+
user_name = user[c_user_name]
|
|
1727
|
+
password = user[c_password]
|
|
1728
|
+
|
|
1729
|
+
if user_name not in curr_users:
|
|
1730
|
+
try:
|
|
1731
|
+
self.cur.execute(
|
|
1732
|
+
psycopg_sql.SQL("CREATE USER {} WITH PASSWORD {}").format(
|
|
1733
|
+
psycopg_sql.Identifier(user_name),
|
|
1734
|
+
psycopg_sql.Literal(password),
|
|
1735
|
+
)
|
|
1736
|
+
)
|
|
1737
|
+
except self.db_error as err:
|
|
1738
|
+
self._print_err_msg(err, "Could not create user")
|
|
1739
|
+
self.close()
|
|
1740
|
+
sys.exit()
|
|
1741
|
+
self.success = True
|
|
1742
|
+
|
|
1743
|
+
def delete_users(self, p_admin_user, p_del_users):
|
|
1744
|
+
"""Delete PostgreSQL users that exist."""
|
|
1745
|
+
c_user_name = 0
|
|
1746
|
+
|
|
1747
|
+
self.cur.execute("SELECT rolname FROM pg_roles")
|
|
1748
|
+
curr_users = [row[0] for row in self.cur.fetchall()]
|
|
1749
|
+
|
|
1750
|
+
for user in p_del_users:
|
|
1751
|
+
user_name = user[c_user_name]
|
|
1752
|
+
|
|
1753
|
+
if user_name in curr_users:
|
|
1754
|
+
try:
|
|
1755
|
+
self.cur.execute(f"DROP USER {self.quote_identifier(user_name)}")
|
|
1756
|
+
except self.db_error as err:
|
|
1757
|
+
self._print_err_msg(err, "Could not delete user")
|
|
1758
|
+
self.close()
|
|
1759
|
+
sys.exit()
|
|
1760
|
+
self.success = True
|
|
1761
|
+
|
|
1762
|
+
def grant_rights(self, p_admin_user, p_user_rights):
|
|
1763
|
+
"""Grant configured PostgreSQL rights to users.
|
|
1764
|
+
|
|
1765
|
+
Expected right format:
|
|
1766
|
+
[user_name, host, database, table, right1, right2, ...]
|
|
1767
|
+
The host value is ignored because PostgreSQL does not grant privileges
|
|
1768
|
+
by user/host pair like MySQL.
|
|
1769
|
+
"""
|
|
1770
|
+
c_user_name = 0
|
|
1771
|
+
c_db = 2
|
|
1772
|
+
c_table = 3
|
|
1773
|
+
c_rights = 4
|
|
1774
|
+
|
|
1775
|
+
for right in p_user_rights:
|
|
1776
|
+
try:
|
|
1777
|
+
rights_sql = ",".join(right[c_rights:])
|
|
1778
|
+
user_sql = self.quote_identifier(right[c_user_name])
|
|
1779
|
+
db_sql = self.quote_identifier(right[c_db])
|
|
1780
|
+
table_sql = self.quote_identifier(right[c_table])
|
|
1781
|
+
|
|
1782
|
+
if right[c_table] == "*":
|
|
1783
|
+
sql_str = f"GRANT {rights_sql} ON DATABASE {db_sql} TO {user_sql}"
|
|
1784
|
+
else:
|
|
1785
|
+
sql_str = f"GRANT {rights_sql} ON TABLE {table_sql} TO {user_sql}"
|
|
1786
|
+
|
|
1787
|
+
self.cur.execute(sql_str)
|
|
1788
|
+
self.conn.commit()
|
|
1789
|
+
except self.db_error as err:
|
|
1790
|
+
self._print_err_msg(err)
|
|
1791
|
+
self.close()
|
|
1792
|
+
sys.exit()
|
|
1793
|
+
self.success = True
|
|
File without changes
|
|
File without changes
|