thoth-dbmanager 0.4.1__tar.gz → 0.4.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {thoth_dbmanager-0.4.1/thoth_dbmanager.egg-info → thoth_dbmanager-0.4.3}/PKG-INFO +13 -8
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/pyproject.toml +18 -15
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/tests/test_thoth_sqlite_manager.py +132 -68
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/__init__.py +1 -7
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/dynamic_imports.py +36 -44
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3/thoth_dbmanager.egg-info}/PKG-INFO +13 -8
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager.egg-info/requires.txt +12 -8
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager.egg-info/top_level.txt +1 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/LICENSE +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/MANIFEST.in +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/README.md +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/setup.cfg +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/tests/test_integration_new_architecture.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/tests/test_lsh_query.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/tests/test_new_architecture.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/tests/test_parameter_validation.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/tests/test_thoth_db_manager_base.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/tests/test_thoth_informix_manager.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/tests/test_thoth_mariadb_manager.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/tests/test_thoth_mysql_manager.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/tests/test_thoth_oracle_manager.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/tests/test_thoth_pg_manager.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/tests/test_thoth_sqlserver_manager.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/tests/test_thoth_supabase_manager.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/ThothDbManager.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/adapters/__init__.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/adapters/mariadb.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/adapters/mysql.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/adapters/oracle.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/adapters/postgresql.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/adapters/sqlite.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/adapters/sqlserver.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/adapters/supabase.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/core/__init__.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/core/factory.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/core/interfaces.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/core/registry.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/documents.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/helpers/__init__.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/helpers/multi_db_generator.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/helpers/preprocess_values.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/helpers/schema.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/helpers/search.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/lsh/__init__.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/lsh/core.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/lsh/factory.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/lsh/manager.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/lsh/storage.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/plugins/__init__.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/plugins/mariadb.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/plugins/mysql.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/plugins/oracle.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/plugins/postgresql.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/plugins/sqlite.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/plugins/sqlserver.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/plugins/supabase.py +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager.egg-info/SOURCES.txt +0 -0
- {thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager.egg-info/dependency_links.txt +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: thoth_dbmanager
|
3
|
-
Version: 0.4.
|
3
|
+
Version: 0.4.3
|
4
4
|
Summary: A Python library for managing SQL databases with support for multiple database types, LSH-based similarity search, and a modern plugin architecture.
|
5
5
|
Author-email: Marco Pancotti <mp@tylconsulting.it>
|
6
6
|
License: MIT
|
@@ -28,6 +28,8 @@ Requires-Dist: datasketch>=1.5.0
|
|
28
28
|
Requires-Dist: tqdm>=4.60.0
|
29
29
|
Requires-Dist: SQLAlchemy>=1.4.0
|
30
30
|
Requires-Dist: pydantic>=2.0.0
|
31
|
+
Requires-Dist: pandas>=1.3.0
|
32
|
+
Requires-Dist: requests>=2.25.0
|
31
33
|
Provides-Extra: postgresql
|
32
34
|
Requires-Dist: psycopg2-binary>=2.9.0; extra == "postgresql"
|
33
35
|
Provides-Extra: mysql
|
@@ -36,25 +38,24 @@ Provides-Extra: mariadb
|
|
36
38
|
Requires-Dist: mariadb>=1.1.0; extra == "mariadb"
|
37
39
|
Provides-Extra: sqlserver
|
38
40
|
Requires-Dist: pyodbc>=4.0.0; extra == "sqlserver"
|
41
|
+
Requires-Dist: pymssql>=2.3.0; extra == "sqlserver"
|
39
42
|
Provides-Extra: oracle
|
40
43
|
Requires-Dist: cx_Oracle>=8.3.0; extra == "oracle"
|
41
|
-
|
42
|
-
Requires-Dist: informixdb>=2.2.0; extra == "informix"
|
44
|
+
Requires-Dist: oracledb>=3.0.0; extra == "oracle"
|
43
45
|
Provides-Extra: supabase
|
44
46
|
Requires-Dist: supabase>=2.0.0; extra == "supabase"
|
45
|
-
Requires-Dist: postgrest-py>=0.
|
46
|
-
Requires-Dist: gotrue-py>=
|
47
|
+
Requires-Dist: postgrest-py>=0.10.0; extra == "supabase"
|
48
|
+
Requires-Dist: gotrue-py>=1.0.0; extra == "supabase"
|
47
49
|
Provides-Extra: sqlite
|
48
50
|
Provides-Extra: all
|
49
51
|
Requires-Dist: psycopg2-binary>=2.9.0; extra == "all"
|
50
52
|
Requires-Dist: mysql-connector-python>=8.0.0; extra == "all"
|
51
53
|
Requires-Dist: mariadb>=1.1.0; extra == "all"
|
52
54
|
Requires-Dist: pyodbc>=4.0.0; extra == "all"
|
55
|
+
Requires-Dist: pymssql>=2.3.0; extra == "all"
|
53
56
|
Requires-Dist: cx_Oracle>=8.3.0; extra == "all"
|
54
|
-
Requires-Dist:
|
57
|
+
Requires-Dist: oracledb>=3.0.0; extra == "all"
|
55
58
|
Requires-Dist: supabase>=2.0.0; extra == "all"
|
56
|
-
Requires-Dist: postgrest-py>=0.16.0; extra == "all"
|
57
|
-
Requires-Dist: gotrue-py>=2.0.0; extra == "all"
|
58
59
|
Provides-Extra: dev
|
59
60
|
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
60
61
|
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
@@ -62,6 +63,10 @@ Requires-Dist: black>=22.0.0; extra == "dev"
|
|
62
63
|
Requires-Dist: flake8>=5.0.0; extra == "dev"
|
63
64
|
Requires-Dist: mypy>=1.0.0; extra == "dev"
|
64
65
|
Requires-Dist: pre-commit>=3.0.0; extra == "dev"
|
66
|
+
Requires-Dist: build>=1.0.0; extra == "dev"
|
67
|
+
Requires-Dist: twine>=4.0.0; extra == "dev"
|
68
|
+
Requires-Dist: psutil>=5.8.0; extra == "dev"
|
69
|
+
Requires-Dist: docker>=6.0.0; extra == "dev"
|
65
70
|
Provides-Extra: test-postgresql
|
66
71
|
Requires-Dist: pytest>=7.0.0; extra == "test-postgresql"
|
67
72
|
Requires-Dist: psycopg2-binary>=2.9.0; extra == "test-postgresql"
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
4
4
|
|
5
5
|
[project]
|
6
6
|
name = "thoth_dbmanager"
|
7
|
-
version = "0.4.
|
7
|
+
version = "0.4.3"
|
8
8
|
authors = [
|
9
9
|
{ name="Marco Pancotti", email="mp@tylconsulting.it" },
|
10
10
|
]
|
@@ -30,41 +30,44 @@ classifiers = [
|
|
30
30
|
dependencies = [
|
31
31
|
"datasketch>=1.5.0",
|
32
32
|
"tqdm>=4.60.0",
|
33
|
-
"SQLAlchemy>=1.4.0",
|
33
|
+
"SQLAlchemy>=1.4.0",
|
34
34
|
"pydantic>=2.0.0",
|
35
|
+
"pandas>=1.3.0",
|
36
|
+
"requests>=2.25.0",
|
35
37
|
]
|
36
|
-
|
37
38
|
[project.optional-dependencies]
|
38
39
|
postgresql = ["psycopg2-binary>=2.9.0"]
|
39
40
|
mysql = ["mysql-connector-python>=8.0.0"]
|
40
41
|
mariadb = ["mariadb>=1.1.0"]
|
41
|
-
sqlserver = ["pyodbc>=4.0.0"]
|
42
|
-
oracle = ["cx_Oracle>=8.3.0"]
|
43
|
-
|
44
|
-
|
45
|
-
sqlite = [] # Built into Python
|
42
|
+
sqlserver = ["pyodbc>=4.0.0", "pymssql>=2.3.0"]
|
43
|
+
oracle = ["cx_Oracle>=8.3.0", "oracledb>=3.0.0"]
|
44
|
+
supabase = ["supabase>=2.0.0", "postgrest-py>=0.10.0", "gotrue-py>=1.0.0"]
|
45
|
+
sqlite = []
|
46
46
|
|
47
47
|
# Convenience groups
|
48
48
|
all = [
|
49
49
|
"psycopg2-binary>=2.9.0",
|
50
|
-
"mysql-connector-python>=8.0.0",
|
50
|
+
"mysql-connector-python>=8.0.0",
|
51
51
|
"mariadb>=1.1.0",
|
52
52
|
"pyodbc>=4.0.0",
|
53
|
+
"pymssql>=2.3.0",
|
53
54
|
"cx_Oracle>=8.3.0",
|
54
|
-
"
|
55
|
+
"oracledb>=3.0.0",
|
55
56
|
"supabase>=2.0.0",
|
56
|
-
"postgrest-py>=0.16.0",
|
57
|
-
"gotrue-py>=2.0.0"
|
58
57
|
]
|
59
58
|
|
60
59
|
# Development dependencies
|
61
60
|
dev = [
|
62
|
-
"pytest>=7.0.0",
|
61
|
+
"pytest>=7.0.0",
|
63
62
|
"pytest-cov>=4.0.0",
|
64
|
-
"black>=22.0.0",
|
63
|
+
"black>=22.0.0",
|
65
64
|
"flake8>=5.0.0",
|
66
65
|
"mypy>=1.0.0",
|
67
|
-
"pre-commit>=3.0.0"
|
66
|
+
"pre-commit>=3.0.0",
|
67
|
+
"build>=1.0.0",
|
68
|
+
"twine>=4.0.0",
|
69
|
+
"psutil>=5.8.0", # Add this for system resource monitoring
|
70
|
+
"docker>=6.0.0", # Add this for Docker management in tests
|
68
71
|
]
|
69
72
|
|
70
73
|
# Testing with specific databases
|
@@ -3,7 +3,7 @@ import os
|
|
3
3
|
import logging
|
4
4
|
from pathlib import Path
|
5
5
|
|
6
|
-
from
|
6
|
+
from thoth_dbmanager import ThothSqliteManager
|
7
7
|
|
8
8
|
# Configure logging
|
9
9
|
logging.basicConfig(level=logging.INFO,
|
@@ -47,7 +47,7 @@ class TestThothSqliteManager(unittest.TestCase):
|
|
47
47
|
try:
|
48
48
|
cls.db_manager = ThothSqliteManager.get_instance(
|
49
49
|
db_id=cls.db_id,
|
50
|
-
db_root_path=
|
50
|
+
db_root_path=cls.db_root_path,
|
51
51
|
db_mode=cls.db_mode
|
52
52
|
)
|
53
53
|
logger.info("Successfully connected to california_schools SQLite database")
|
@@ -59,7 +59,7 @@ class TestThothSqliteManager(unittest.TestCase):
|
|
59
59
|
"""Test that get_instance returns the same instance for same parameters."""
|
60
60
|
second_instance = ThothSqliteManager.get_instance(
|
61
61
|
db_id=self.db_id,
|
62
|
-
db_root_path=
|
62
|
+
db_root_path=self.db_root_path,
|
63
63
|
db_mode=self.db_mode
|
64
64
|
)
|
65
65
|
|
@@ -71,7 +71,7 @@ class TestThothSqliteManager(unittest.TestCase):
|
|
71
71
|
# Using a different db_mode should create a new instance
|
72
72
|
different_instance = ThothSqliteManager.get_instance(
|
73
73
|
db_id=self.db_id,
|
74
|
-
db_root_path=
|
74
|
+
db_root_path=self.db_root_path,
|
75
75
|
db_mode="different_mode"
|
76
76
|
)
|
77
77
|
|
@@ -200,41 +200,58 @@ class TestThothSqliteManager(unittest.TestCase):
|
|
200
200
|
|
201
201
|
def test_transaction_rollback(self):
|
202
202
|
"""Test transaction rollback on error."""
|
203
|
-
# First,
|
204
|
-
|
203
|
+
# First, clean up any existing test table
|
204
|
+
cleanup_sql = "DROP TABLE IF EXISTS test_rollback"
|
205
|
+
try:
|
206
|
+
self.db_manager.execute_sql(cleanup_sql, fetch=None)
|
207
|
+
except:
|
208
|
+
pass # Ignore if table doesn't exist
|
209
|
+
|
210
|
+
# Create a temporary table
|
211
|
+
create_table_sql = """CREATE TABLE test_rollback (
|
205
212
|
id INTEGER PRIMARY KEY,
|
206
213
|
name TEXT
|
207
214
|
)"""
|
208
215
|
|
209
216
|
try:
|
210
|
-
|
217
|
+
# Create the table (use fetch=None to avoid fetchall() on non-SELECT)
|
218
|
+
create_result = self.db_manager.execute_sql(create_table_sql, fetch=None)
|
219
|
+
logger.info(f"Table creation result: {create_result}")
|
211
220
|
|
212
|
-
# Insert a valid row
|
221
|
+
# Insert a valid row (use fetch=None to avoid fetchall() on non-SELECT)
|
213
222
|
insert_sql = """INSERT INTO test_rollback (name) VALUES ('test_value')"""
|
214
|
-
self.db_manager.execute_sql(insert_sql)
|
223
|
+
insert_result = self.db_manager.execute_sql(insert_sql, fetch=None)
|
224
|
+
logger.info(f"Insert result: {insert_result}")
|
215
225
|
|
216
|
-
# Try an invalid insert that should cause
|
226
|
+
# Try an invalid insert that should cause an error
|
217
227
|
invalid_insert = """INSERT INTO test_rollback (non_existent_column) VALUES ('test')"""
|
218
228
|
|
219
229
|
with self.assertRaises(Exception):
|
220
|
-
self.db_manager.execute_sql(invalid_insert)
|
230
|
+
self.db_manager.execute_sql(invalid_insert, fetch=None)
|
221
231
|
|
222
232
|
# Check that the valid insert was committed (SQLite behavior differs from PostgreSQL)
|
223
|
-
count_sql = """SELECT COUNT(*) as
|
224
|
-
result = self.db_manager.execute_sql(count_sql
|
233
|
+
count_sql = """SELECT COUNT(*) as count FROM test_rollback"""
|
234
|
+
result = self.db_manager.execute_sql(count_sql)
|
225
235
|
|
226
236
|
# In SQLite, each statement is its own transaction by default,
|
227
237
|
# so the first INSERT should still be there
|
228
|
-
|
229
|
-
|
238
|
+
# Legacy manager returns list of dictionaries
|
239
|
+
self.assertIsNotNone(result, "Count result should not be None")
|
240
|
+
self.assertIsInstance(result, list, "Result should be a list")
|
241
|
+
self.assertEqual(len(result), 1, "Should have one result row")
|
242
|
+
self.assertEqual(result[0]['count'], 1, "Valid insert should be committed")
|
230
243
|
|
231
|
-
|
232
|
-
cleanup_sql = "DROP TABLE IF EXISTS test_rollback"
|
233
|
-
self.db_manager.execute_sql(cleanup_sql)
|
244
|
+
logger.info("Transaction rollback test completed successfully")
|
234
245
|
|
235
246
|
except Exception as e:
|
236
|
-
logger.
|
237
|
-
|
247
|
+
logger.error(f"Transaction test failed: {str(e)}")
|
248
|
+
raise
|
249
|
+
finally:
|
250
|
+
# Clean up - drop the test table
|
251
|
+
try:
|
252
|
+
self.db_manager.execute_sql("DROP TABLE IF EXISTS test_rollback", fetch=None)
|
253
|
+
except Exception as cleanup_error:
|
254
|
+
logger.warning(f"Cleanup failed: {cleanup_error}")
|
238
255
|
|
239
256
|
def test_database_file_exists(self):
|
240
257
|
"""Test that the database file exists at the expected location."""
|
@@ -250,69 +267,116 @@ class TestThothSqliteManager(unittest.TestCase):
|
|
250
267
|
|
251
268
|
def test_data_types(self):
|
252
269
|
"""Test handling of different data types in SQLite."""
|
253
|
-
#
|
254
|
-
|
270
|
+
# Create a test table with various data types
|
271
|
+
create_table_sql = """CREATE TABLE IF NOT EXISTS test_data_types (
|
272
|
+
id INTEGER PRIMARY KEY,
|
273
|
+
text_col TEXT,
|
274
|
+
integer_col INTEGER,
|
275
|
+
real_col REAL,
|
276
|
+
blob_col BLOB,
|
277
|
+
null_col TEXT
|
278
|
+
)"""
|
255
279
|
|
256
|
-
|
257
|
-
|
258
|
-
self.
|
280
|
+
try:
|
281
|
+
# Create the table
|
282
|
+
self.db_manager.execute_sql(create_table_sql, fetch=None)
|
259
283
|
|
260
|
-
|
284
|
+
# Insert test data with various types
|
285
|
+
insert_sql = """INSERT INTO test_data_types
|
286
|
+
(text_col, integer_col, real_col, blob_col, null_col)
|
287
|
+
VALUES ('test_string', 42, 3.14159, 'binary_data', NULL)"""
|
288
|
+
self.db_manager.execute_sql(insert_sql, fetch=None)
|
261
289
|
|
262
|
-
|
263
|
-
|
290
|
+
# Get column information including data types
|
291
|
+
columns_sql = "PRAGMA table_info(test_data_types)"
|
292
|
+
columns = self.db_manager.execute_sql(columns_sql)
|
264
293
|
|
265
|
-
|
294
|
+
# Log column types for the test table
|
295
|
+
column_types = {col['name']: col['type'] for col in columns}
|
296
|
+
logger.info(f"Column types in test_data_types: {column_types}")
|
266
297
|
|
267
|
-
|
268
|
-
|
269
|
-
|
298
|
+
# Test a query that returns different data types
|
299
|
+
query_sql = "SELECT * FROM test_data_types LIMIT 1"
|
300
|
+
result = self.db_manager.execute_sql(query_sql, fetch="one")
|
270
301
|
|
271
|
-
|
272
|
-
|
302
|
+
if result:
|
303
|
+
# Log the types of values returned
|
304
|
+
value_types = {key: type(value).__name__ for key, value in result.items()}
|
305
|
+
logger.info(f"Value types in result: {value_types}")
|
273
306
|
|
274
|
-
|
307
|
+
# Verify that we can access the values
|
308
|
+
for key, value in result.items():
|
309
|
+
self.assertIsNotNone(key, "Column name should not be None")
|
310
|
+
# Value can be None, so we don't assert on that
|
275
311
|
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
312
|
+
# Test specific data type handling
|
313
|
+
self.assertEqual(result['text_col'], 'test_string')
|
314
|
+
self.assertEqual(result['integer_col'], 42)
|
315
|
+
self.assertAlmostEqual(result['real_col'], 3.14159, places=5)
|
316
|
+
self.assertIsNone(result['null_col'])
|
280
317
|
|
281
|
-
#
|
282
|
-
|
283
|
-
|
284
|
-
|
318
|
+
# Clean up
|
319
|
+
self.db_manager.execute_sql("DROP TABLE IF EXISTS test_data_types", fetch=None)
|
320
|
+
|
321
|
+
except Exception as e:
|
322
|
+
logger.error(f"Data types test failed: {str(e)}")
|
323
|
+
# Clean up on error
|
324
|
+
try:
|
325
|
+
self.db_manager.execute_sql("DROP TABLE IF EXISTS test_data_types", fetch=None)
|
326
|
+
except:
|
327
|
+
pass
|
328
|
+
raise
|
285
329
|
|
286
330
|
def test_large_result_set(self):
|
287
331
|
"""Test handling of larger result sets."""
|
288
|
-
#
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
sample_table = tables[0]['name']
|
296
|
-
|
297
|
-
# Get row count for the sample table
|
298
|
-
count_sql = f"SELECT COUNT(*) as row_count FROM {sample_table}"
|
299
|
-
|
300
|
-
count_result = self.db_manager.execute_sql(count_sql, fetch="one")
|
301
|
-
row_count = count_result['row_count']
|
302
|
-
|
303
|
-
logger.info(f"Table {sample_table} has {row_count} rows")
|
304
|
-
|
305
|
-
if row_count < 10:
|
306
|
-
self.skipTest(f"Table {sample_table} has too few rows ({row_count}) to test large result sets")
|
332
|
+
# Create a test table with sufficient data
|
333
|
+
create_table_sql = """CREATE TABLE IF NOT EXISTS test_large_data (
|
334
|
+
id INTEGER PRIMARY KEY,
|
335
|
+
name TEXT,
|
336
|
+
value INTEGER
|
337
|
+
)"""
|
307
338
|
|
308
|
-
|
309
|
-
|
339
|
+
try:
|
340
|
+
self.db_manager.execute_sql(create_table_sql, fetch=None)
|
341
|
+
|
342
|
+
# Insert multiple rows to create a larger dataset
|
343
|
+
# Insert 50 rows of test data
|
344
|
+
for i in range(50):
|
345
|
+
# Use fetch=None for INSERT statements
|
346
|
+
self.db_manager.execute_sql(
|
347
|
+
f"INSERT INTO test_large_data (name, value) VALUES ('test_name_{i}', {i})",
|
348
|
+
fetch=None
|
349
|
+
)
|
350
|
+
|
351
|
+
# Test fetching all rows
|
352
|
+
query_sql = "SELECT * FROM test_large_data"
|
353
|
+
result = self.db_manager.execute_sql(query_sql)
|
354
|
+
|
355
|
+
self.assertIsNotNone(result, "Query result should not be None")
|
356
|
+
self.assertIsInstance(result, list, "Result should be a list")
|
357
|
+
self.assertEqual(len(result), 50, "Should have 50 rows")
|
358
|
+
logger.info(f"Successfully fetched {len(result)} rows from test_large_data")
|
359
|
+
|
360
|
+
# Test fetching with LIMIT
|
361
|
+
limit_query_sql = "SELECT * FROM test_large_data LIMIT 25"
|
362
|
+
limit_result = self.db_manager.execute_sql(limit_query_sql)
|
363
|
+
|
364
|
+
self.assertIsNotNone(limit_result, "Limited query result should not be None")
|
365
|
+
self.assertIsInstance(limit_result, list, "Limited result should be a list")
|
366
|
+
self.assertEqual(len(limit_result), 25, "Should have 25 rows with LIMIT")
|
310
367
|
|
311
|
-
|
368
|
+
# Clean up - drop the test table
|
369
|
+
cleanup_sql = "DROP TABLE IF EXISTS test_large_data"
|
370
|
+
self.db_manager.execute_sql(cleanup_sql, fetch=None)
|
312
371
|
|
313
|
-
|
314
|
-
|
315
|
-
|
372
|
+
except Exception as e:
|
373
|
+
logger.error(f"Large result set test failed: {str(e)}")
|
374
|
+
# Clean up on error
|
375
|
+
try:
|
376
|
+
self.db_manager.execute_sql("DROP TABLE IF EXISTS test_large_data", fetch=None)
|
377
|
+
except:
|
378
|
+
pass
|
379
|
+
raise
|
316
380
|
|
317
381
|
@classmethod
|
318
382
|
def tearDownClass(cls):
|
@@ -43,7 +43,6 @@ from .dynamic_imports import (
|
|
43
43
|
import_sqlserver,
|
44
44
|
import_oracle,
|
45
45
|
import_mariadb,
|
46
|
-
import_informix,
|
47
46
|
import_supabase,
|
48
47
|
)
|
49
48
|
|
@@ -62,8 +61,6 @@ def __getattr__(name: str):
|
|
62
61
|
return import_manager('sqlserver')
|
63
62
|
elif name == 'ThothOracleManager':
|
64
63
|
return import_manager('oracle')
|
65
|
-
elif name == 'ThothInformixManager':
|
66
|
-
return import_manager('informix')
|
67
64
|
elif name == 'ThothSupabaseManager':
|
68
65
|
return import_manager('supabase')
|
69
66
|
|
@@ -79,7 +76,6 @@ __all__ = [
|
|
79
76
|
"ThothMariaDbManager",
|
80
77
|
"ThothSqlServerManager",
|
81
78
|
"ThothOracleManager",
|
82
|
-
"ThothInformixManager",
|
83
79
|
"ThothSupabaseManager",
|
84
80
|
|
85
81
|
# New architecture
|
@@ -106,7 +102,6 @@ __all__ = [
|
|
106
102
|
"MariaDBPlugin",
|
107
103
|
"SQLServerPlugin",
|
108
104
|
"OraclePlugin",
|
109
|
-
"InformixPlugin",
|
110
105
|
"SupabasePlugin",
|
111
106
|
|
112
107
|
# Adapters
|
@@ -116,7 +111,6 @@ __all__ = [
|
|
116
111
|
"MariaDBAdapter",
|
117
112
|
"SQLServerAdapter",
|
118
113
|
"OracleAdapter",
|
119
|
-
"InformixAdapter",
|
120
114
|
"SupabaseAdapter",
|
121
115
|
|
122
116
|
# LSH functionality
|
@@ -133,4 +127,4 @@ __all__ = [
|
|
133
127
|
"DatabaseImportError",
|
134
128
|
]
|
135
129
|
|
136
|
-
__version__ = "0.4.
|
130
|
+
__version__ = "0.4.3"
|
@@ -14,45 +14,32 @@ DATABASE_DEPENDENCIES = {
|
|
14
14
|
'mariadb': ['mariadb'],
|
15
15
|
'sqlserver': ['pyodbc'],
|
16
16
|
'oracle': ['cx_Oracle'],
|
17
|
-
'informix': ['informixdb'],
|
18
17
|
'supabase': ['supabase', 'postgrest', 'gotrue'],
|
19
18
|
'sqlite': [], # Built into Python
|
20
19
|
}
|
21
20
|
|
22
|
-
#
|
23
|
-
DATABASE_MANAGERS = {
|
24
|
-
'postgresql': 'dbmanager.impl.ThothPgManager.ThothPgManager',
|
25
|
-
'mysql': 'dbmanager.impl.ThothMySqlManager.ThothMySqlManager',
|
26
|
-
'mariadb': 'dbmanager.impl.ThothMariaDbManager.ThothMariaDbManager',
|
27
|
-
'sqlserver': 'dbmanager.impl.ThothSqlServerManager.ThothSqlServerManager',
|
28
|
-
'oracle': 'dbmanager.impl.ThothOracleManager.ThothOracleManager',
|
29
|
-
'informix': 'dbmanager.impl.ThothInformixManager.ThothInformixManager',
|
30
|
-
'supabase': 'dbmanager.impl.ThothSupabaseManager.ThothSupabaseManager',
|
31
|
-
'sqlite': 'dbmanager.impl.ThothSqliteManager.ThothSqliteManager',
|
32
|
-
}
|
21
|
+
# Note: DATABASE_MANAGERS is no longer used - managers are created via factory pattern
|
33
22
|
|
34
23
|
# Mapping of database names to their adapter classes
|
35
24
|
DATABASE_ADAPTERS = {
|
36
|
-
'postgresql': '
|
37
|
-
'mysql': '
|
38
|
-
'mariadb': '
|
39
|
-
'sqlserver': '
|
40
|
-
'oracle': '
|
41
|
-
'
|
42
|
-
'
|
43
|
-
'sqlite': 'dbmanager.adapters.sqlite.SQLiteAdapter',
|
25
|
+
'postgresql': 'thoth_dbmanager.adapters.postgresql.PostgreSQLAdapter',
|
26
|
+
'mysql': 'thoth_dbmanager.adapters.mysql.MySQLAdapter',
|
27
|
+
'mariadb': 'thoth_dbmanager.adapters.mariadb.MariaDBAdapter',
|
28
|
+
'sqlserver': 'thoth_dbmanager.adapters.sqlserver.SQLServerAdapter',
|
29
|
+
'oracle': 'thoth_dbmanager.adapters.oracle.OracleAdapter',
|
30
|
+
'supabase': 'thoth_dbmanager.adapters.supabase.SupabaseAdapter',
|
31
|
+
'sqlite': 'thoth_dbmanager.adapters.sqlite.SQLiteAdapter',
|
44
32
|
}
|
45
33
|
|
46
34
|
# Mapping of database names to their plugin classes
|
47
35
|
DATABASE_PLUGINS = {
|
48
|
-
'postgresql': '
|
49
|
-
'mysql': '
|
50
|
-
'mariadb': '
|
51
|
-
'sqlserver': '
|
52
|
-
'oracle': '
|
53
|
-
'
|
54
|
-
'
|
55
|
-
'sqlite': 'dbmanager.plugins.sqlite.SQLitePlugin',
|
36
|
+
'postgresql': 'thoth_dbmanager.plugins.postgresql.PostgreSQLPlugin',
|
37
|
+
'mysql': 'thoth_dbmanager.plugins.mysql.MySQLPlugin',
|
38
|
+
'mariadb': 'thoth_dbmanager.plugins.mariadb.MariaDBPlugin',
|
39
|
+
'sqlserver': 'thoth_dbmanager.plugins.sqlserver.SQLServerPlugin',
|
40
|
+
'oracle': 'thoth_dbmanager.plugins.oracle.OraclePlugin',
|
41
|
+
'supabase': 'thoth_dbmanager.plugins.supabase.SupabasePlugin',
|
42
|
+
'sqlite': 'thoth_dbmanager.plugins.sqlite.SQLitePlugin',
|
56
43
|
}
|
57
44
|
|
58
45
|
|
@@ -93,30 +80,39 @@ def check_dependencies(database: str) -> List[str]:
|
|
93
80
|
|
94
81
|
def import_manager(database: str) -> Any:
|
95
82
|
"""
|
96
|
-
Dynamically import a database manager
|
97
|
-
|
83
|
+
Dynamically import a database manager using the factory pattern.
|
84
|
+
|
98
85
|
Args:
|
99
86
|
database: Name of the database
|
100
|
-
|
87
|
+
|
101
88
|
Returns:
|
102
|
-
The database manager class
|
103
|
-
|
89
|
+
The database manager class (factory-created)
|
90
|
+
|
104
91
|
Raises:
|
105
92
|
DatabaseImportError: If dependencies are missing
|
106
93
|
ImportError: If the manager class cannot be imported
|
107
94
|
"""
|
108
|
-
if database not in
|
95
|
+
if database not in DATABASE_PLUGINS:
|
109
96
|
raise ValueError(f"Unknown database: {database}")
|
110
|
-
|
97
|
+
|
111
98
|
# Check dependencies
|
112
99
|
missing_deps = check_dependencies(database)
|
113
100
|
if missing_deps:
|
114
101
|
raise DatabaseImportError(database, missing_deps)
|
115
|
-
|
116
|
-
# Import the manager class
|
117
|
-
|
118
|
-
|
119
|
-
|
102
|
+
|
103
|
+
# Import the factory and create a manager class
|
104
|
+
from thoth_dbmanager.core.factory import ThothDbFactory
|
105
|
+
|
106
|
+
# Create a wrapper class that can be instantiated like the old managers
|
107
|
+
class DatabaseManagerWrapper:
|
108
|
+
def __init__(self, *args, **kwargs):
|
109
|
+
# Create manager using factory
|
110
|
+
self._manager = ThothDbFactory.create_manager(database, *args, **kwargs)
|
111
|
+
|
112
|
+
def __getattr__(self, name):
|
113
|
+
return getattr(self._manager, name)
|
114
|
+
|
115
|
+
return DatabaseManagerWrapper
|
120
116
|
|
121
117
|
|
122
118
|
def import_adapter(database: str) -> Any:
|
@@ -241,10 +237,6 @@ def import_mariadb():
|
|
241
237
|
"""Import MariaDB components."""
|
242
238
|
return import_database_components(['mariadb'])['mariadb']
|
243
239
|
|
244
|
-
def import_informix():
|
245
|
-
"""Import Informix components."""
|
246
|
-
return import_database_components(['informix'])['informix']
|
247
|
-
|
248
240
|
def import_supabase():
|
249
241
|
"""Import Supabase components."""
|
250
242
|
return import_database_components(['supabase'])['supabase']
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: thoth_dbmanager
|
3
|
-
Version: 0.4.
|
3
|
+
Version: 0.4.3
|
4
4
|
Summary: A Python library for managing SQL databases with support for multiple database types, LSH-based similarity search, and a modern plugin architecture.
|
5
5
|
Author-email: Marco Pancotti <mp@tylconsulting.it>
|
6
6
|
License: MIT
|
@@ -28,6 +28,8 @@ Requires-Dist: datasketch>=1.5.0
|
|
28
28
|
Requires-Dist: tqdm>=4.60.0
|
29
29
|
Requires-Dist: SQLAlchemy>=1.4.0
|
30
30
|
Requires-Dist: pydantic>=2.0.0
|
31
|
+
Requires-Dist: pandas>=1.3.0
|
32
|
+
Requires-Dist: requests>=2.25.0
|
31
33
|
Provides-Extra: postgresql
|
32
34
|
Requires-Dist: psycopg2-binary>=2.9.0; extra == "postgresql"
|
33
35
|
Provides-Extra: mysql
|
@@ -36,25 +38,24 @@ Provides-Extra: mariadb
|
|
36
38
|
Requires-Dist: mariadb>=1.1.0; extra == "mariadb"
|
37
39
|
Provides-Extra: sqlserver
|
38
40
|
Requires-Dist: pyodbc>=4.0.0; extra == "sqlserver"
|
41
|
+
Requires-Dist: pymssql>=2.3.0; extra == "sqlserver"
|
39
42
|
Provides-Extra: oracle
|
40
43
|
Requires-Dist: cx_Oracle>=8.3.0; extra == "oracle"
|
41
|
-
|
42
|
-
Requires-Dist: informixdb>=2.2.0; extra == "informix"
|
44
|
+
Requires-Dist: oracledb>=3.0.0; extra == "oracle"
|
43
45
|
Provides-Extra: supabase
|
44
46
|
Requires-Dist: supabase>=2.0.0; extra == "supabase"
|
45
|
-
Requires-Dist: postgrest-py>=0.
|
46
|
-
Requires-Dist: gotrue-py>=
|
47
|
+
Requires-Dist: postgrest-py>=0.10.0; extra == "supabase"
|
48
|
+
Requires-Dist: gotrue-py>=1.0.0; extra == "supabase"
|
47
49
|
Provides-Extra: sqlite
|
48
50
|
Provides-Extra: all
|
49
51
|
Requires-Dist: psycopg2-binary>=2.9.0; extra == "all"
|
50
52
|
Requires-Dist: mysql-connector-python>=8.0.0; extra == "all"
|
51
53
|
Requires-Dist: mariadb>=1.1.0; extra == "all"
|
52
54
|
Requires-Dist: pyodbc>=4.0.0; extra == "all"
|
55
|
+
Requires-Dist: pymssql>=2.3.0; extra == "all"
|
53
56
|
Requires-Dist: cx_Oracle>=8.3.0; extra == "all"
|
54
|
-
Requires-Dist:
|
57
|
+
Requires-Dist: oracledb>=3.0.0; extra == "all"
|
55
58
|
Requires-Dist: supabase>=2.0.0; extra == "all"
|
56
|
-
Requires-Dist: postgrest-py>=0.16.0; extra == "all"
|
57
|
-
Requires-Dist: gotrue-py>=2.0.0; extra == "all"
|
58
59
|
Provides-Extra: dev
|
59
60
|
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
60
61
|
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
@@ -62,6 +63,10 @@ Requires-Dist: black>=22.0.0; extra == "dev"
|
|
62
63
|
Requires-Dist: flake8>=5.0.0; extra == "dev"
|
63
64
|
Requires-Dist: mypy>=1.0.0; extra == "dev"
|
64
65
|
Requires-Dist: pre-commit>=3.0.0; extra == "dev"
|
66
|
+
Requires-Dist: build>=1.0.0; extra == "dev"
|
67
|
+
Requires-Dist: twine>=4.0.0; extra == "dev"
|
68
|
+
Requires-Dist: psutil>=5.8.0; extra == "dev"
|
69
|
+
Requires-Dist: docker>=6.0.0; extra == "dev"
|
65
70
|
Provides-Extra: test-postgresql
|
66
71
|
Requires-Dist: pytest>=7.0.0; extra == "test-postgresql"
|
67
72
|
Requires-Dist: psycopg2-binary>=2.9.0; extra == "test-postgresql"
|
@@ -2,17 +2,18 @@ datasketch>=1.5.0
|
|
2
2
|
tqdm>=4.60.0
|
3
3
|
SQLAlchemy>=1.4.0
|
4
4
|
pydantic>=2.0.0
|
5
|
+
pandas>=1.3.0
|
6
|
+
requests>=2.25.0
|
5
7
|
|
6
8
|
[all]
|
7
9
|
psycopg2-binary>=2.9.0
|
8
10
|
mysql-connector-python>=8.0.0
|
9
11
|
mariadb>=1.1.0
|
10
12
|
pyodbc>=4.0.0
|
13
|
+
pymssql>=2.3.0
|
11
14
|
cx_Oracle>=8.3.0
|
12
|
-
|
15
|
+
oracledb>=3.0.0
|
13
16
|
supabase>=2.0.0
|
14
|
-
postgrest-py>=0.16.0
|
15
|
-
gotrue-py>=2.0.0
|
16
17
|
|
17
18
|
[dev]
|
18
19
|
pytest>=7.0.0
|
@@ -21,9 +22,10 @@ black>=22.0.0
|
|
21
22
|
flake8>=5.0.0
|
22
23
|
mypy>=1.0.0
|
23
24
|
pre-commit>=3.0.0
|
24
|
-
|
25
|
-
|
26
|
-
|
25
|
+
build>=1.0.0
|
26
|
+
twine>=4.0.0
|
27
|
+
psutil>=5.8.0
|
28
|
+
docker>=6.0.0
|
27
29
|
|
28
30
|
[mariadb]
|
29
31
|
mariadb>=1.1.0
|
@@ -33,6 +35,7 @@ mysql-connector-python>=8.0.0
|
|
33
35
|
|
34
36
|
[oracle]
|
35
37
|
cx_Oracle>=8.3.0
|
38
|
+
oracledb>=3.0.0
|
36
39
|
|
37
40
|
[postgresql]
|
38
41
|
psycopg2-binary>=2.9.0
|
@@ -41,11 +44,12 @@ psycopg2-binary>=2.9.0
|
|
41
44
|
|
42
45
|
[sqlserver]
|
43
46
|
pyodbc>=4.0.0
|
47
|
+
pymssql>=2.3.0
|
44
48
|
|
45
49
|
[supabase]
|
46
50
|
supabase>=2.0.0
|
47
|
-
postgrest-py>=0.
|
48
|
-
gotrue-py>=
|
51
|
+
postgrest-py>=0.10.0
|
52
|
+
gotrue-py>=1.0.0
|
49
53
|
|
50
54
|
[test-mysql]
|
51
55
|
pytest>=7.0.0
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/helpers/multi_db_generator.py
RENAMED
File without changes
|
{thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager/helpers/preprocess_values.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{thoth_dbmanager-0.4.1 → thoth_dbmanager-0.4.3}/thoth_dbmanager.egg-info/dependency_links.txt
RENAMED
File without changes
|