dbtk 0.8.0__tar.gz → 0.8.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dbtk-0.8.0 → dbtk-0.8.2}/PKG-INFO +50 -7
- {dbtk-0.8.0 → dbtk-0.8.2}/README.md +45 -4
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/__init__.py +1 -1
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/cli.py +2 -1
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/base_surge.py +2 -2
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/bulk_surge.py +2 -2
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/managers.py +2 -1
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/table.py +1 -1
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/transforms/address.py +1 -1
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/logging_utils.py +1 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/base.py +1 -2
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/fixed_width.py +2 -2
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/base.py +5 -5
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/xml.py +3 -3
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk.egg-info/PKG-INFO +50 -7
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk.egg-info/requires.txt +10 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/pyproject.toml +9 -3
- {dbtk-0.8.0 → dbtk-0.8.2}/LICENSE.txt +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/config.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/cursors.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/database.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/dbtk_sample.yml +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/defaults.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/__init__.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/config_generators.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/data_surge.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/transforms/__init__.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/transforms/core.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/transforms/database.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/transforms/datetime.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/transforms/email.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/transforms/phone.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/formats/__init__.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/formats/edi.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/__init__.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/csv.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/data_frame.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/excel.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/json.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/utils.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/xml.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/record.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/utils.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/__init__.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/csv.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/database.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/excel.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/fixed_width.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/json.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/utils.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk.egg-info/SOURCES.txt +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk.egg-info/dependency_links.txt +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk.egg-info/entry_points.txt +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/dbtk.egg-info/top_level.txt +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/setup.cfg +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_bulk.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_config.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_data_bending.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_edi_writer.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_excel_writer.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_fixed_width_record.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_logging_utils.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_readers.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_table.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_table_new_features.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_transforms_core.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_transforms_datetime.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_transforms_phone.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_validation_collector.py +0 -0
- {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_writers.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dbtk
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.2
|
|
4
4
|
Summary: Data Benders Toolkit - A lightweight toolkit for data integration, ELT and ETL
|
|
5
5
|
Author: Scott Bailey <scottrbailey@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -39,7 +39,8 @@ Requires-Dist: lxml>=4.6.0; extra == "recommended"
|
|
|
39
39
|
Requires-Dist: openpyxl>=3.0.0; extra == "recommended"
|
|
40
40
|
Requires-Dist: phonenumbers>=8.12.0; extra == "recommended"
|
|
41
41
|
Requires-Dist: python-dateutil>=2.8.0; extra == "recommended"
|
|
42
|
-
Requires-Dist: usaddress
|
|
42
|
+
Requires-Dist: usaddress==0.5.10; python_version < "3.7" and extra == "recommended"
|
|
43
|
+
Requires-Dist: usaddress>=0.5.11; python_version >= "3.7" and extra == "recommended"
|
|
43
44
|
Provides-Extra: postgresql
|
|
44
45
|
Requires-Dist: psycopg2-binary>=2.9; extra == "postgresql"
|
|
45
46
|
Requires-Dist: psycopg[binary]>=3.1; extra == "postgresql"
|
|
@@ -57,7 +58,8 @@ Requires-Dist: openpyxl>=3.0.0; extra == "all"
|
|
|
57
58
|
Requires-Dist: xlrd>=1.2.0; extra == "all"
|
|
58
59
|
Requires-Dist: phonenumbers>=8.12.0; extra == "all"
|
|
59
60
|
Requires-Dist: python-dateutil>=2.8.0; extra == "all"
|
|
60
|
-
Requires-Dist: usaddress
|
|
61
|
+
Requires-Dist: usaddress==0.5.10; python_version < "3.7" and extra == "all"
|
|
62
|
+
Requires-Dist: usaddress>=0.5.11; python_version >= "3.7" and extra == "all"
|
|
61
63
|
Provides-Extra: dev
|
|
62
64
|
Requires-Dist: pytest>=6.0; extra == "dev"
|
|
63
65
|
Requires-Dist: pytest-cov>=2.10; extra == "dev"
|
|
@@ -70,7 +72,7 @@ Dynamic: license-file
|
|
|
70
72
|
# DBTK - Data Benders Toolkit
|
|
71
73
|
|
|
72
74
|
<div style="float: right; padding: 20px">
|
|
73
|
-
<img src="/docs/assets/databender.png" height="240" align="right" />
|
|
75
|
+
<img src="https://raw.githubusercontent.com/scottrbailey/dbtk/main/docs/assets/databender.png" height="240" align="right" />
|
|
74
76
|
</div>
|
|
75
77
|
|
|
76
78
|
**Control and Manipulate the Flow of Data** - A lightweight Python toolkit for data integration, transformation, and movement between systems.
|
|
@@ -80,9 +82,9 @@ Extract data from various sources, transform it through powerful operations, and
|
|
|
80
82
|
This library is designed by and for data integrators.
|
|
81
83
|
|
|
82
84
|
**DBTK aims to be fast and memory-efficient at every turn.** But it was designed to boost your productivity first and foremost.
|
|
83
|
-
You have dozens
|
|
84
|
-
environment has
|
|
85
|
-
boilerplate code over and over
|
|
85
|
+
You have dozens, possibly hundreds, of interfaces, impossible deadlines, and multiple projects all happening at once. Your
|
|
86
|
+
environment has different relational databases, each has its own quirks. You just want to get stuff done instead of writing the same
|
|
87
|
+
boilerplate code over and over.
|
|
86
88
|
|
|
87
89
|
**Design philosophy:** Modern databases excel at aggregating and transforming data at scale. DBTK embraces
|
|
88
90
|
this by focusing on what Python does well: flexible record-by-record transformations,
|
|
@@ -93,6 +95,47 @@ your database handle it. If you need dataframes and heavy analytics - reach for
|
|
|
93
95
|
or polars. DBTK sits in between: getting your data where it needs to be, cleaned and
|
|
94
96
|
validated along the way.
|
|
95
97
|
|
|
98
|
+
## Why DBTK?
|
|
99
|
+
|
|
100
|
+
Most ETL tools fall into one of two traps:
|
|
101
|
+
|
|
102
|
+
- **Verbose boilerplate hell** — SQLAlchemy + pandas + custom loops, repeated across every
|
|
103
|
+
interface with minor variations, slowly accreting defensive code for every edge case
|
|
104
|
+
- **Opaque black box** — so much is hidden that debugging feels like reverse-engineering
|
|
105
|
+
someone else's code
|
|
106
|
+
|
|
107
|
+
DBTK threads the needle. It's declarative enough to eliminate repetition, but explicit enough
|
|
108
|
+
that you stay in control. When something breaks, you know exactly where to look.
|
|
109
|
+
|
|
110
|
+
The architecture is intentionally layered — use what you need, skip what you don't:
|
|
111
|
+
|
|
112
|
+
```
|
|
113
|
+
Record → ergonomic row handling, memory-efficient at scale
|
|
114
|
+
Table → field mapping, transforms, validation, upserts
|
|
115
|
+
DataSurge → batched inserts with progress tracking and stats
|
|
116
|
+
BulkSurge → direct bulk loads (SQL*Loader, BCP, COPY) for maximum throughput
|
|
117
|
+
readers/writers → consistent API across every file format and compression type
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
When developers convert existing jobs to DBTK, the result can be **half to a quarter the
|
|
121
|
+
original code**. That reduction comes from specific things DBTK just handles:
|
|
122
|
+
|
|
123
|
+
- **Key column validation** — DBTK throws a clear error if a key field is missing from the source.
|
|
124
|
+
No more writing null-guards before every upsert.
|
|
125
|
+
- **Safe partial updates** — if a field is missing from the source, DBTK excludes it from the
|
|
126
|
+
UPDATE rather than overwriting with NULL. No more "did I just wipe a column?" paranoia.
|
|
127
|
+
- **Batch loop elimination** — DataSurge handles chunking, committing, retries, and statistics.
|
|
128
|
+
No more hand-rolled batch loops.
|
|
129
|
+
- **Zero-config logging** — one line sets up timestamped log files with auto-cleanup and a global
|
|
130
|
+
error flag. No logging boilerplate scattered through your pipeline.
|
|
131
|
+
- **TableLookup shorthand** — define a lookup or validation in ~40 characters:
|
|
132
|
+
`'fn': 'validate:ranks:rank_code:preload'`. The `preload` hint pre-caches the table before
|
|
133
|
+
processing starts. What would otherwise be a custom class or 30-line function is a string.
|
|
134
|
+
|
|
135
|
+
The code that remains is shorter, clearer, and still has all the functionality and checks. You
|
|
136
|
+
finish the job and think *"that was satisfyingly elegant"* — not because corners were cut, but
|
|
137
|
+
because the tool was collaborating with you instead of making you fight it.
|
|
138
|
+
|
|
96
139
|
**Speed and Memory** The primary objective of DBTK is to give data integrators an elegant toolkit to speed up your development.
|
|
97
140
|
But DBTK's throughput and memory usage are very good. BulkSurge streaming from a polars and doing direct loads to PostgreSQL will
|
|
98
141
|
process 1M rows in 3-4 seconds. But even with a standard Python csv reader and numerous column transforms, DataSurge is able to
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# DBTK - Data Benders Toolkit
|
|
2
2
|
|
|
3
3
|
<div style="float: right; padding: 20px">
|
|
4
|
-
<img src="/docs/assets/databender.png" height="240" align="right" />
|
|
4
|
+
<img src="https://raw.githubusercontent.com/scottrbailey/dbtk/main/docs/assets/databender.png" height="240" align="right" />
|
|
5
5
|
</div>
|
|
6
6
|
|
|
7
7
|
**Control and Manipulate the Flow of Data** - A lightweight Python toolkit for data integration, transformation, and movement between systems.
|
|
@@ -11,9 +11,9 @@ Extract data from various sources, transform it through powerful operations, and
|
|
|
11
11
|
This library is designed by and for data integrators.
|
|
12
12
|
|
|
13
13
|
**DBTK aims to be fast and memory-efficient at every turn.** But it was designed to boost your productivity first and foremost.
|
|
14
|
-
You have dozens
|
|
15
|
-
environment has
|
|
16
|
-
boilerplate code over and over
|
|
14
|
+
You have dozens, possibly hundreds, of interfaces, impossible deadlines, and multiple projects all happening at once. Your
|
|
15
|
+
environment has different relational databases, each has its own quirks. You just want to get stuff done instead of writing the same
|
|
16
|
+
boilerplate code over and over.
|
|
17
17
|
|
|
18
18
|
**Design philosophy:** Modern databases excel at aggregating and transforming data at scale. DBTK embraces
|
|
19
19
|
this by focusing on what Python does well: flexible record-by-record transformations,
|
|
@@ -24,6 +24,47 @@ your database handle it. If you need dataframes and heavy analytics - reach for
|
|
|
24
24
|
or polars. DBTK sits in between: getting your data where it needs to be, cleaned and
|
|
25
25
|
validated along the way.
|
|
26
26
|
|
|
27
|
+
## Why DBTK?
|
|
28
|
+
|
|
29
|
+
Most ETL tools fall into one of two traps:
|
|
30
|
+
|
|
31
|
+
- **Verbose boilerplate hell** — SQLAlchemy + pandas + custom loops, repeated across every
|
|
32
|
+
interface with minor variations, slowly accreting defensive code for every edge case
|
|
33
|
+
- **Opaque black box** — so much is hidden that debugging feels like reverse-engineering
|
|
34
|
+
someone else's code
|
|
35
|
+
|
|
36
|
+
DBTK threads the needle. It's declarative enough to eliminate repetition, but explicit enough
|
|
37
|
+
that you stay in control. When something breaks, you know exactly where to look.
|
|
38
|
+
|
|
39
|
+
The architecture is intentionally layered — use what you need, skip what you don't:
|
|
40
|
+
|
|
41
|
+
```
|
|
42
|
+
Record → ergonomic row handling, memory-efficient at scale
|
|
43
|
+
Table → field mapping, transforms, validation, upserts
|
|
44
|
+
DataSurge → batched inserts with progress tracking and stats
|
|
45
|
+
BulkSurge → direct bulk loads (SQL*Loader, BCP, COPY) for maximum throughput
|
|
46
|
+
readers/writers → consistent API across every file format and compression type
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
When developers convert existing jobs to DBTK, the result can be **half to a quarter the
|
|
50
|
+
original code**. That reduction comes from specific things DBTK just handles:
|
|
51
|
+
|
|
52
|
+
- **Key column validation** — DBTK throws a clear error if a key field is missing from the source.
|
|
53
|
+
No more writing null-guards before every upsert.
|
|
54
|
+
- **Safe partial updates** — if a field is missing from the source, DBTK excludes it from the
|
|
55
|
+
UPDATE rather than overwriting with NULL. No more "did I just wipe a column?" paranoia.
|
|
56
|
+
- **Batch loop elimination** — DataSurge handles chunking, committing, retries, and statistics.
|
|
57
|
+
No more hand-rolled batch loops.
|
|
58
|
+
- **Zero-config logging** — one line sets up timestamped log files with auto-cleanup and a global
|
|
59
|
+
error flag. No logging boilerplate scattered through your pipeline.
|
|
60
|
+
- **TableLookup shorthand** — define a lookup or validation in ~40 characters:
|
|
61
|
+
`'fn': 'validate:ranks:rank_code:preload'`. The `preload` hint pre-caches the table before
|
|
62
|
+
processing starts. What would otherwise be a custom class or 30-line function is a string.
|
|
63
|
+
|
|
64
|
+
The code that remains is shorter, clearer, and still has all the functionality and checks. You
|
|
65
|
+
finish the job and think *"that was satisfyingly elegant"* — not because corners were cut, but
|
|
66
|
+
because the tool was collaborating with you instead of making you fight it.
|
|
67
|
+
|
|
27
68
|
**Speed and Memory** The primary objective of DBTK is to give data integrators an elegant toolkit to speed up your development.
|
|
28
69
|
But DBTK's throughput and memory usage are very good. BulkSurge streaming from a polars and doing direct loads to PostgreSQL will
|
|
29
70
|
process 1M rows in 3-4 seconds. But even with a standard Python csv reader and numerous column transforms, DataSurge is able to
|
|
@@ -36,9 +36,9 @@ from .database import Database
|
|
|
36
36
|
from .config import connect, set_config_file
|
|
37
37
|
from .cursors import Cursor
|
|
38
38
|
from .logging_utils import setup_logging, cleanup_old_logs, errors_logged
|
|
39
|
-
from . import etl
|
|
40
39
|
from . import readers
|
|
41
40
|
from . import writers
|
|
41
|
+
from . import etl
|
|
42
42
|
|
|
43
43
|
# Simple, clean exports
|
|
44
44
|
__all__ = [
|
|
@@ -114,7 +114,8 @@ def checkup():
|
|
|
114
114
|
|
|
115
115
|
def main():
|
|
116
116
|
parser = argparse.ArgumentParser(prog='dbtk', description='DBTK command-line utilities')
|
|
117
|
-
subparsers = parser.add_subparsers(dest='command'
|
|
117
|
+
subparsers = parser.add_subparsers(dest='command')
|
|
118
|
+
subparsers.required = True # required= kwarg not supported in Python 3.6
|
|
118
119
|
|
|
119
120
|
# checkup
|
|
120
121
|
subparsers.add_parser('checkup', help='Check for dependencies and configuration issues')
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import logging
|
|
3
3
|
import time
|
|
4
4
|
from abc import ABC, abstractmethod
|
|
5
|
-
from typing import Iterable, Generator, Optional
|
|
5
|
+
from typing import Iterable, Generator, Optional, Union
|
|
6
6
|
import datetime as dt
|
|
7
7
|
import tempfile
|
|
8
8
|
from pathlib import Path
|
|
@@ -183,7 +183,7 @@ class BaseSurge(ABC):
|
|
|
183
183
|
if batch:
|
|
184
184
|
yield batch
|
|
185
185
|
|
|
186
|
-
def _resolve_file_path(self, path_input: Optional[str
|
|
186
|
+
def _resolve_file_path(self, path_input: Optional[Union[str, Path]] = None, extension: str = '.csv') -> Path:
|
|
187
187
|
"""
|
|
188
188
|
Resolve an output file path from user input.
|
|
189
189
|
|
|
@@ -464,7 +464,7 @@ class BulkSurge(BaseSurge):
|
|
|
464
464
|
logger.debug(f'sqlldr userid={user}/<PASSWORD>@{db} control={ctl_path} log={log_path}')
|
|
465
465
|
|
|
466
466
|
try:
|
|
467
|
-
result = subprocess.run(cmd,
|
|
467
|
+
result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
|
|
468
468
|
|
|
469
469
|
if result.returncode == 0:
|
|
470
470
|
logger.info("SQL*Loader completed successfully")
|
|
@@ -565,7 +565,7 @@ class BulkSurge(BaseSurge):
|
|
|
565
565
|
else:
|
|
566
566
|
cmd += ['-T',] # integrated auth
|
|
567
567
|
# Run BCP
|
|
568
|
-
result = subprocess.run(cmd,
|
|
568
|
+
result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
|
|
569
569
|
if result.returncode != 0:
|
|
570
570
|
error_msg = f"bcp failed with exit code {result.returncode}"
|
|
571
571
|
if result.stderr:
|
|
@@ -606,7 +606,8 @@ class ValidationCollector:
|
|
|
606
606
|
if self.desc_field:
|
|
607
607
|
return result.get(self.desc_field, "")
|
|
608
608
|
for field in ("title", "description", "name", "label"):
|
|
609
|
-
|
|
609
|
+
val = result.get(field)
|
|
610
|
+
if val:
|
|
610
611
|
return val
|
|
611
612
|
return str(result)
|
|
612
613
|
|
|
@@ -63,7 +63,7 @@ class Table:
|
|
|
63
63
|
'user_id': {'default': lambda: conf_vars['user_id']},
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
-
* **fn** (callable
|
|
66
|
+
* **fn** (Union[callable, List[callable], str], optional):
|
|
67
67
|
Transformation function(s) to apply to the source value.
|
|
68
68
|
callable: applied directly; list: functions applied in order (pipeline);
|
|
69
69
|
str: magic shorthand (e.g. 'int', 'maxlen:255', 'lookup:table:col:val').
|
|
@@ -93,7 +93,7 @@ def _check_usaddress():
|
|
|
93
93
|
if not HAS_USADDRESS:
|
|
94
94
|
raise ImportError(
|
|
95
95
|
"Address functionality requires the usaddress library. "
|
|
96
|
-
"Install with: pip install usaddress"
|
|
96
|
+
"Install with: pip install usaddress==0.5.10 (Python 3.6) or pip install usaddress (Python 3.7+)"
|
|
97
97
|
)
|
|
98
98
|
|
|
99
99
|
|
|
@@ -151,7 +151,6 @@ class Reader(ABC):
|
|
|
151
151
|
|
|
152
152
|
Common Features
|
|
153
153
|
---------------
|
|
154
|
-
* **Automatic header cleaning** - Standardize messy column names
|
|
155
154
|
* **Row number tracking** - Automatic _row_num field for debugging
|
|
156
155
|
* **Record skipping** - Skip header rows or bad data
|
|
157
156
|
* **Record limiting** - Process only first N records
|
|
@@ -172,7 +171,7 @@ class Reader(ABC):
|
|
|
172
171
|
Maximum number of rows to read. None (default) reads all rows.
|
|
173
172
|
null_values : str, list, tuple, or set, optional
|
|
174
173
|
Values to convert to None. Can be a single string or a collection of strings.
|
|
175
|
-
Common examples: '\\N'
|
|
174
|
+
Common examples: '\\N', 'NULL', 'NA', '' (empty string)
|
|
176
175
|
|
|
177
176
|
Example
|
|
178
177
|
-------
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import logging
|
|
6
6
|
import os
|
|
7
|
-
from typing import TextIO, List, Dict, Any, Optional, Iterator
|
|
7
|
+
from typing import TextIO, List, Dict, Any, Optional, Iterator, Type
|
|
8
8
|
|
|
9
9
|
from .base import Reader
|
|
10
10
|
from ..record import Record, FixedWidthRecord
|
|
@@ -271,7 +271,7 @@ class EDIReader(FixedReader):
|
|
|
271
271
|
def _get_columns(self, type_code: str) -> List[FixedColumn]:
|
|
272
272
|
return self.columns.get(type_code)
|
|
273
273
|
|
|
274
|
-
def _get_factory(self, type_code: str) ->
|
|
274
|
+
def _get_factory(self, type_code: str) -> Type[Record]:
|
|
275
275
|
if type_code not in self._type_factories:
|
|
276
276
|
cols = self._get_columns(type_code)
|
|
277
277
|
if cols is None:
|
|
@@ -8,7 +8,7 @@ import logging
|
|
|
8
8
|
import sys
|
|
9
9
|
from abc import ABC, abstractmethod
|
|
10
10
|
from pathlib import Path
|
|
11
|
-
from typing import Any, BinaryIO, Iterable, Iterator, List, Optional, TextIO, Union
|
|
11
|
+
from typing import Any, BinaryIO, Iterable, Iterator, List, Optional, TextIO, Tuple, Union
|
|
12
12
|
|
|
13
13
|
from ..utils import RecordLike, to_string
|
|
14
14
|
|
|
@@ -170,7 +170,7 @@ class BaseWriter(ABC):
|
|
|
170
170
|
raise ValueError(f"{self.__class__.__name__} requires an output file path")
|
|
171
171
|
self.output_path = Path(file)
|
|
172
172
|
|
|
173
|
-
def _open_file_handle(self, mode: str = "w") ->
|
|
173
|
+
def _open_file_handle(self, mode: str = "w") -> Tuple[Union[TextIO, BinaryIO], bool]:
|
|
174
174
|
"""
|
|
175
175
|
Open the output file/stream and return (handle, should_close).
|
|
176
176
|
|
|
@@ -181,7 +181,7 @@ class BaseWriter(ABC):
|
|
|
181
181
|
|
|
182
182
|
Returns
|
|
183
183
|
-------
|
|
184
|
-
|
|
184
|
+
Tuple[Union[TextIO, BinaryIO], bool]
|
|
185
185
|
(file_handle, should_close_flag)
|
|
186
186
|
"""
|
|
187
187
|
if self.file is None:
|
|
@@ -267,7 +267,7 @@ class BaseWriter(ABC):
|
|
|
267
267
|
|
|
268
268
|
def _get_data_iterator(
|
|
269
269
|
self, data: Iterable[RecordLike], columns: Optional[List[str]] = None
|
|
270
|
-
) ->
|
|
270
|
+
) -> Tuple[Optional[Iterator], Optional[List[str]]]:
|
|
271
271
|
"""
|
|
272
272
|
Get data iterator and column names.
|
|
273
273
|
|
|
@@ -280,7 +280,7 @@ class BaseWriter(ABC):
|
|
|
280
280
|
|
|
281
281
|
Returns
|
|
282
282
|
-------
|
|
283
|
-
|
|
283
|
+
Tuple[Optional[Iterator], Optional[List[str]]]
|
|
284
284
|
(iterator, column_names)
|
|
285
285
|
"""
|
|
286
286
|
if not data:
|
|
@@ -7,7 +7,7 @@ import io
|
|
|
7
7
|
import logging
|
|
8
8
|
import re
|
|
9
9
|
import sys
|
|
10
|
-
from typing import Any, BinaryIO, List, Optional, TextIO, Union
|
|
10
|
+
from typing import Any, BinaryIO, List, Optional, TextIO, Tuple, Union
|
|
11
11
|
from pathlib import Path
|
|
12
12
|
|
|
13
13
|
try:
|
|
@@ -196,7 +196,7 @@ class XMLStreamer(BatchWriter):
|
|
|
196
196
|
preserve_types=True, # We'll convert in _prepare_record_for_xml
|
|
197
197
|
)
|
|
198
198
|
|
|
199
|
-
def _open_file_handle(self, mode: str = 'wb') ->
|
|
199
|
+
def _open_file_handle(self, mode: str = 'wb') -> Tuple[BinaryIO, bool]:
|
|
200
200
|
"""
|
|
201
201
|
Override to use binary mode and validate binary streams.
|
|
202
202
|
|
|
@@ -207,7 +207,7 @@ class XMLStreamer(BatchWriter):
|
|
|
207
207
|
|
|
208
208
|
Returns
|
|
209
209
|
-------
|
|
210
|
-
|
|
210
|
+
Tuple[BinaryIO, bool]
|
|
211
211
|
(file_handle, should_close_flag)
|
|
212
212
|
|
|
213
213
|
Raises
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dbtk
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.2
|
|
4
4
|
Summary: Data Benders Toolkit - A lightweight toolkit for data integration, ELT and ETL
|
|
5
5
|
Author: Scott Bailey <scottrbailey@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -39,7 +39,8 @@ Requires-Dist: lxml>=4.6.0; extra == "recommended"
|
|
|
39
39
|
Requires-Dist: openpyxl>=3.0.0; extra == "recommended"
|
|
40
40
|
Requires-Dist: phonenumbers>=8.12.0; extra == "recommended"
|
|
41
41
|
Requires-Dist: python-dateutil>=2.8.0; extra == "recommended"
|
|
42
|
-
Requires-Dist: usaddress
|
|
42
|
+
Requires-Dist: usaddress==0.5.10; python_version < "3.7" and extra == "recommended"
|
|
43
|
+
Requires-Dist: usaddress>=0.5.11; python_version >= "3.7" and extra == "recommended"
|
|
43
44
|
Provides-Extra: postgresql
|
|
44
45
|
Requires-Dist: psycopg2-binary>=2.9; extra == "postgresql"
|
|
45
46
|
Requires-Dist: psycopg[binary]>=3.1; extra == "postgresql"
|
|
@@ -57,7 +58,8 @@ Requires-Dist: openpyxl>=3.0.0; extra == "all"
|
|
|
57
58
|
Requires-Dist: xlrd>=1.2.0; extra == "all"
|
|
58
59
|
Requires-Dist: phonenumbers>=8.12.0; extra == "all"
|
|
59
60
|
Requires-Dist: python-dateutil>=2.8.0; extra == "all"
|
|
60
|
-
Requires-Dist: usaddress
|
|
61
|
+
Requires-Dist: usaddress==0.5.10; python_version < "3.7" and extra == "all"
|
|
62
|
+
Requires-Dist: usaddress>=0.5.11; python_version >= "3.7" and extra == "all"
|
|
61
63
|
Provides-Extra: dev
|
|
62
64
|
Requires-Dist: pytest>=6.0; extra == "dev"
|
|
63
65
|
Requires-Dist: pytest-cov>=2.10; extra == "dev"
|
|
@@ -70,7 +72,7 @@ Dynamic: license-file
|
|
|
70
72
|
# DBTK - Data Benders Toolkit
|
|
71
73
|
|
|
72
74
|
<div style="float: right; padding: 20px">
|
|
73
|
-
<img src="/docs/assets/databender.png" height="240" align="right" />
|
|
75
|
+
<img src="https://raw.githubusercontent.com/scottrbailey/dbtk/main/docs/assets/databender.png" height="240" align="right" />
|
|
74
76
|
</div>
|
|
75
77
|
|
|
76
78
|
**Control and Manipulate the Flow of Data** - A lightweight Python toolkit for data integration, transformation, and movement between systems.
|
|
@@ -80,9 +82,9 @@ Extract data from various sources, transform it through powerful operations, and
|
|
|
80
82
|
This library is designed by and for data integrators.
|
|
81
83
|
|
|
82
84
|
**DBTK aims to be fast and memory-efficient at every turn.** But it was designed to boost your productivity first and foremost.
|
|
83
|
-
You have dozens
|
|
84
|
-
environment has
|
|
85
|
-
boilerplate code over and over
|
|
85
|
+
You have dozens, possibly hundreds, of interfaces, impossible deadlines, and multiple projects all happening at once. Your
|
|
86
|
+
environment has different relational databases, each has its own quirks. You just want to get stuff done instead of writing the same
|
|
87
|
+
boilerplate code over and over.
|
|
86
88
|
|
|
87
89
|
**Design philosophy:** Modern databases excel at aggregating and transforming data at scale. DBTK embraces
|
|
88
90
|
this by focusing on what Python does well: flexible record-by-record transformations,
|
|
@@ -93,6 +95,47 @@ your database handle it. If you need dataframes and heavy analytics - reach for
|
|
|
93
95
|
or polars. DBTK sits in between: getting your data where it needs to be, cleaned and
|
|
94
96
|
validated along the way.
|
|
95
97
|
|
|
98
|
+
## Why DBTK?
|
|
99
|
+
|
|
100
|
+
Most ETL tools fall into one of two traps:
|
|
101
|
+
|
|
102
|
+
- **Verbose boilerplate hell** — SQLAlchemy + pandas + custom loops, repeated across every
|
|
103
|
+
interface with minor variations, slowly accreting defensive code for every edge case
|
|
104
|
+
- **Opaque black box** — so much is hidden that debugging feels like reverse-engineering
|
|
105
|
+
someone else's code
|
|
106
|
+
|
|
107
|
+
DBTK threads the needle. It's declarative enough to eliminate repetition, but explicit enough
|
|
108
|
+
that you stay in control. When something breaks, you know exactly where to look.
|
|
109
|
+
|
|
110
|
+
The architecture is intentionally layered — use what you need, skip what you don't:
|
|
111
|
+
|
|
112
|
+
```
|
|
113
|
+
Record → ergonomic row handling, memory-efficient at scale
|
|
114
|
+
Table → field mapping, transforms, validation, upserts
|
|
115
|
+
DataSurge → batched inserts with progress tracking and stats
|
|
116
|
+
BulkSurge → direct bulk loads (SQL*Loader, BCP, COPY) for maximum throughput
|
|
117
|
+
readers/writers → consistent API across every file format and compression type
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
When developers convert existing jobs to DBTK, the result can be **half to a quarter the
|
|
121
|
+
original code**. That reduction comes from specific things DBTK just handles:
|
|
122
|
+
|
|
123
|
+
- **Key column validation** — DBTK throws a clear error if a key field is missing from the source.
|
|
124
|
+
No more writing null-guards before every upsert.
|
|
125
|
+
- **Safe partial updates** — if a field is missing from the source, DBTK excludes it from the
|
|
126
|
+
UPDATE rather than overwriting with NULL. No more "did I just wipe a column?" paranoia.
|
|
127
|
+
- **Batch loop elimination** — DataSurge handles chunking, committing, retries, and statistics.
|
|
128
|
+
No more hand-rolled batch loops.
|
|
129
|
+
- **Zero-config logging** — one line sets up timestamped log files with auto-cleanup and a global
|
|
130
|
+
error flag. No logging boilerplate scattered through your pipeline.
|
|
131
|
+
- **TableLookup shorthand** — define a lookup or validation in ~40 characters:
|
|
132
|
+
`'fn': 'validate:ranks:rank_code:preload'`. The `preload` hint pre-caches the table before
|
|
133
|
+
processing starts. What would otherwise be a custom class or 30-line function is a string.
|
|
134
|
+
|
|
135
|
+
The code that remains is shorter, clearer, and still has all the functionality and checks. You
|
|
136
|
+
finish the job and think *"that was satisfyingly elegant"* — not because corners were cut, but
|
|
137
|
+
because the tool was collaborating with you instead of making you fight it.
|
|
138
|
+
|
|
96
139
|
**Speed and Memory** The primary objective of DBTK is to give data integrators an elegant toolkit to speed up your development.
|
|
97
140
|
But DBTK's throughput and memory usage are very good. BulkSurge streaming from a polars and doing direct loads to PostgreSQL will
|
|
98
141
|
process 1M rows in 3-4 seconds. But even with a standard Python csv reader and numerous column transforms, DataSurge is able to
|
|
@@ -11,6 +11,11 @@ openpyxl>=3.0.0
|
|
|
11
11
|
xlrd>=1.2.0
|
|
12
12
|
phonenumbers>=8.12.0
|
|
13
13
|
python-dateutil>=2.8.0
|
|
14
|
+
|
|
15
|
+
[all:python_version < "3.7"]
|
|
16
|
+
usaddress==0.5.10
|
|
17
|
+
|
|
18
|
+
[all:python_version >= "3.7"]
|
|
14
19
|
usaddress>=0.5.11
|
|
15
20
|
|
|
16
21
|
[dev]
|
|
@@ -50,6 +55,11 @@ lxml>=4.6.0
|
|
|
50
55
|
openpyxl>=3.0.0
|
|
51
56
|
phonenumbers>=8.12.0
|
|
52
57
|
python-dateutil>=2.8.0
|
|
58
|
+
|
|
59
|
+
[recommended:python_version < "3.7"]
|
|
60
|
+
usaddress==0.5.10
|
|
61
|
+
|
|
62
|
+
[recommended:python_version >= "3.7"]
|
|
53
63
|
usaddress>=0.5.11
|
|
54
64
|
|
|
55
65
|
[sqlserver]
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "dbtk"
|
|
7
|
-
version = "0.8.
|
|
7
|
+
version = "0.8.2"
|
|
8
8
|
description = "Data Benders Toolkit - A lightweight toolkit for data integration, ELT and ETL"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.6"
|
|
@@ -44,7 +44,8 @@ recommended = [
|
|
|
44
44
|
"openpyxl>=3.0.0",
|
|
45
45
|
"phonenumbers>=8.12.0",
|
|
46
46
|
"python-dateutil>=2.8.0",
|
|
47
|
-
"usaddress
|
|
47
|
+
"usaddress==0.5.10;python_version<'3.7'",
|
|
48
|
+
"usaddress>=0.5.11;python_version>='3.7'"
|
|
48
49
|
]
|
|
49
50
|
postgresql = ["psycopg2-binary>=2.9", "psycopg[binary]>=3.1"]
|
|
50
51
|
oracle = ["oracledb>=1.0;python_version>='3.7'", # For modern systems
|
|
@@ -59,7 +60,8 @@ all = [
|
|
|
59
60
|
"xlrd>=1.2.0",
|
|
60
61
|
"phonenumbers>=8.12.0",
|
|
61
62
|
"python-dateutil>=2.8.0",
|
|
62
|
-
"usaddress
|
|
63
|
+
"usaddress==0.5.10;python_version<'3.7'",
|
|
64
|
+
"usaddress>=0.5.11;python_version>='3.7'"
|
|
63
65
|
]
|
|
64
66
|
dev = ["pytest>=6.0", "pytest-cov>=2.10"]
|
|
65
67
|
docs = [
|
|
@@ -76,5 +78,9 @@ Homepage = "https://github.com/scottrbailey/dbtk"
|
|
|
76
78
|
Documentation = "https://dbtk.readthedocs.io"
|
|
77
79
|
Repository = "https://github.com/scottrbailey/dbtk.git"
|
|
78
80
|
|
|
81
|
+
[tool.setuptools.packages.find]
|
|
82
|
+
where = ["."]
|
|
83
|
+
include = ["dbtk*"]
|
|
84
|
+
|
|
79
85
|
[tool.setuptools.package-data]
|
|
80
86
|
dbtk = ["dbtk_sample.yml"]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|