dbtk 0.8.0__tar.gz → 0.8.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. {dbtk-0.8.0 → dbtk-0.8.2}/PKG-INFO +50 -7
  2. {dbtk-0.8.0 → dbtk-0.8.2}/README.md +45 -4
  3. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/__init__.py +1 -1
  4. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/cli.py +2 -1
  5. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/base_surge.py +2 -2
  6. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/bulk_surge.py +2 -2
  7. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/managers.py +2 -1
  8. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/table.py +1 -1
  9. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/transforms/address.py +1 -1
  10. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/logging_utils.py +1 -0
  11. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/base.py +1 -2
  12. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/fixed_width.py +2 -2
  13. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/base.py +5 -5
  14. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/xml.py +3 -3
  15. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk.egg-info/PKG-INFO +50 -7
  16. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk.egg-info/requires.txt +10 -0
  17. {dbtk-0.8.0 → dbtk-0.8.2}/pyproject.toml +9 -3
  18. {dbtk-0.8.0 → dbtk-0.8.2}/LICENSE.txt +0 -0
  19. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/config.py +0 -0
  20. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/cursors.py +0 -0
  21. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/database.py +0 -0
  22. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/dbtk_sample.yml +0 -0
  23. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/defaults.py +0 -0
  24. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/__init__.py +0 -0
  25. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/config_generators.py +0 -0
  26. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/data_surge.py +0 -0
  27. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/transforms/__init__.py +0 -0
  28. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/transforms/core.py +0 -0
  29. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/transforms/database.py +0 -0
  30. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/transforms/datetime.py +0 -0
  31. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/transforms/email.py +0 -0
  32. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/etl/transforms/phone.py +0 -0
  33. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/formats/__init__.py +0 -0
  34. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/formats/edi.py +0 -0
  35. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/__init__.py +0 -0
  36. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/csv.py +0 -0
  37. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/data_frame.py +0 -0
  38. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/excel.py +0 -0
  39. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/json.py +0 -0
  40. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/utils.py +0 -0
  41. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/readers/xml.py +0 -0
  42. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/record.py +0 -0
  43. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/utils.py +0 -0
  44. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/__init__.py +0 -0
  45. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/csv.py +0 -0
  46. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/database.py +0 -0
  47. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/excel.py +0 -0
  48. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/fixed_width.py +0 -0
  49. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/json.py +0 -0
  50. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk/writers/utils.py +0 -0
  51. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk.egg-info/SOURCES.txt +0 -0
  52. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk.egg-info/dependency_links.txt +0 -0
  53. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk.egg-info/entry_points.txt +0 -0
  54. {dbtk-0.8.0 → dbtk-0.8.2}/dbtk.egg-info/top_level.txt +0 -0
  55. {dbtk-0.8.0 → dbtk-0.8.2}/setup.cfg +0 -0
  56. {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_bulk.py +0 -0
  57. {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_config.py +0 -0
  58. {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_data_bending.py +0 -0
  59. {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_edi_writer.py +0 -0
  60. {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_excel_writer.py +0 -0
  61. {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_fixed_width_record.py +0 -0
  62. {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_logging_utils.py +0 -0
  63. {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_readers.py +0 -0
  64. {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_table.py +0 -0
  65. {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_table_new_features.py +0 -0
  66. {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_transforms_core.py +0 -0
  67. {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_transforms_datetime.py +0 -0
  68. {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_transforms_phone.py +0 -0
  69. {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_validation_collector.py +0 -0
  70. {dbtk-0.8.0 → dbtk-0.8.2}/tests/test_writers.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dbtk
3
- Version: 0.8.0
3
+ Version: 0.8.2
4
4
  Summary: Data Benders Toolkit - A lightweight toolkit for data integration, ELT and ETL
5
5
  Author: Scott Bailey <scottrbailey@gmail.com>
6
6
  License: MIT
@@ -39,7 +39,8 @@ Requires-Dist: lxml>=4.6.0; extra == "recommended"
39
39
  Requires-Dist: openpyxl>=3.0.0; extra == "recommended"
40
40
  Requires-Dist: phonenumbers>=8.12.0; extra == "recommended"
41
41
  Requires-Dist: python-dateutil>=2.8.0; extra == "recommended"
42
- Requires-Dist: usaddress>=0.5.11; extra == "recommended"
42
+ Requires-Dist: usaddress==0.5.10; python_version < "3.7" and extra == "recommended"
43
+ Requires-Dist: usaddress>=0.5.11; python_version >= "3.7" and extra == "recommended"
43
44
  Provides-Extra: postgresql
44
45
  Requires-Dist: psycopg2-binary>=2.9; extra == "postgresql"
45
46
  Requires-Dist: psycopg[binary]>=3.1; extra == "postgresql"
@@ -57,7 +58,8 @@ Requires-Dist: openpyxl>=3.0.0; extra == "all"
57
58
  Requires-Dist: xlrd>=1.2.0; extra == "all"
58
59
  Requires-Dist: phonenumbers>=8.12.0; extra == "all"
59
60
  Requires-Dist: python-dateutil>=2.8.0; extra == "all"
60
- Requires-Dist: usaddress>=0.5.11; extra == "all"
61
+ Requires-Dist: usaddress==0.5.10; python_version < "3.7" and extra == "all"
62
+ Requires-Dist: usaddress>=0.5.11; python_version >= "3.7" and extra == "all"
61
63
  Provides-Extra: dev
62
64
  Requires-Dist: pytest>=6.0; extra == "dev"
63
65
  Requires-Dist: pytest-cov>=2.10; extra == "dev"
@@ -70,7 +72,7 @@ Dynamic: license-file
70
72
  # DBTK - Data Benders Toolkit
71
73
 
72
74
  <div style="float: right; padding: 20px">
73
- <img src="/docs/assets/databender.png" height="240" align="right" />
75
+ <img src="https://raw.githubusercontent.com/scottrbailey/dbtk/main/docs/assets/databender.png" height="240" align="right" />
74
76
  </div>
75
77
 
76
78
  **Control and Manipulate the Flow of Data** - A lightweight Python toolkit for data integration, transformation, and movement between systems.
@@ -80,9 +82,9 @@ Extract data from various sources, transform it through powerful operations, and
80
82
  This library is designed by and for data integrators.
81
83
 
82
84
  **DBTK aims to be fast and memory-efficient at every turn.** But it was designed to boost your productivity first and foremost.
83
- You have dozens (possibly hundreds) of interfaces, impossible deadlines, and multiple projects all happening at once. Your
84
- environment has three or more different relational databases. You just want to get stuff done instead of writing the same
85
- boilerplate code over and over or stressing over differences on a database server you don't use very often.
85
+ You have dozens, possibly hundreds, of interfaces, impossible deadlines, and multiple projects all happening at once. Your
86
+ environment has different relational databases, each has its own quirks. You just want to get stuff done instead of writing the same
87
+ boilerplate code over and over.
86
88
 
87
89
  **Design philosophy:** Modern databases excel at aggregating and transforming data at scale. DBTK embraces
88
90
  this by focusing on what Python does well: flexible record-by-record transformations,
@@ -93,6 +95,47 @@ your database handle it. If you need dataframes and heavy analytics - reach for
93
95
  or polars. DBTK sits in between: getting your data where it needs to be, cleaned and
94
96
  validated along the way.
95
97
 
98
+ ## Why DBTK?
99
+
100
+ Most ETL tools fall into one of two traps:
101
+
102
+ - **Verbose boilerplate hell** — SQLAlchemy + pandas + custom loops, repeated across every
103
+ interface with minor variations, slowly accreting defensive code for every edge case
104
+ - **Opaque black box** — so much is hidden that debugging feels like reverse-engineering
105
+ someone else's code
106
+
107
+ DBTK threads the needle. It's declarative enough to eliminate repetition, but explicit enough
108
+ that you stay in control. When something breaks, you know exactly where to look.
109
+
110
+ The architecture is intentionally layered — use what you need, skip what you don't:
111
+
112
+ ```
113
+ Record → ergonomic row handling, memory-efficient at scale
114
+ Table → field mapping, transforms, validation, upserts
115
+ DataSurge → batched inserts with progress tracking and stats
116
+ BulkSurge → direct bulk loads (SQL*Loader, BCP, COPY) for maximum throughput
117
+ readers/writers → consistent API across every file format and compression type
118
+ ```
119
+
120
+ When developers convert existing jobs to DBTK, the result can be **half to a quarter the
121
+ original code**. That reduction comes from specific things DBTK just handles:
122
+
123
+ - **Key column validation** — DBTK throws a clear error if a key field is missing from the source.
124
+ No more writing null-guards before every upsert.
125
+ - **Safe partial updates** — if a field is missing from the source, DBTK excludes it from the
126
+ UPDATE rather than overwriting with NULL. No more "did I just wipe a column?" paranoia.
127
+ - **Batch loop elimination** — DataSurge handles chunking, committing, retries, and statistics.
128
+ No more hand-rolled batch loops.
129
+ - **Zero-config logging** — one line sets up timestamped log files with auto-cleanup and a global
130
+ error flag. No logging boilerplate scattered through your pipeline.
131
+ - **TableLookup shorthand** — define a lookup or validation in ~40 characters:
132
+ `'fn': 'validate:ranks:rank_code:preload'`. The `preload` hint pre-caches the table before
133
+ processing starts. What would otherwise be a custom class or 30-line function is a string.
134
+
135
+ The code that remains is shorter, clearer, and still has all the functionality and checks. You
136
+ finish the job and think *"that was satisfyingly elegant"* — not because corners were cut, but
137
+ because the tool was collaborating with you instead of making you fight it.
138
+
96
139
  **Speed and Memory** The primary objective of DBTK is to give data integrators an elegant toolkit to speed up your development.
97
140
  But DBTK's throughput and memory usage are very good. BulkSurge streaming from a polars and doing direct loads to PostgreSQL will
98
141
  process 1M rows in 3-4 seconds. But even with a standard Python csv reader and numerous column transforms, DataSurge is able to
@@ -1,7 +1,7 @@
1
1
  # DBTK - Data Benders Toolkit
2
2
 
3
3
  <div style="float: right; padding: 20px">
4
- <img src="/docs/assets/databender.png" height="240" align="right" />
4
+ <img src="https://raw.githubusercontent.com/scottrbailey/dbtk/main/docs/assets/databender.png" height="240" align="right" />
5
5
  </div>
6
6
 
7
7
  **Control and Manipulate the Flow of Data** - A lightweight Python toolkit for data integration, transformation, and movement between systems.
@@ -11,9 +11,9 @@ Extract data from various sources, transform it through powerful operations, and
11
11
  This library is designed by and for data integrators.
12
12
 
13
13
  **DBTK aims to be fast and memory-efficient at every turn.** But it was designed to boost your productivity first and foremost.
14
- You have dozens (possibly hundreds) of interfaces, impossible deadlines, and multiple projects all happening at once. Your
15
- environment has three or more different relational databases. You just want to get stuff done instead of writing the same
16
- boilerplate code over and over or stressing over differences on a database server you don't use very often.
14
+ You have dozens, possibly hundreds, of interfaces, impossible deadlines, and multiple projects all happening at once. Your
15
+ environment has different relational databases, each has its own quirks. You just want to get stuff done instead of writing the same
16
+ boilerplate code over and over.
17
17
 
18
18
  **Design philosophy:** Modern databases excel at aggregating and transforming data at scale. DBTK embraces
19
19
  this by focusing on what Python does well: flexible record-by-record transformations,
@@ -24,6 +24,47 @@ your database handle it. If you need dataframes and heavy analytics - reach for
24
24
  or polars. DBTK sits in between: getting your data where it needs to be, cleaned and
25
25
  validated along the way.
26
26
 
27
+ ## Why DBTK?
28
+
29
+ Most ETL tools fall into one of two traps:
30
+
31
+ - **Verbose boilerplate hell** — SQLAlchemy + pandas + custom loops, repeated across every
32
+ interface with minor variations, slowly accreting defensive code for every edge case
33
+ - **Opaque black box** — so much is hidden that debugging feels like reverse-engineering
34
+ someone else's code
35
+
36
+ DBTK threads the needle. It's declarative enough to eliminate repetition, but explicit enough
37
+ that you stay in control. When something breaks, you know exactly where to look.
38
+
39
+ The architecture is intentionally layered — use what you need, skip what you don't:
40
+
41
+ ```
42
+ Record → ergonomic row handling, memory-efficient at scale
43
+ Table → field mapping, transforms, validation, upserts
44
+ DataSurge → batched inserts with progress tracking and stats
45
+ BulkSurge → direct bulk loads (SQL*Loader, BCP, COPY) for maximum throughput
46
+ readers/writers → consistent API across every file format and compression type
47
+ ```
48
+
49
+ When developers convert existing jobs to DBTK, the result can be **half to a quarter the
50
+ original code**. That reduction comes from specific things DBTK just handles:
51
+
52
+ - **Key column validation** — DBTK throws a clear error if a key field is missing from the source.
53
+ No more writing null-guards before every upsert.
54
+ - **Safe partial updates** — if a field is missing from the source, DBTK excludes it from the
55
+ UPDATE rather than overwriting with NULL. No more "did I just wipe a column?" paranoia.
56
+ - **Batch loop elimination** — DataSurge handles chunking, committing, retries, and statistics.
57
+ No more hand-rolled batch loops.
58
+ - **Zero-config logging** — one line sets up timestamped log files with auto-cleanup and a global
59
+ error flag. No logging boilerplate scattered through your pipeline.
60
+ - **TableLookup shorthand** — define a lookup or validation in ~40 characters:
61
+ `'fn': 'validate:ranks:rank_code:preload'`. The `preload` hint pre-caches the table before
62
+ processing starts. What would otherwise be a custom class or 30-line function is a string.
63
+
64
+ The code that remains is shorter, clearer, and still has all the functionality and checks. You
65
+ finish the job and think *"that was satisfyingly elegant"* — not because corners were cut, but
66
+ because the tool was collaborating with you instead of making you fight it.
67
+
27
68
  **Speed and Memory** The primary objective of DBTK is to give data integrators an elegant toolkit to speed up your development.
28
69
  But DBTK's throughput and memory usage are very good. BulkSurge streaming from a polars and doing direct loads to PostgreSQL will
29
70
  process 1M rows in 3-4 seconds. But even with a standard Python csv reader and numerous column transforms, DataSurge is able to
@@ -36,9 +36,9 @@ from .database import Database
36
36
  from .config import connect, set_config_file
37
37
  from .cursors import Cursor
38
38
  from .logging_utils import setup_logging, cleanup_old_logs, errors_logged
39
- from . import etl
40
39
  from . import readers
41
40
  from . import writers
41
+ from . import etl
42
42
 
43
43
  # Simple, clean exports
44
44
  __all__ = [
@@ -114,7 +114,8 @@ def checkup():
114
114
 
115
115
  def main():
116
116
  parser = argparse.ArgumentParser(prog='dbtk', description='DBTK command-line utilities')
117
- subparsers = parser.add_subparsers(dest='command', required=True)
117
+ subparsers = parser.add_subparsers(dest='command')
118
+ subparsers.required = True # required= kwarg not supported in Python 3.6
118
119
 
119
120
  # checkup
120
121
  subparsers.add_parser('checkup', help='Check for dependencies and configuration issues')
@@ -2,7 +2,7 @@
2
2
  import logging
3
3
  import time
4
4
  from abc import ABC, abstractmethod
5
- from typing import Iterable, Generator, Optional
5
+ from typing import Iterable, Generator, Optional, Union
6
6
  import datetime as dt
7
7
  import tempfile
8
8
  from pathlib import Path
@@ -183,7 +183,7 @@ class BaseSurge(ABC):
183
183
  if batch:
184
184
  yield batch
185
185
 
186
- def _resolve_file_path(self, path_input: Optional[str | Path] = None, extension: str = '.csv') -> Path:
186
+ def _resolve_file_path(self, path_input: Optional[Union[str, Path]] = None, extension: str = '.csv') -> Path:
187
187
  """
188
188
  Resolve an output file path from user input.
189
189
 
@@ -464,7 +464,7 @@ class BulkSurge(BaseSurge):
464
464
  logger.debug(f'sqlldr userid={user}/<PASSWORD>@{db} control={ctl_path} log={log_path}')
465
465
 
466
466
  try:
467
- result = subprocess.run(cmd, capture_output=True, text=True)
467
+ result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
468
468
 
469
469
  if result.returncode == 0:
470
470
  logger.info("SQL*Loader completed successfully")
@@ -565,7 +565,7 @@ class BulkSurge(BaseSurge):
565
565
  else:
566
566
  cmd += ['-T',] # integrated auth
567
567
  # Run BCP
568
- result = subprocess.run(cmd, capture_output=True, text=True)
568
+ result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
569
569
  if result.returncode != 0:
570
570
  error_msg = f"bcp failed with exit code {result.returncode}"
571
571
  if result.stderr:
@@ -606,7 +606,8 @@ class ValidationCollector:
606
606
  if self.desc_field:
607
607
  return result.get(self.desc_field, "")
608
608
  for field in ("title", "description", "name", "label"):
609
- if val := result.get(field):
609
+ val = result.get(field)
610
+ if val:
610
611
  return val
611
612
  return str(result)
612
613
 
@@ -63,7 +63,7 @@ class Table:
63
63
  'user_id': {'default': lambda: conf_vars['user_id']},
64
64
  }
65
65
 
66
- * **fn** (callable | list[callable] | str, optional):
66
+ * **fn** (Union[callable, List[callable], str], optional):
67
67
  Transformation function(s) to apply to the source value.
68
68
  callable: applied directly; list: functions applied in order (pipeline);
69
69
  str: magic shorthand (e.g. 'int', 'maxlen:255', 'lookup:table:col:val').
@@ -93,7 +93,7 @@ def _check_usaddress():
93
93
  if not HAS_USADDRESS:
94
94
  raise ImportError(
95
95
  "Address functionality requires the usaddress library. "
96
- "Install with: pip install usaddress"
96
+ "Install with: pip install usaddress==0.5.10 (Python 3.6) or pip install usaddress (Python 3.7+)"
97
97
  )
98
98
 
99
99
 
@@ -76,6 +76,7 @@ def setup_logging(
76
76
  Example
77
77
  -------
78
78
  ::
79
+
79
80
  import dbtk
80
81
 
81
82
  # Simple - uses defaults from config
@@ -151,7 +151,6 @@ class Reader(ABC):
151
151
 
152
152
  Common Features
153
153
  ---------------
154
- * **Automatic header cleaning** - Standardize messy column names
155
154
  * **Row number tracking** - Automatic _row_num field for debugging
156
155
  * **Record skipping** - Skip header rows or bad data
157
156
  * **Record limiting** - Process only first N records
@@ -172,7 +171,7 @@ class Reader(ABC):
172
171
  Maximum number of rows to read. None (default) reads all rows.
173
172
  null_values : str, list, tuple, or set, optional
174
173
  Values to convert to None. Can be a single string or a collection of strings.
175
- Common examples: '\\N' (IMDB files), 'NULL', 'NA', '' (empty string)
174
+ Common examples: '\\N', 'NULL', 'NA', '' (empty string)
176
175
 
177
176
  Example
178
177
  -------
@@ -4,7 +4,7 @@
4
4
 
5
5
  import logging
6
6
  import os
7
- from typing import TextIO, List, Dict, Any, Optional, Iterator
7
+ from typing import TextIO, List, Dict, Any, Optional, Iterator, Type
8
8
 
9
9
  from .base import Reader
10
10
  from ..record import Record, FixedWidthRecord
@@ -271,7 +271,7 @@ class EDIReader(FixedReader):
271
271
  def _get_columns(self, type_code: str) -> List[FixedColumn]:
272
272
  return self.columns.get(type_code)
273
273
 
274
- def _get_factory(self, type_code: str) -> type[Record]:
274
+ def _get_factory(self, type_code: str) -> Type[Record]:
275
275
  if type_code not in self._type_factories:
276
276
  cols = self._get_columns(type_code)
277
277
  if cols is None:
@@ -8,7 +8,7 @@ import logging
8
8
  import sys
9
9
  from abc import ABC, abstractmethod
10
10
  from pathlib import Path
11
- from typing import Any, BinaryIO, Iterable, Iterator, List, Optional, TextIO, Union
11
+ from typing import Any, BinaryIO, Iterable, Iterator, List, Optional, TextIO, Tuple, Union
12
12
 
13
13
  from ..utils import RecordLike, to_string
14
14
 
@@ -170,7 +170,7 @@ class BaseWriter(ABC):
170
170
  raise ValueError(f"{self.__class__.__name__} requires an output file path")
171
171
  self.output_path = Path(file)
172
172
 
173
- def _open_file_handle(self, mode: str = "w") -> tuple[Union[TextIO, BinaryIO], bool]:
173
+ def _open_file_handle(self, mode: str = "w") -> Tuple[Union[TextIO, BinaryIO], bool]:
174
174
  """
175
175
  Open the output file/stream and return (handle, should_close).
176
176
 
@@ -181,7 +181,7 @@ class BaseWriter(ABC):
181
181
 
182
182
  Returns
183
183
  -------
184
- tuple[Union[TextIO, BinaryIO], bool]
184
+ Tuple[Union[TextIO, BinaryIO], bool]
185
185
  (file_handle, should_close_flag)
186
186
  """
187
187
  if self.file is None:
@@ -267,7 +267,7 @@ class BaseWriter(ABC):
267
267
 
268
268
  def _get_data_iterator(
269
269
  self, data: Iterable[RecordLike], columns: Optional[List[str]] = None
270
- ) -> tuple[Optional[Iterator], Optional[List[str]]]:
270
+ ) -> Tuple[Optional[Iterator], Optional[List[str]]]:
271
271
  """
272
272
  Get data iterator and column names.
273
273
 
@@ -280,7 +280,7 @@ class BaseWriter(ABC):
280
280
 
281
281
  Returns
282
282
  -------
283
- tuple[Optional[Iterator], Optional[List[str]]]
283
+ Tuple[Optional[Iterator], Optional[List[str]]]
284
284
  (iterator, column_names)
285
285
  """
286
286
  if not data:
@@ -7,7 +7,7 @@ import io
7
7
  import logging
8
8
  import re
9
9
  import sys
10
- from typing import Any, BinaryIO, List, Optional, TextIO, Union
10
+ from typing import Any, BinaryIO, List, Optional, TextIO, Tuple, Union
11
11
  from pathlib import Path
12
12
 
13
13
  try:
@@ -196,7 +196,7 @@ class XMLStreamer(BatchWriter):
196
196
  preserve_types=True, # We'll convert in _prepare_record_for_xml
197
197
  )
198
198
 
199
- def _open_file_handle(self, mode: str = 'wb') -> tuple[BinaryIO, bool]:
199
+ def _open_file_handle(self, mode: str = 'wb') -> Tuple[BinaryIO, bool]:
200
200
  """
201
201
  Override to use binary mode and validate binary streams.
202
202
 
@@ -207,7 +207,7 @@ class XMLStreamer(BatchWriter):
207
207
 
208
208
  Returns
209
209
  -------
210
- tuple[BinaryIO, bool]
210
+ Tuple[BinaryIO, bool]
211
211
  (file_handle, should_close_flag)
212
212
 
213
213
  Raises
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dbtk
3
- Version: 0.8.0
3
+ Version: 0.8.2
4
4
  Summary: Data Benders Toolkit - A lightweight toolkit for data integration, ELT and ETL
5
5
  Author: Scott Bailey <scottrbailey@gmail.com>
6
6
  License: MIT
@@ -39,7 +39,8 @@ Requires-Dist: lxml>=4.6.0; extra == "recommended"
39
39
  Requires-Dist: openpyxl>=3.0.0; extra == "recommended"
40
40
  Requires-Dist: phonenumbers>=8.12.0; extra == "recommended"
41
41
  Requires-Dist: python-dateutil>=2.8.0; extra == "recommended"
42
- Requires-Dist: usaddress>=0.5.11; extra == "recommended"
42
+ Requires-Dist: usaddress==0.5.10; python_version < "3.7" and extra == "recommended"
43
+ Requires-Dist: usaddress>=0.5.11; python_version >= "3.7" and extra == "recommended"
43
44
  Provides-Extra: postgresql
44
45
  Requires-Dist: psycopg2-binary>=2.9; extra == "postgresql"
45
46
  Requires-Dist: psycopg[binary]>=3.1; extra == "postgresql"
@@ -57,7 +58,8 @@ Requires-Dist: openpyxl>=3.0.0; extra == "all"
57
58
  Requires-Dist: xlrd>=1.2.0; extra == "all"
58
59
  Requires-Dist: phonenumbers>=8.12.0; extra == "all"
59
60
  Requires-Dist: python-dateutil>=2.8.0; extra == "all"
60
- Requires-Dist: usaddress>=0.5.11; extra == "all"
61
+ Requires-Dist: usaddress==0.5.10; python_version < "3.7" and extra == "all"
62
+ Requires-Dist: usaddress>=0.5.11; python_version >= "3.7" and extra == "all"
61
63
  Provides-Extra: dev
62
64
  Requires-Dist: pytest>=6.0; extra == "dev"
63
65
  Requires-Dist: pytest-cov>=2.10; extra == "dev"
@@ -70,7 +72,7 @@ Dynamic: license-file
70
72
  # DBTK - Data Benders Toolkit
71
73
 
72
74
  <div style="float: right; padding: 20px">
73
- <img src="/docs/assets/databender.png" height="240" align="right" />
75
+ <img src="https://raw.githubusercontent.com/scottrbailey/dbtk/main/docs/assets/databender.png" height="240" align="right" />
74
76
  </div>
75
77
 
76
78
  **Control and Manipulate the Flow of Data** - A lightweight Python toolkit for data integration, transformation, and movement between systems.
@@ -80,9 +82,9 @@ Extract data from various sources, transform it through powerful operations, and
80
82
  This library is designed by and for data integrators.
81
83
 
82
84
  **DBTK aims to be fast and memory-efficient at every turn.** But it was designed to boost your productivity first and foremost.
83
- You have dozens (possibly hundreds) of interfaces, impossible deadlines, and multiple projects all happening at once. Your
84
- environment has three or more different relational databases. You just want to get stuff done instead of writing the same
85
- boilerplate code over and over or stressing over differences on a database server you don't use very often.
85
+ You have dozens, possibly hundreds, of interfaces, impossible deadlines, and multiple projects all happening at once. Your
86
+ environment has different relational databases, each has its own quirks. You just want to get stuff done instead of writing the same
87
+ boilerplate code over and over.
86
88
 
87
89
  **Design philosophy:** Modern databases excel at aggregating and transforming data at scale. DBTK embraces
88
90
  this by focusing on what Python does well: flexible record-by-record transformations,
@@ -93,6 +95,47 @@ your database handle it. If you need dataframes and heavy analytics - reach for
93
95
  or polars. DBTK sits in between: getting your data where it needs to be, cleaned and
94
96
  validated along the way.
95
97
 
98
+ ## Why DBTK?
99
+
100
+ Most ETL tools fall into one of two traps:
101
+
102
+ - **Verbose boilerplate hell** — SQLAlchemy + pandas + custom loops, repeated across every
103
+ interface with minor variations, slowly accreting defensive code for every edge case
104
+ - **Opaque black box** — so much is hidden that debugging feels like reverse-engineering
105
+ someone else's code
106
+
107
+ DBTK threads the needle. It's declarative enough to eliminate repetition, but explicit enough
108
+ that you stay in control. When something breaks, you know exactly where to look.
109
+
110
+ The architecture is intentionally layered — use what you need, skip what you don't:
111
+
112
+ ```
113
+ Record → ergonomic row handling, memory-efficient at scale
114
+ Table → field mapping, transforms, validation, upserts
115
+ DataSurge → batched inserts with progress tracking and stats
116
+ BulkSurge → direct bulk loads (SQL*Loader, BCP, COPY) for maximum throughput
117
+ readers/writers → consistent API across every file format and compression type
118
+ ```
119
+
120
+ When developers convert existing jobs to DBTK, the result can be **half to a quarter the
121
+ original code**. That reduction comes from specific things DBTK just handles:
122
+
123
+ - **Key column validation** — DBTK throws a clear error if a key field is missing from the source.
124
+ No more writing null-guards before every upsert.
125
+ - **Safe partial updates** — if a field is missing from the source, DBTK excludes it from the
126
+ UPDATE rather than overwriting with NULL. No more "did I just wipe a column?" paranoia.
127
+ - **Batch loop elimination** — DataSurge handles chunking, committing, retries, and statistics.
128
+ No more hand-rolled batch loops.
129
+ - **Zero-config logging** — one line sets up timestamped log files with auto-cleanup and a global
130
+ error flag. No logging boilerplate scattered through your pipeline.
131
+ - **TableLookup shorthand** — define a lookup or validation in ~40 characters:
132
+ `'fn': 'validate:ranks:rank_code:preload'`. The `preload` hint pre-caches the table before
133
+ processing starts. What would otherwise be a custom class or 30-line function is a string.
134
+
135
+ The code that remains is shorter, clearer, and still has all the functionality and checks. You
136
+ finish the job and think *"that was satisfyingly elegant"* — not because corners were cut, but
137
+ because the tool was collaborating with you instead of making you fight it.
138
+
96
139
  **Speed and Memory** The primary objective of DBTK is to give data integrators an elegant toolkit to speed up your development.
97
140
  But DBTK's throughput and memory usage are very good. BulkSurge streaming from a polars and doing direct loads to PostgreSQL will
98
141
  process 1M rows in 3-4 seconds. But even with a standard Python csv reader and numerous column transforms, DataSurge is able to
@@ -11,6 +11,11 @@ openpyxl>=3.0.0
11
11
  xlrd>=1.2.0
12
12
  phonenumbers>=8.12.0
13
13
  python-dateutil>=2.8.0
14
+
15
+ [all:python_version < "3.7"]
16
+ usaddress==0.5.10
17
+
18
+ [all:python_version >= "3.7"]
14
19
  usaddress>=0.5.11
15
20
 
16
21
  [dev]
@@ -50,6 +55,11 @@ lxml>=4.6.0
50
55
  openpyxl>=3.0.0
51
56
  phonenumbers>=8.12.0
52
57
  python-dateutil>=2.8.0
58
+
59
+ [recommended:python_version < "3.7"]
60
+ usaddress==0.5.10
61
+
62
+ [recommended:python_version >= "3.7"]
53
63
  usaddress>=0.5.11
54
64
 
55
65
  [sqlserver]
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "dbtk"
7
- version = "0.8.0"
7
+ version = "0.8.2"
8
8
  description = "Data Benders Toolkit - A lightweight toolkit for data integration, ELT and ETL"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.6"
@@ -44,7 +44,8 @@ recommended = [
44
44
  "openpyxl>=3.0.0",
45
45
  "phonenumbers>=8.12.0",
46
46
  "python-dateutil>=2.8.0",
47
- "usaddress>=0.5.11"
47
+ "usaddress==0.5.10;python_version<'3.7'",
48
+ "usaddress>=0.5.11;python_version>='3.7'"
48
49
  ]
49
50
  postgresql = ["psycopg2-binary>=2.9", "psycopg[binary]>=3.1"]
50
51
  oracle = ["oracledb>=1.0;python_version>='3.7'", # For modern systems
@@ -59,7 +60,8 @@ all = [
59
60
  "xlrd>=1.2.0",
60
61
  "phonenumbers>=8.12.0",
61
62
  "python-dateutil>=2.8.0",
62
- "usaddress>=0.5.11"
63
+ "usaddress==0.5.10;python_version<'3.7'",
64
+ "usaddress>=0.5.11;python_version>='3.7'"
63
65
  ]
64
66
  dev = ["pytest>=6.0", "pytest-cov>=2.10"]
65
67
  docs = [
@@ -76,5 +78,9 @@ Homepage = "https://github.com/scottrbailey/dbtk"
76
78
  Documentation = "https://dbtk.readthedocs.io"
77
79
  Repository = "https://github.com/scottrbailey/dbtk.git"
78
80
 
81
+ [tool.setuptools.packages.find]
82
+ where = ["."]
83
+ include = ["dbtk*"]
84
+
79
85
  [tool.setuptools.package-data]
80
86
  dbtk = ["dbtk_sample.yml"]
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes