dbtk 0.8.2__tar.gz → 0.8.3__tar.gz

{dbtk-0.8.2 → dbtk-0.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dbtk
-Version: 0.8.2
+Version: 0.8.3
 Summary: Data Benders Toolkit - A lightweight toolkit for data integration, ELT and ETL
 Author: Scott Bailey <scottrbailey@gmail.com>
 License: MIT
@@ -43,7 +43,6 @@ Requires-Dist: usaddress==0.5.10; python_version < "3.7" and extra == "recommend
 Requires-Dist: usaddress>=0.5.11; python_version >= "3.7" and extra == "recommended"
 Provides-Extra: postgresql
 Requires-Dist: psycopg2-binary>=2.9; extra == "postgresql"
-Requires-Dist: psycopg[binary]>=3.1; extra == "postgresql"
 Provides-Extra: oracle
 Requires-Dist: oracledb>=1.0; python_version >= "3.7" and extra == "oracle"
 Requires-Dist: cx_Oracle>=8.0; python_version < "3.7" and extra == "oracle"
@@ -58,8 +57,13 @@ Requires-Dist: openpyxl>=3.0.0; extra == "all"
 Requires-Dist: xlrd>=1.2.0; extra == "all"
 Requires-Dist: phonenumbers>=8.12.0; extra == "all"
 Requires-Dist: python-dateutil>=2.8.0; extra == "all"
-Requires-Dist: usaddress==0.5.10; python_version < "3.7" and extra == "all"
 Requires-Dist: usaddress>=0.5.11; python_version >= "3.7" and extra == "all"
+Requires-Dist: usaddress==0.5.10; python_version < "3.7" and extra == "all"
+Requires-Dist: oracledb>=1.0; python_version >= "3.7" and extra == "all"
+Requires-Dist: cx_Oracle>=8.0; python_version < "3.7" and extra == "all"
+Requires-Dist: oracledb>=1.0; python_version >= "3.7" and extra == "all"
+Requires-Dist: psycopg2-binary>=2.9; extra == "all"
+Requires-Dist: pymysql>=1.0; extra == "all"
 Provides-Extra: dev
 Requires-Dist: pytest>=6.0; extra == "dev"
 Requires-Dist: pytest-cov>=2.10; extra == "dev"
@@ -110,13 +114,17 @@ that you stay in control. When something breaks, you know exactly where to look.
 The architecture is intentionally layered — use what you need, skip what you don't:
 
 ```
+Configuration   → encrypted YAML, env vars, named connections, driver overrides, smart logging
+Connection      → consistent connection and parameter handling, clean reference hierarchy
 Record          → ergonomic row handling, memory-efficient at scale
 Table           → field mapping, transforms, validation, upserts
 DataSurge       → batched inserts with progress tracking and stats
 BulkSurge       → direct bulk loads (SQL*Loader, BCP, COPY) for maximum throughput
-readers/writers → consistent API across every file format and compression type
+Readers/Writers → consistent API across every file format and compression type
 ```
 
+<img src="https://raw.githubusercontent.com/scottrbailey/dbtk/main/docs/assets/dbtk_flowchart.png" width="800" align="center" />
+
 When developers convert existing jobs to DBTK, the result can be **half to a quarter the
 original code**. That reduction comes from specific things DBTK just handles:
 
@@ -137,7 +145,7 @@ finish the job and think *"that was satisfyingly elegant"* — not because corne
 because the tool was collaborating with you instead of making you fight it.
 
 **Speed and Memory** The primary objective of DBTK is to give data integrators an elegant toolkit to speed up your development.
-But DBTK's throughput and memory usage are very good. BulkSurge streaming from a polars and doing direct loads to PostgreSQL will
+But DBTK's throughput and memory usage are very good. BulkSurge streaming from polars and doing direct loads to PostgreSQL will
 process 1M rows in 3-4 seconds. But even with a standard Python csv reader and numerous column transforms, DataSurge is able to
 write 1M rows to every supported database in 5-10 seconds.
 

{dbtk-0.8.2 → dbtk-0.8.3}/README.md

@@ -39,13 +39,17 @@ that you stay in control. When something breaks, you know exactly where to look.
 The architecture is intentionally layered — use what you need, skip what you don't:
 
 ```
+Configuration   → encrypted YAML, env vars, named connections, driver overrides, smart logging
+Connection      → consistent connection and parameter handling, clean reference hierarchy
 Record          → ergonomic row handling, memory-efficient at scale
 Table           → field mapping, transforms, validation, upserts
 DataSurge       → batched inserts with progress tracking and stats
 BulkSurge       → direct bulk loads (SQL*Loader, BCP, COPY) for maximum throughput
-readers/writers → consistent API across every file format and compression type
+Readers/Writers → consistent API across every file format and compression type
 ```
 
+<img src="https://raw.githubusercontent.com/scottrbailey/dbtk/main/docs/assets/dbtk_flowchart.png" width="800" align="center" />
+
 When developers convert existing jobs to DBTK, the result can be **half to a quarter the
 original code**. That reduction comes from specific things DBTK just handles:
 
@@ -66,7 +70,7 @@ finish the job and think *"that was satisfyingly elegant"* — not because corne
 because the tool was collaborating with you instead of making you fight it.
 
 **Speed and Memory** The primary objective of DBTK is to give data integrators an elegant toolkit to speed up your development.
-But DBTK's throughput and memory usage are very good. BulkSurge streaming from a polars and doing direct loads to PostgreSQL will
+But DBTK's throughput and memory usage are very good. BulkSurge streaming from polars and doing direct loads to PostgreSQL will
 process 1M rows in 3-4 seconds. But even with a standard Python csv reader and numerous column transforms, DataSurge is able to
 write 1M rows to every supported database in 5-10 seconds.
 

{dbtk-0.8.2 → dbtk-0.8.3}/dbtk/__init__.py

@@ -29,13 +29,15 @@ Direct connections:
     cursor = db.cursor()  # Returns Record objects
 """
 
-__version__ = '0.8.0'
+__version__ = '0.8.3'
 __author__ = 'Scott Bailey <scottrbailey@gmail.com>'
 
 from .database import Database
 from .config import connect, set_config_file
-from .cursors import Cursor
+from .cursors import Cursor, PreparedStatement
 from .logging_utils import setup_logging, cleanup_old_logs, errors_logged
+from .record import fixed_record_factory, FixedWidthRecord
+from .utils import FixedColumn
 from . import readers
 from . import writers
 from . import etl
@@ -45,6 +47,10 @@ __all__ = [
     'connect',
     'Database',
     'Cursor',
+    'PreparedStatement',
+    'FixedColumn',
+    'FixedWidthRecord',
+    'fixed_record_factory',
     'etl',
     'readers',
     'writers',

{dbtk-0.8.2 → dbtk-0.8.3}/dbtk/cursors.py

@@ -4,6 +4,7 @@ Cursor classes that wrap database cursors and provide different return types.
 All cursors delegate to the underlying database cursor stored in _cursor.
 """
 
+import inspect
 import logging
 from pathlib import Path
 from typing import List, Any, Optional, Iterator, Callable, Union
@@ -16,6 +17,21 @@ logger = logging.getLogger(__name__)
 __all__ = ['Cursor', 'PreparedStatement']
 
 
+def _resolve_sql_path(filename: Union[str, Path], caller_file: str) -> Path:
+    """Resolve a SQL filename to an existing path.
+
+    Checks the path as given first (absolute or relative to CWD), then falls
+    back to the directory containing *caller_file* so that scripts can refer to
+    sibling SQL files by bare name.
+    """
+    path = Path(filename)
+    if not path.is_absolute() and not path.exists():
+        candidate = Path(caller_file).parent / path
+        if candidate.exists():
+            return candidate
+    return path
+
+
 class PreparedStatement:
     """
     A prepared SQL statement loaded from a file with cached parameter mapping.
@@ -94,11 +110,12 @@ class PreparedStatement:
 
 class Cursor:
     """
-    Basic cursor that returns query results as lists.
+    Cursor that returns query results as Records.
 
-    This is the base class for all DBTK cursor types. It wraps database-specific cursor
-    objects and provides a consistent interface plus additional functionality like SQL
-    file execution, parameter conversion, and prepared statements.
+    It wraps database-specific cursor objects and provides a consistent interface plus additional
+    functionality like SQL file execution, parameter conversion, and prepared statements.
+    It also maintains a clean reference hierarchy to the connection (cursor.connection) and
+    to the driver (cursor.connection.driver)
 
     Cursor returns Record objects, which provide flexible access via dictionary keys,
     attributes, or integer indices.
@@ -123,8 +140,8 @@ class Cursor:
     -------
     ::
 
-        # Usually created via Database.cursor()
-        cursor = db.cursor('list')
+        db = dbtk.connect('prod_ods')
+        cursor = db.cursor()
         cursor.execute("SELECT id, name, email FROM users WHERE status = :status",
                       {'status': 'active'})
 
@@ -404,9 +421,40 @@ class Cursor:
                 self._create_record_factory()
         return True
 
-    def execute(self, query: str, bind_vars: tuple = ()) -> None:
-        """Execute a database query."""
+    def execute(self, query: str,
+                bind_vars: Union[tuple, dict] = (),
+                convert_params: bool = False) -> None:
+        """
+        Execute a database query.
+
+        Pass convert_params=True to have the query rewritten to the cursor's paramstyle
+        and parameters handled automatically (same as PreparedStatement and execute_file).
+
+        Parameters
+        ----------
+        bind_vars : tuple or dict, default ()
+            Bind parameters to pass to the database.
+
+            When convert_params is False (the default), passed directly to the
+            underlying cursor and must already be in the format required by the
+            cursor's paramstyle.
+
+            When convert_params is True, must be a dict (or Record).
+
+        convert_params : bool, default False
+            If True, parameter order will be extracted from the query and the query
+            will be rewritten to match the cursor's paramstyle. Missing parameters
+            will be defaulted to None and extra parameters will be ignored.
+        """
+
         self._row_factory_invalid = True
+        if convert_params:
+            if not hasattr(bind_vars, 'items'):
+                if bind_vars:
+                    raise ValueError(f'bind_vars must be a dict when convert_params=True')
+                bind_vars = {}
+            query, param_names = process_sql_parameters(query, self.paramstyle)
+            bind_vars = self.prepare_params(param_names, bind_vars)
 
         if self.debug:
             logger.debug(f'Query:\n{query}')
@@ -433,7 +481,8 @@ class Cursor:
         executed multiple times, use prepare_file() instead for better performance.
 
         Args:
-            filename: Path to SQL file (relative to CWD)
+            filename: Path to SQL file. Resolved relative to CWD first; if not
+                found there, falls back to the directory of the calling script.
             bind_vars: Dictionary of named parameters
             **kwargs:
                 encoding: File encoding (default: utf-8-sig)
@@ -445,10 +494,11 @@ class Cursor:
             cursor.execute_file('queries/get_user.sql', {'user_id': 123})
         """
         encoding = kwargs.get('encoding', 'utf-8-sig')
+        path = _resolve_sql_path(filename, inspect.stack()[1].filename)
 
         try:
             # Read SQL from file
-            with open(filename, encoding=encoding) as f:
+            with open(path, encoding=encoding) as f:
                 sql = f.read()
 
             # Transform SQL for this cursor's paramstyle
@@ -466,7 +516,7 @@ class Cursor:
         except Exception as e:
             statement = locals().get('transformed_sql', 'N/A')
             logger.error(
-                f"Error executing SQL file: {filename}\n"
+                f"Error executing SQL file: {path}\n"
                 f"Transformed SQL: {statement}\n"
                 f"Parameters: {bind_vars}"
             )
@@ -480,7 +530,8 @@ class Cursor:
         The returned PreparedStatement can be executed multiple times efficiently.
 
         Args:
-            filename: Path to SQL file (relative to CWD)
+            filename: Path to SQL file. Resolved relative to CWD first; if not
+                found there, falls back to the directory of the calling script.
             encoding: File encoding (default: utf-8-sig)
 
         Returns:
@@ -494,7 +545,32 @@ class Cursor:
             for user in users:
                 stmt.execute({'user_id': user.id, 'name': user.name})
         """
-        return PreparedStatement(self, filename=filename, encoding=encoding)
+        path = _resolve_sql_path(filename, inspect.stack()[1].filename)
+        return PreparedStatement(self, filename=path, encoding=encoding)
+
+    def prepare_query(self, query: str) -> PreparedStatement:
+        """
+        Prepare a SQL statement from a query string for repeated execution.
+
+        The parameter conversion is performed once.
+        The returned PreparedStatement can be executed multiple times efficiently.
+
+        Args:
+            query:
+
+        Returns:
+            PreparedStatement object
+
+        Example
+        -------
+        ::
+
+            stmt = cursor.prepare_query('SELECT * FROM users WHERE user_id = :user_id')
+            for user in users:
+                stmt.execute({'user_id': user.id})
+        """
+        return PreparedStatement(self, query=query)
+
 
     def executemany(self, query: str, bind_vars: List[tuple]) -> None:
         """Execute a query against multiple parameter sets."""

{dbtk-0.8.2 → dbtk-0.8.3}/dbtk/dbtk_sample.yml

@@ -4,7 +4,6 @@
 settings:
   default_batch_size: 1000
   default_country: US
-  default_header_clean: 2  # Clean.LOWER_NOSPACE
   default_timezone: UTC
 
   # Logging configuration for integration scripts

{dbtk-0.8.2 → dbtk-0.8.3}/dbtk/defaults.py

@@ -5,7 +5,6 @@ settings = {
     'default_batch_size': 1000,
     'default_country': 'US',
     'default_db_type': 'postgres',
-    'default_header_clean': 2,  # Clean.LOWER_NOSPACE as int
     'data_dump_dir': '/tmp', # default directory for data dumps
     'compressed_file_buffer_size': 1024 * 1024,  # 1MB buffer for .gz/.bz2/.xz files
     'null_string': '',       # how null is represented in text outputs

{dbtk-0.8.2 → dbtk-0.8.3}/dbtk/readers/__init__.py

@@ -8,7 +8,7 @@ consistent interface and automatic format detection.
 
 """
 
-from .base import Reader, Clean
+from .base import Reader
 from .utils import get_reader, open_file
 from .csv import CSVReader
 from .data_frame import DataFrameReader
@@ -20,7 +20,7 @@ from .xml import XMLReader, XMLColumn
 
 # Re-export everything
 __all__ = [
-    'Reader', 'Clean', 'get_reader', 'open_file',
+    'Reader', 'get_reader', 'open_file',
     'CSVReader', 'EDIReader', 'FixedColumn', 'FixedReader', 'JSONReader', 'NDJSONReader', 'XLSReader', 'XLSXReader',
     'XMLColumn', 'XMLReader', 'DataFrameReader', 'open_workbook', 'get_sheet_by_index'
 ]

{dbtk-0.8.2 → dbtk-0.8.3}/dbtk/readers/base.py

@@ -21,75 +21,6 @@ from ..defaults import settings
 logger = logging.getLogger(__name__)
 
 
-class Clean:
-    """Header cleaning levels for column names."""
-    NOOP = 0  # Leave unchanged
-    LOWER = 1  # Lower case header
-    LOWER_NOSPACE = 2  # Lower case and replace spaces with _
-    LOWER_ALPHANUM = 3  # Lower case, remove all non-alphanum characters
-    STANDARDIZE = 4  # Lower case, remove non-alphanum, strip "code" endings
-    DEFAULT = LOWER_NOSPACE
-
-    @classmethod
-    def from_string(cls, value):
-        """Convert string to Clean constant, or pass through if already int."""
-        if isinstance(value, int):
-            return value
-
-        string_map = {
-            'noop': cls.NOOP,
-            'lower': cls.LOWER,
-            'lower_nospace': cls.LOWER_NOSPACE,
-            'lower_alphanum': cls.LOWER_ALPHANUM,
-            'standardize': cls.STANDARDIZE
-        }
-        return string_map.get(value.lower(), cls.NOOP)
-
-    @staticmethod
-    def normalize(val: Any, clean_level: Union[int, 'Clean', None] = None) -> str:
-        """
-        Normalize column header names.
-
-        Args:
-            val: Column name (any type, will be converted to string)
-            clean_level: Cleaning level, either an integer (0-4), a Clean class constant,
-                        or None to use Clean.DEFAULT
-
-        Returns:
-            Normalized column name string
-
-        Examples:
-            Clean.normalize("#Term Code", Clean.NOOP)           # -> "#Term Code"
-            Clean.normalize("#Term Code", Clean.LOWER)          # -> "#term code"
-            Clean.normalize("#Term Code", Clean.LOWER_NOSPACE)  # -> "#term_code"
-            Clean.normalize("#Term Code", Clean.LOWER_ALPHANUM) # -> "termcode"
-            Clean.normalize("#Term Code", Clean.STANDARDIZE)    # -> "term"
-            Clean.normalize("#Term Code")                       # -> "#term_code" (default to Clean.LOWER_NOSPACE)
-        """
-        # Use Clean.DEFAULT if clean_level is None
-        if clean_level is None:
-            clean_level = Clean.LOWER_NOSPACE
-        # Convert clean_level to integer if it's a Clean constant
-        clean_level = clean_level if isinstance(clean_level, int) else clean_level.value
-        # Validate clean_level
-        if clean_level not in {Clean.NOOP, Clean.LOWER, Clean.LOWER_NOSPACE, Clean.LOWER_ALPHANUM, Clean.STANDARDIZE}:
-            raise ValueError(f"Invalid clean_level: {clean_level}. Must be 0-4 or a Clean constant.")
-
-        if val in (None, '') or clean_level == Clean.NOOP:
-            return str(val) if val is not None else ''
-        val = str(val).lower().strip()
-        if clean_level == Clean.LOWER:
-            return val
-        elif clean_level == Clean.LOWER_NOSPACE:
-            return val.replace(' ', '_')
-        elif clean_level == Clean.LOWER_ALPHANUM:
-            return re.sub(r'[^a-z0-9]', '', val)
-        elif clean_level == Clean.STANDARDIZE:
-            return re.sub(r'[^a-z0-9]+|code$', '', val)
-        else:
-            return val
-
-
 class _Progress:
     __slots__ = ('tell', 'byte_total', 'row_total')
 
@@ -215,7 +146,6 @@ class Reader(ABC):
     XLSXReader : Read Excel .xlsx files
     XMLReader : Read XML files
     FixedReader : Read fixed-width text files
-    Clean : Header cleaning options
     Record : Flexible row objects with multiple access patterns
 
     Notes

{dbtk-0.8.2 → dbtk-0.8.3}/dbtk/readers/csv.py

@@ -5,7 +5,7 @@
 import csv
 import os
 from typing import TextIO, List, Any, Iterator, Optional
-from .base import Reader, Clean, logger
+from .base import Reader, logger
 
 
 class CSVReader(Reader):
@@ -84,7 +84,6 @@ class CSVReader(Reader):
     --------
     Reader : Base class with common reader features
     readers.get_reader : Automatic reader selection based on file extension
-    Clean : Header cleaning options
 
     Notes
     -----

{dbtk-0.8.2 → dbtk-0.8.3}/dbtk/readers/data_frame.py

@@ -1,7 +1,7 @@
 # dbtk/readers/data_frame.py
 import itertools
 from typing import Iterable, Optional
-from .base import Reader, Record, Clean, logger
+from .base import Reader, Record, logger
 
 
 class DataFrameReader(Reader):

{dbtk-0.8.2 → dbtk-0.8.3}/dbtk/readers/excel.py

@@ -6,7 +6,7 @@ import datetime as dt
 import logging
 from pathlib import Path
 from typing import List, Any, Iterator, Optional, Union
-from .base import Reader, Clean
+from .base import Reader
 
 logger = logging.getLogger(__name__)
 

{dbtk-0.8.2 → dbtk-0.8.3}/dbtk/readers/json.py

@@ -5,7 +5,7 @@
 import json
 import os
 from typing import List, Any, Dict, Optional, TextIO, Iterator
-from .base import Reader, Clean
+from .base import Reader
 
 
 class JSONReader(Reader):

{dbtk-0.8.2 → dbtk-0.8.3}/dbtk/readers/utils.py

@@ -252,7 +252,6 @@ def get_reader(filename: Union[str, Path],
             for record in reader:
                 print(record)
 
-        # Fixed width file (uses Clean.NOOP by default)
         config = [
             FixedColumn('name', 1, 20),
             FixedColumn('age', 21, 23, 'int')

{dbtk-0.8.2 → dbtk-0.8.3}/dbtk/readers/xml.py

@@ -6,14 +6,32 @@ import logging
 import os
 from pathlib import Path
 from typing import List, Any, Dict, Optional, TextIO, Union, Iterator
-from .base import Reader, Clean
+from .base import Reader
 
 try:
     from lxml import etree
     HAS_LXML = True
 except ImportError:
+    from xml.etree import ElementTree as etree
     HAS_LXML = False
 
+# lxml and stdlib use different exception types
+_XMLParseError = etree.XMLSyntaxError if HAS_LXML else etree.ParseError
+_XPathError    = etree.XPathEvalError  if HAS_LXML else SyntaxError
+
+
+def _xpath(node, expr):
+    """Run an XPath expression using lxml or stdlib ElementTree."""
+    if HAS_LXML:
+        return node.xpath(expr)
+    # stdlib findall supports a limited XPath subset; // must be relative
+    if expr.startswith('//'):
+        expr = '.' + expr
+    try:
+        return node.findall(expr)
+    except SyntaxError:
+        return []
+
 logger = logging.getLogger(__name__)
 
 
@@ -102,13 +120,13 @@ class XMLReader(Reader):
         """Parse the XML file and prepare for reading."""
         try:
             self._tree = etree.parse(self.fp)
-        except etree.XMLSyntaxError as e:
+        except _XMLParseError as e:
             raise ValueError(f"Invalid XML: {e}")
 
         # Find all record nodes
         try:
-            self._record_nodes = self._tree.xpath(self.record_xpath)
-        except etree.XPathEvalError as e:
+            self._record_nodes = _xpath(self._tree, self.record_xpath)
+        except _XPathError as e:
             raise ValueError(f"Invalid XPath expression '{self.record_xpath}': {e}")
 
         if not self._record_nodes:
@@ -159,7 +177,7 @@ class XMLReader(Reader):
         # Check if it has custom XPath
         if xml_column.xpath:
             try:
-                result = record_node.xpath(xml_column.xpath)
+                result = _xpath(record_node, xml_column.xpath)
                 if result:
                     # Handle different XPath result types
                     if isinstance(result[0], str):
@@ -175,7 +193,7 @@ class XMLReader(Reader):
                     return value.strip() if value else None
                 else:
                     return None
-            except etree.XPathEvalError:
+            except _XPathError:
                 return None
 
         # Look for simple child element by column name
@@ -249,12 +267,9 @@ def open_xml(filename: Union[str, Path], **kwargs) -> XMLReader:
             for record in reader:
                 print(record.name)
     """
-    fp = open(filename, 'rb')  # lxml prefers binary mode
+    fp = open(filename, 'rb')
     return XMLReader(fp, **kwargs)
 
 
-def check_dependencies():
-    if not HAS_LXML:
-        logger.error('lxml is required for XML support. Install with "pip install lxml".')
-
-check_dependencies()
+if not HAS_LXML:
+    logger.debug('lxml not available; falling back to stdlib xml.etree.ElementTree (limited XPath support).')