execsql2 2.4.0__tar.gz → 2.4.1__tar.gz

{execsql2-2.4.0 → execsql2-2.4.1}/.github/workflows/ci-cd.yml

@@ -51,7 +51,6 @@ jobs:
                   python -m pip install --upgrade pip
                   python -m pip install tox
                   python -m pip install ".[dev]"
-                  python -m pip install ".[formats]"
             - name: Install tkinter (Ubuntu)
               if: runner.os == 'Linux'
               run: sudo apt-get install -y python3-tk
@@ -67,10 +66,63 @@ jobs:
                   token: ${{ secrets.CODECOV_TOKEN }}
                   files: coverage.xml
 
+    integration-tests:
+        name: integration-tests-py${{ matrix.python-version }}
+        runs-on: ubuntu-latest
+        strategy:
+            fail-fast: false
+            matrix:
+                python-version:
+                    - "3.13"
+        permissions:
+            contents: read
+        services:
+            postgres:
+                image: postgres:16
+                env:
+                    POSTGRES_USER: execsql
+                    POSTGRES_PASSWORD: execsql
+                    POSTGRES_DB: execsql_test
+                ports:
+                    - 5432:5432
+                options: >-
+                    --health-cmd pg_isready
+                    --health-interval 10s
+                    --health-timeout 5s
+                    --health-retries 5
+            mysql:
+                image: mysql:8
+                env:
+                    MYSQL_ROOT_PASSWORD: root
+                    MYSQL_USER: execsql
+                    MYSQL_PASSWORD: execsql
+                    MYSQL_DATABASE: execsql_test
+                ports:
+                    - 3306:3306
+                options: >-
+                    --health-cmd "mysqladmin ping"
+                    --health-interval 10s
+                    --health-timeout 5s
+                    --health-retries 5
+        steps:
+            - name: Check out repository code
+              uses: actions/checkout@v4
+            - name: Setup Python ${{ matrix.python-version }}
+              uses: actions/setup-python@v5
+              with:
+                  python-version: ${{ matrix.python-version }}
+            - name: Install dependencies
+              run: |
+                  python -m pip install --upgrade pip
+                  python -m pip install ".[dev,postgres,mysql]"
+            - name: Run integration tests
+              run: |
+                  python -m pytest tests/integration/test_postgres.py tests/integration/test_mysql.py -v --override-ini="addopts="
+
     build:
         name: Build distribution 📦
         runs-on: ubuntu-latest
-        needs: [tests]
+        needs: [tests, integration-tests]
         if: startsWith(github.ref, 'refs/tags/v')
         permissions:
             contents: read

{execsql2-2.4.0 → execsql2-2.4.1}/CHANGELOG.md

@@ -13,6 +13,10 @@ ______________________________________________________________________
 
 ______________________________________________________________________
 
+## [2.4.1] - 2026-03-30
+
+______________________________________________________________________
+
 ## [2.4.0] - 2026-03-30
 
 ### Changed
@@ -29,11 +33,19 @@ ______________________________________________________________________
 
 - Python 3.14 support — added to CI matrix, tox environments, and PyPI classifiers.
 - `formats` extra included in `dev` dependencies so ODS/Excel/Jinja2 tests run without manual installation.
+- PostgreSQL integration tests (9 tests) — full lifecycle via `--dsn` connection strings.
+- MySQL/MariaDB integration tests (9 tests, 1 xfail for pre-existing import adapter bug).
+- `docker-compose.yml` for local PostgreSQL and MySQL test databases.
+- CI integration test job with GitHub Actions services (PostgreSQL 16, MySQL 8).
 - Roadmap items in `templates/README.md` for integrating execsql-compare and execsql-upsert documentation into the main docs site.
 
 ### Fixed
 
 - Fix odfpy import — `import of` corrected to `import odf as of` in `exporters/ods.py` and test skip guards. ODS export was broken since the modular refactor.
+- Pass `--dsn` password through to all database backends (MySQL, SQL Server, Oracle, Firebird, DSN). Previously only PostgreSQL received the password from connection strings.
+- Fix importer error reporting — `exception_info()` (returns tuple) replaced with `exception_desc()` (returns string) in 6 call sites across `importers/base.py`, `importers/csv.py`, and `importers/feather.py`. This caused `AttributeError: 'tuple' has no attribute 'replace'` on any import failure.
+- Map Python encoding names to MySQL charset names in `LOAD DATA LOCAL INFILE` (e.g., `utf-8` → `utf8mb4`). Previously caused `Unknown character set` errors on MySQL imports.
+- `--dsn` now overrides conf-file connection settings (server, database, user, port). Previously conf-file values took precedence, silently ignoring the DSN.
 
 ______________________________________________________________________
 

{execsql2-2.4.0 → execsql2-2.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: execsql2
-Version: 2.4.0
+Version: 2.4.1
 Summary: Runs a SQL script against a PostgreSQL, SQLite, MariaDB/MySQL, DuckDB, Firebird, MS-Access, MS-SQL-Server, or Oracle database, or an ODBC DSN. Provides metacommands to import and export data, copy data between databases, conditionally execute SQL and metacommands, and dynamically alter SQL and metacommands with substitution variables.
 Project-URL: Repository, https://github.com/geocoug/execsql
 Project-URL: Issues, https://github.com/geocoug/execsql/issues

{execsql2-2.4.0 → execsql2-2.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "execsql2"
-version = "2.4.0"
+version = "2.4.1"
 description = "Runs a SQL script against a PostgreSQL, SQLite, MariaDB/MySQL, DuckDB, Firebird, MS-Access, MS-SQL-Server, or Oracle database, or an ODBC DSN. Provides metacommands to import and export data, copy data between databases, conditionally execute SQL and metacommands, and dynamically alter SQL and metacommands with substitution variables."
 readme = { file = "README.md", content-type = "text/markdown" }
 license = { file = "LICENSE.txt" }
@@ -155,7 +155,7 @@ skip-magic-trailing-comma = false
 line-ending = "auto"
 
 [tool.bumpversion]
-current_version = "2.4.0"
+current_version = "2.4.1"
 commit = true
 commit_args = "--no-verify"
 tag = true

{execsql2-2.4.0 → execsql2-2.4.1}/src/execsql/cli/run.py

@@ -141,18 +141,19 @@ def _run(
             raise SystemExit(1)
         db_type = db_type or parsed_dsn["db_type"]
         conf.db_type = db_type
-        if parsed_dsn["server"] and not conf.server:
+        # DSN values override conf-file values — the CLI flag is explicit.
+        if parsed_dsn["server"]:
             conf.server = parsed_dsn["server"]
-        if parsed_dsn["db"] and not conf.db:
+        if parsed_dsn["db"]:
             conf.db = parsed_dsn["db"]
-        if parsed_dsn["db_file"] and not conf.db_file:
+        if parsed_dsn["db_file"]:
             conf.db_file = parsed_dsn["db_file"]
-        if parsed_dsn["user"] and not user:
+        if parsed_dsn["user"]:
             user = parsed_dsn["user"]
         if parsed_dsn["password"]:
             conf.db_password = parsed_dsn["password"]
             conf.passwd_prompt = False
-        if parsed_dsn["port"] and not port:
+        if parsed_dsn["port"]:
             port = parsed_dsn["port"]
 
     # Apply CLI options over config-file values
@@ -538,6 +539,7 @@ def _connect_initial_db(conf: ConfigData):
             pw_needed=conf.passwd_prompt,
             port=conf.port,
             encoding=conf.db_encoding,
+            password=getattr(conf, "db_password", None),
         )
     elif conf.db_type == "l":
         if conf.db_file is None:
@@ -553,6 +555,7 @@ def _connect_initial_db(conf: ConfigData):
             pw_needed=conf.passwd_prompt,
             port=conf.port,
             encoding=conf.db_encoding,
+            password=getattr(conf, "db_password", None),
         )
     elif conf.db_type == "k":
         if conf.db_file is None:
@@ -568,6 +571,7 @@ def _connect_initial_db(conf: ConfigData):
             pw_needed=conf.passwd_prompt,
             port=conf.port,
             encoding=conf.db_encoding,
+            password=getattr(conf, "db_password", None),
         )
     elif conf.db_type == "f":
         return db_Firebird(
@@ -577,6 +581,7 @@ def _connect_initial_db(conf: ConfigData):
             pw_needed=conf.passwd_prompt,
             port=conf.port,
             encoding=conf.db_encoding,
+            password=getattr(conf, "db_password", None),
         )
     elif conf.db_type == "d":
         return db_Dsn(
@@ -584,6 +589,7 @@ def _connect_initial_db(conf: ConfigData):
             user=conf.username,
             pw_needed=conf.passwd_prompt,
             encoding=conf.db_encoding,
+            password=getattr(conf, "db_password", None),
         )
     else:
         from execsql.utils.errors import fatal_error

{execsql2-2.4.0 → execsql2-2.4.1}/src/execsql/db/factory.py

@@ -93,9 +93,10 @@ def db_SqlServer(
     pw_needed: bool = True,
     port: int | None = None,
     encoding: str | None = None,
+    password: str | None = None,
 ) -> SqlServerDatabase:
     """Open a Microsoft SQL Server connection via pyodbc."""
-    return SqlServerDatabase(server_name, database_name, user, pw_needed, port, encoding)
+    return SqlServerDatabase(server_name, database_name, user, pw_needed, port, encoding, password=password)
 
 
 def db_MySQL(
@@ -105,9 +106,10 @@ def db_MySQL(
     pw_needed: bool = True,
     port: int | None = None,
     encoding: str | None = None,
+    password: str | None = None,
 ) -> MySQLDatabase:
     """Open a MySQL or MariaDB connection via pymysql."""
-    return MySQLDatabase(server_name, database_name, user, pw_needed, port, encoding)
+    return MySQLDatabase(server_name, database_name, user, pw_needed, port, encoding, password=password)
 
 
 def db_DuckDB(
@@ -136,9 +138,10 @@ def db_Oracle(
     pw_needed: bool = True,
     port: int | None = None,
     encoding: str | None = None,
+    password: str | None = None,
 ) -> OracleDatabase:
     """Open an Oracle database connection via cx_Oracle (python-oracledb)."""
-    return OracleDatabase(server_name, database_name, user, pw_needed, port, encoding)
+    return OracleDatabase(server_name, database_name, user, pw_needed, port, encoding, password=password)
 
 
 def db_Firebird(
@@ -148,9 +151,10 @@ def db_Firebird(
     pw_needed: bool = True,
     port: int | None = None,
     encoding: str | None = None,
+    password: str | None = None,
 ) -> FirebirdDatabase:
     """Open a Firebird database connection via the firebird-driver package."""
-    return FirebirdDatabase(server_name, database_name, user, pw_needed, port, encoding)
+    return FirebirdDatabase(server_name, database_name, user, pw_needed, port, encoding, password=password)
 
 
 def db_Dsn(
@@ -158,6 +162,7 @@ def db_Dsn(
     user: str | None = None,
     pw_needed: bool = True,
     encoding: str | None = None,
+    password: str | None = None,
 ) -> DsnDatabase:
     """Open a connection to any ODBC data source registered under *dsn_name*."""
-    return DsnDatabase(dsn_name=dsn_name, user_name=user, need_passwd=pw_needed, encoding=encoding)
+    return DsnDatabase(dsn_name=dsn_name, user_name=user, need_passwd=pw_needed, encoding=encoding, password=password)

{execsql2-2.4.0 → execsql2-2.4.1}/src/execsql/db/mysql.py

@@ -18,6 +18,18 @@ import execsql.state as _state
 
 __all__ = ["MySQLDatabase"]
 
+# Map Python encoding names to MySQL CHARACTER SET names for LOAD DATA INFILE.
+_PYTHON_TO_MYSQL_CHARSET: dict[str, str] = {
+    "utf-8": "utf8mb4",
+    "utf8": "utf8mb4",
+    "latin-1": "latin1",
+    "iso-8859-1": "latin1",
+    "iso8859-1": "latin1",
+    "ascii": "ascii",
+    "cp1252": "cp1252",
+    "windows-1252": "cp1252",
+}
+
 
 class MySQLDatabase(Database):
     """MySQL and MariaDB adapter using the pymysql package."""
@@ -190,7 +202,8 @@ class MySQLDatabase(Database):
         ):
             import_sql = f"load data local infile '{csv_file_obj.csvfname}' into table {sq_name}"
             if csv_file_obj.encoding:
-                import_sql = f"{import_sql} character set {csv_file_obj.encoding}"
+                charset = _PYTHON_TO_MYSQL_CHARSET.get(csv_file_obj.encoding.lower(), csv_file_obj.encoding)
+                import_sql = f"{import_sql} character set {charset}"
             if csv_file_obj.delimiter or csv_file_obj.quotechar:
                 import_sql = import_sql + " columns"
                 if csv_file_obj.delimiter:

{execsql2-2.4.0 → execsql2-2.4.1}/src/execsql/importers/base.py

@@ -28,7 +28,7 @@ def import_data_table(
     hdrs: list[str],
     data: list[Any],
 ) -> None:
-    from execsql.utils.errors import exception_info
+    from execsql.utils.errors import exception_desc
 
     conf = _state.conf
     if any(x is None or len(x.strip()) == 0 for x in hdrs):
@@ -91,7 +91,7 @@ def import_data_table(
             raise ErrInfo(
                 type="db",
                 command_text=sql,
-                exception_msg=exception_info(),
+                exception_msg=exception_desc(),
                 other_msg=f"Could not create new table ({tablename}) for IMPORT metacommand",
             ) from e
     table_cols = db.table_columns(tablename, schemaname)
@@ -111,4 +111,4 @@ def import_data_table(
     except ErrInfo:
         raise
     except Exception as e:
-        raise ErrInfo("db", "Call to populate_table when importing data", exception_msg=exception_info()) from e
+        raise ErrInfo("db", "Call to populate_table when importing data", exception_msg=exception_desc()) from e

{execsql2-2.4.0 → execsql2-2.4.1}/src/execsql/importers/csv.py

@@ -32,7 +32,7 @@ def importtable(
     encoding: str | None = None,
     junk_header_lines: int = 0,
 ) -> None:
-    from execsql.utils.errors import exception_info
+    from execsql.utils.errors import exception_desc
 
     conf = _state.conf
     if not Path(filename).is_file():
@@ -66,7 +66,7 @@ def importtable(
             raise ErrInfo(
                 type="db",
                 command_text=sql,
-                exception_msg=exception_info(),
+                exception_msg=exception_desc(),
                 other_msg=f"Could not create new table ({tablename}) for IMPORT metacommand",
             ) from e
     else:
@@ -91,7 +91,7 @@ def importtable(
         fq_tablename = db.schema_qualified_table_name(schemaname, tablename)
         raise ErrInfo(
             "exception",
-            exception_msg=exception_info(),
+            exception_msg=exception_desc(),
             other_msg=f"Can't import tabular file ({filename}) to table ({fq_tablename})",
         ) from e
     inf.close()
@@ -104,7 +104,7 @@ def importfile(
     columname: str,
     filename: str,
 ) -> None:
-    from execsql.utils.errors import exception_info
+    from execsql.utils.errors import exception_desc
 
     if schemaname is not None:
         if not db.table_exists(tablename, schemaname):
@@ -127,6 +127,6 @@ def importfile(
         fq_tablename = db.schema_qualified_table_name(schemaname, tablename)
         raise ErrInfo(
             "exception",
-            exception_msg=exception_info(),
+            exception_msg=exception_desc(),
             other_msg=f"Can't import file ({filename}) to table ({fq_tablename})",
         ) from e

{execsql2-2.4.0 → execsql2-2.4.1}/src/execsql/importers/feather.py

@@ -24,14 +24,14 @@ def import_feather(
     filename: str,
     is_new: Any,
 ) -> None:
-    from execsql.utils.errors import exception_info
+    from execsql.utils.errors import exception_desc
 
     try:
         import polars as pl
     except Exception as e:
         raise ErrInfo(
             "exception",
-            exception_msg=exception_info(),
+            exception_msg=exception_desc(),
             other_msg="The polars Python library must be installed to import data from the Feather format.",
         ) from e
     df = pl.read_ipc(filename)
@@ -47,14 +47,14 @@ def import_parquet(
     filename: str,
     is_new: Any,
 ) -> None:
-    from execsql.utils.errors import exception_info
+    from execsql.utils.errors import exception_desc
 
     try:
         import polars as pl
     except Exception as e:
         raise ErrInfo(
             "exception",
-            exception_msg=exception_info(),
+            exception_msg=exception_desc(),
             other_msg="The polars Python library must be installed to import data from the Parquet format.",
         ) from e
     df = pl.read_parquet(filename)

execsql2-2.4.1/tests/integration/conftest.py

@@ -0,0 +1,36 @@
+"""Shared helpers for integration tests.
+
+Each backend-specific test module provides its own ``_write_conf`` and
+verification helpers.  The helpers here are backend-agnostic.
+"""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+import textwrap
+
+
+def write_script(tmp_path, sql_text, name="test_script.sql"):
+    """Write a .sql script file into *tmp_path*."""
+    script = tmp_path / name
+    script.write_text(textwrap.dedent(sql_text))
+    return script
+
+
+def run_execsql(tmp_path, script_path, extra_args=None, timeout=30):
+    """Run execsql on the given script via subprocess.
+
+    Returns the completed process.  The working directory is set to *tmp_path*
+    so that execsql.conf is picked up automatically.
+    """
+    cmd = [sys.executable, "-m", "execsql", str(script_path)]
+    if extra_args:
+        cmd.extend(extra_args)
+    return subprocess.run(
+        cmd,
+        cwd=str(tmp_path),
+        capture_output=True,
+        text=True,
+        timeout=timeout,
+    )

execsql2-2.4.1/tests/integration/test_dsn.py

@@ -0,0 +1,169 @@
+"""Tests for --dsn / --connection-string CLI flag interaction with execsql.conf.
+
+Verifies that --dsn overrides conf-file connection settings, and that
+the two can coexist (DSN provides connection, conf provides encoding/options).
+Uses SQLite since it requires no external services.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+import subprocess
+import sys
+import textwrap
+
+from tests.integration.conftest import write_script
+
+
+def _run_execsql(tmp_path, script_path, dsn=None, extra_args=None, timeout=30):
+    """Run execsql with optional --dsn flag."""
+    cmd = [sys.executable, "-m", "execsql"]
+    if dsn:
+        cmd.extend(["--dsn", dsn])
+    cmd.append(str(script_path))
+    if extra_args:
+        cmd.extend(extra_args)
+    return subprocess.run(
+        cmd,
+        cwd=str(tmp_path),
+        capture_output=True,
+        text=True,
+        timeout=timeout,
+    )
+
+
+def _write_conf(tmp_path, content):
+    """Write an execsql.conf with arbitrary content."""
+    conf = tmp_path / "execsql.conf"
+    conf.write_text(textwrap.dedent(content))
+    return conf
+
+
+class TestDsnOverridesConf:
+    """--dsn flag should override connection settings from execsql.conf."""
+
+    def test_dsn_overrides_conf_db_file(self, tmp_path):
+        """DSN database path takes precedence over conf db_file."""
+        dsn_db = tmp_path / "dsn.db"
+        conf_db = tmp_path / "conf.db"
+
+        # Conf points to conf.db
+        _write_conf(
+            tmp_path,
+            f"""\
+            [connect]
+            db_type = l
+            db_file = {conf_db}
+            new_db = yes
+            password_prompt = no
+        """,
+        )
+
+        script = write_script(
+            tmp_path,
+            """\
+            CREATE TABLE t (id INTEGER);
+            INSERT INTO t VALUES (1);
+        """,
+        )
+
+        # DSN points to dsn.db — should win
+        result = _run_execsql(tmp_path, script, dsn=f"sqlite:///{dsn_db}")
+        assert result.returncode == 0, f"stderr: {result.stderr}"
+
+        # Data should be in dsn.db, not conf.db
+        assert dsn_db.exists(), "DSN database was not created"
+        conn = sqlite3.connect(str(dsn_db))
+        rows = conn.execute("SELECT id FROM t").fetchall()
+        conn.close()
+        assert rows == [(1,)]
+
+    def test_dsn_without_conf_file(self, tmp_path):
+        """DSN works even when no execsql.conf exists."""
+        # Use a relative path — cwd is tmp_path, so test.db lands there.
+        # Conf with new_db=yes is needed so SQLite creates the file.
+        _write_conf(
+            tmp_path,
+            """\
+            [connect]
+            new_db = yes
+        """,
+        )
+
+        script = write_script(
+            tmp_path,
+            """\
+            CREATE TABLE t (val TEXT);
+            INSERT INTO t VALUES ('hello');
+        """,
+        )
+
+        result = _run_execsql(tmp_path, script, dsn="sqlite:///test.db")
+        assert result.returncode == 0, f"stderr: {result.stderr}"
+
+        db_path = tmp_path / "test.db"
+        conn = sqlite3.connect(str(db_path))
+        rows = conn.execute("SELECT val FROM t").fetchall()
+        conn.close()
+        assert rows == [("hello",)]
+
+    def test_conf_encoding_used_with_dsn_connection(self, tmp_path):
+        """Conf file encoding settings are respected even when --dsn provides the connection."""
+        _write_conf(
+            tmp_path,
+            """\
+            [connect]
+            new_db = yes
+
+            [encoding]
+            script = utf-8
+            output = utf-8
+        """,
+        )
+
+        csv_path = tmp_path / "out.csv"
+        script = write_script(
+            tmp_path,
+            f"""\
+            CREATE TABLE t (id INTEGER, name TEXT);
+            INSERT INTO t VALUES (1, 'Alice');
+
+            -- !x! EXPORT QUERY << SELECT id, name FROM t; >> TO {csv_path} AS CSV
+        """,
+        )
+
+        result = _run_execsql(tmp_path, script, dsn="sqlite:///test.db")
+        assert result.returncode == 0, f"stderr: {result.stderr}"
+        assert csv_path.exists()
+
+    def test_dsn_db_type_overrides_conf_db_type(self, tmp_path):
+        """DSN scheme determines db_type even if conf specifies a different type."""
+        # Conf says PostgreSQL, but DSN says SQLite — DSN should win
+        _write_conf(
+            tmp_path,
+            """\
+            [connect]
+            db_type = p
+            server = nonexistent
+            database = fake
+            new_db = yes
+            password_prompt = no
+        """,
+        )
+
+        script = write_script(
+            tmp_path,
+            """\
+            CREATE TABLE t (id INTEGER);
+            INSERT INTO t VALUES (42);
+        """,
+        )
+
+        result = _run_execsql(tmp_path, script, dsn="sqlite:///test.db")
+        assert result.returncode == 0, f"stderr: {result.stderr}"
+
+        db_path = tmp_path / "test.db"
+        conn = sqlite3.connect(str(db_path))
+        rows = conn.execute("SELECT id FROM t").fetchall()
+        conn.close()
+        assert rows == [(42,)]