hdx-python-database 1.3.9__tar.gz → 1.4.0__tar.gz

{hdx_python_database-1.3.9 → hdx_python_database-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: hdx-python-database
-Version: 1.3.9
+Version: 1.4.0
 Summary: HDX Python database utilities
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-database
 Author-email: Michael Rans <rans@un.org>
@@ -17,14 +17,15 @@ Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: Unix
 Classifier: Programming Language :: Python
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Requires-Dist: sqlalchemy>=2
 Provides-Extra: docs
 Requires-Dist: mkapi; extra == 'docs'
 Provides-Extra: postgresql
 Requires-Dist: psycopg[binary]; extra == 'postgresql'
 Provides-Extra: sshtunnel
-Requires-Dist: sshtunnel; extra == 'sshtunnel'
+Requires-Dist: cryptography>=48.0.1; extra == 'sshtunnel'
+Requires-Dist: sshtunnel>=0.5.1; extra == 'sshtunnel'
 Description-Content-Type: text/markdown
 
 [![Build Status](https://github.com/OCHA-DAP/hdx-python-database/actions/workflows/run-python-tests.yaml/badge.svg)](https://github.com/OCHA-DAP/hdx-python-database/actions/workflows/run-python-tests.yaml)

{hdx_python_database-1.3.9 → hdx_python_database-1.4.0}/documentation/index.md

@@ -79,20 +79,26 @@ tunnel (which requires installing `hdx-python-database[sshtunnel]`):
 
     # Get SQLAlchemy session object given database parameters and
     # if needed SSH parameters. If database is PostgreSQL, will poll
-    # till it is up.
+    # till it is up (or until pg_timeout seconds if supplied).
     from hdx.database import Database
     with Database(database="db", host="1.2.3.4", username="user",
                   password="pass", dialect="dialect", driver="driver",
 
                   ssh_host="5.6.7.8", ssh_port=2222, ssh_username="sshuser",
-                  ssh_private_key="path_to_key", db_has_tz=True,
-                  reflect=True, prepare_fn=prepare_views) as database:
+                  ssh_private_key="path_to_key", ssh_host_key="AAAAC3...",
+                  db_has_tz=True, reflect=True, prepare_fn=prepare_views,
+                  pg_timeout=60) as database:
         engine = database.get_engine()
         session = database.get_session()
         reflected_classes = database.get_reflected_classes()
         views = database.get_prepare_results()
         session.query(...)
 
+`ssh_host_key` is the SSH server's public host key, used to verify the server's
+identity. Without it, the tunnel is open to man-in-the-middle attacks and a
+warning is logged. It can be obtained with `ssh-keyscan -t ed25519 5.6.7.8`
+(the base64 field of the output).
+
 `db_has_tz` which defaults to `False` indicates whether database datetime
 columns have timezones. If `db_has_tz` is `True`, use `Base` from
 `hdx.database.with_timezone`, otherwise use `Base` from

{hdx_python_database-1.3.9 → hdx_python_database-1.4.0}/pyproject.toml

@@ -26,20 +26,20 @@ classifiers = [
 ]
 readme = "README.md"
 dynamic = ["version"]
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 
 dependencies = [
     "sqlalchemy>=2",
 ]
 
 [project.optional-dependencies]
-sshtunnel = ["sshtunnel"]
+sshtunnel = ["sshtunnel>=0.5.1", "cryptography>=48.0.1"]
 postgresql = ["psycopg[binary]"]
 docs = ["mkapi"]
 
 [dependency-groups]
 dev = [
-    "sshtunnel",
+    "sshtunnel>=0.5.1",
     "psycopg[binary]",
     "pytest",
     "pytest-cov",
@@ -75,7 +75,7 @@ allow-direct-references = true
 # ----------------------------------------------------------------------------
 
 [tool.ruff]
-target-version = "py310"
+target-version = "py311"
 src = ["src"]
 exclude = ["_version.py"]
 

{hdx_python_database-1.3.9 → hdx_python_database-1.4.0}/src/hdx/database/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.3.9'
-__version_tuple__ = version_tuple = (1, 3, 9)
+__version__ = version = '1.4.0'
+__version_tuple__ = version_tuple = (1, 4, 0)
 
 __commit_id__ = commit_id = None

{hdx_python_database-1.3.9 → hdx_python_database-1.4.0}/src/hdx/database/database.py

@@ -65,6 +65,7 @@ class Database:
         reflect (bool): Whether to reflect existing tables. Defaults to False.
         **kwargs: See below
         pg_restore_file (Path | str): Restore database from pg_restore file
+        pg_timeout (int): Seconds to wait for PostgreSQL. Defaults to None (wait forever).
         recreate_schema (bool): Whether to recreate schema
         schema_name (str): Database schema name. Defaults to "public".
         prepare_fn (Callable[[], None]]): Function to call before Base.metadata.create_all.
@@ -73,6 +74,7 @@ class Database:
         ssh_username (str): SSH username
         ssh_password (str): SSH password
         ssh_private_key: Location of SSH private key (instead of ssh_password)
+        ssh_host_key (str): SSH server host key to verify against. Strongly recommended.
         For more advanced usage, see SSHTunnelForwarder documentation.
 
     """
@@ -98,10 +100,12 @@ class Database:
         schema_name = None
         if len(kwargs) == 0:
             pg_restore_file = None
+            pg_timeout = None
             recreate_schema = False
             prepare_fn = do_nothing_fn
         else:
             pg_restore_file = kwargs.pop("pg_restore_file", None)
+            pg_timeout = kwargs.pop("pg_timeout", None)
             recreate_schema = kwargs.pop("recreate_schema", False)
             schema_name = kwargs.pop("schema", "public")
             prepare_fn = kwargs.pop("prepare_fn", do_nothing_fn)
@@ -116,6 +120,10 @@ class Database:
                 raise
             ssh_host = kwargs["ssh_host"]
             del kwargs["ssh_host"]
+            if "ssh_host_key" not in kwargs:
+                logger.warning(
+                    "ssh_host_key not supplied: SSH server identity will not be verified."
+                )
             ssh_port = kwargs.get("ssh_port")
             if ssh_port is not None:
                 ssh_port = int(ssh_port)
@@ -147,7 +155,7 @@ class Database:
                     "No engine, database uri or connection parameters provided!"
                 )
         if not engine and dialect == "postgresql":
-            wait_for_postgresql(db_uri)
+            wait_for_postgresql(db_uri, pg_timeout)
         if pg_restore_file and db_uri:
             restore_from_pgfile(db_uri, pg_restore_file)
         if not table_base:

{hdx_python_database-1.3.9 → hdx_python_database-1.4.0}/src/hdx/database/dburi.py

@@ -1,7 +1,7 @@
 """Connection URI utilities"""
 
 from typing import Any
-from urllib.parse import urlsplit
+from urllib.parse import quote_plus, unquote, urlsplit
 
 
 def get_params_from_connection_uri(
@@ -28,8 +28,8 @@ def get_params_from_connection_uri(
         "database": result.path[1:],
         "host": result.hostname,
         "port": result.port,
-        "username": result.username,
-        "password": result.password,
+        "username": unquote(result.username) if result.username else None,
+        "password": unquote(result.password) if result.password else None,
         "dialect": dialect,
     }
     if include_driver:
@@ -72,9 +72,9 @@ def get_connection_uri(
             strings.append(f"+{driver}")
     strings.append("://")
     if username:
-        strings.append(username)
+        strings.append(quote_plus(username))
         if password:
-            strings.append(f":{password}@")
+            strings.append(f":{quote_plus(password)}@")
         else:
             strings.append("@")
     if host:

{hdx_python_database-1.3.9 → hdx_python_database-1.4.0}/src/hdx/database/no_timezone.py

@@ -1,6 +1,6 @@
 """Utilities for database datetime columns without timezone"""
 
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 
 from sqlalchemy import DateTime, TypeDecorator
 from sqlalchemy.orm import DeclarativeBase, declared_attr
@@ -19,7 +19,7 @@ class ConversionNoTZ(TypeDecorator):
         if value is None:
             return value
         if value.tzinfo:
-            value = value.astimezone(timezone.utc)
+            value = value.astimezone(UTC)
 
         return value.replace(tzinfo=None)
 
@@ -27,9 +27,9 @@ class ConversionNoTZ(TypeDecorator):
         if value is None:
             return value
         if value.tzinfo is None:
-            return value.replace(tzinfo=timezone.utc)
+            return value.replace(tzinfo=UTC)
 
-        return value.astimezone(timezone.utc)
+        return value.astimezone(UTC)
 
 
 class Base(DeclarativeBase):

{hdx_python_database-1.3.9 → hdx_python_database-1.4.0}/src/hdx/database/postgresql.py

@@ -26,17 +26,19 @@ class PostgresError(Exception):
     pass
 
 
-def wait_for_postgresql(db_uri: str) -> None:
+def wait_for_postgresql(db_uri: str, timeout: int | None = None) -> None:
     """Waits for PostgreSQL database to be up
 
     Args:
         db_uri (str): Connection URI
+        timeout (Optional[int]): Seconds to wait. Defaults to None (wait forever).
 
     Returns:
         None
     """
     connecting_string = "Checking for PostgreSQL..."
     db_uri_nd = remove_driver_from_uri(db_uri)
+    start = time.monotonic()
     while True:
         try:
             logger.info(connecting_string)
@@ -45,6 +47,8 @@ def wait_for_postgresql(db_uri: str) -> None:
             logger.info("PostgreSQL is running!")
             break
         except psycopg.OperationalError:
+            if timeout is not None and time.monotonic() - start > timeout:
+                raise PostgresError(f"PostgreSQL not available after {timeout}s")
             time.sleep(1)
 
 
@@ -59,7 +63,7 @@ def restore_from_pgfile(db_uri: str, pg_restore_file: Path | str) -> str:
     Returns:
         str: Output from the pg_restore command
     """
-    pg_restore_file = Path(pg_restore_file)
+    pg_restore_file = Path(pg_restore_file).absolute()
     db_params = get_params_from_connection_uri(db_uri)
     if pg_restore_file.suffix == ".xz":
         decompress = subprocess.Popen(

{hdx_python_database-1.3.9 → hdx_python_database-1.4.0}/src/hdx/database/views.py

@@ -24,12 +24,13 @@ class DropView(DDLElement):
 
 @compiler.compiles(CreateView)
 def _create_view(element, compiler, **kw):
-    return f"CREATE VIEW {element.name} AS {compiler.sql_compiler.process(element.selectable, literal_binds=True)}"
+    name = compiler.preparer.quote(element.name)
+    return f"CREATE VIEW {name} AS {compiler.sql_compiler.process(element.selectable, literal_binds=True)}"
 
 
 @compiler.compiles(DropView)
 def _drop_view(element, compiler, **kw):
-    return f"DROP VIEW {element.name}"
+    return f"DROP VIEW {compiler.preparer.quote(element.name)}"
 
 
 def view_exists(ddl, target, connection, **kw):

{hdx_python_database-1.3.9 → hdx_python_database-1.4.0}/tests/hdx/database/test_database.py

@@ -1,6 +1,6 @@
 """Database Utility Tests"""
 
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from os import remove
 from os.path import exists, join
 from tempfile import gettempdir
@@ -32,7 +32,7 @@ class TestDatabase:
             dbsession = dbdatabase.get_session()
             assert str(dbsession.bind.engine.url) == f"sqlite:///{dbpath}"
             assert dbdatabase._base == NoTZBase
-            now = datetime(2022, 10, 20, 22, 35, 55, tzinfo=timezone.utc)
+            now = datetime(2022, 10, 20, 22, 35, 55, tzinfo=UTC)
             input_dbtestdate = DBTestDate(test_date=now)
             dbsession.add(input_dbtestdate)
             dbsession.commit()
@@ -42,7 +42,7 @@ class TestDatabase:
             dbtestdate = dbsession.execute(select(date_view)).scalar_one()
             assert dbtestdate == now
 
-            now = datetime(2023, 10, 20, 22, 35, 55, tzinfo=timezone.utc)
+            now = datetime(2023, 10, 20, 22, 35, 55, tzinfo=UTC)
             rows = [{"test_date": now}]
             dbdatabase.batch_populate(rows, DBTestDate)
             dbtestdate = dbsession.execute(select(DBTestDate)).all()[1][0].test_date

{hdx_python_database-1.3.9 → hdx_python_database-1.4.0}/tests/hdx/database/test_dburi.py

@@ -82,6 +82,19 @@ class TestDBURI:
         result = get_connection_uri(**TestDBURI.params_sq_driver_none)
         assert result == TestDBURI.connection_uri_sq
 
+    def test_special_characters(self):
+        params = {
+            **TestDBURI.params_pg,
+            "username": "my@user",
+            "password": "p@ss/w:rd#?",
+        }
+        result = get_connection_uri(**params)
+        assert (
+            result
+            == "postgresql+psycopg://my%40user:p%40ss%2Fw%3Ard%23%3F@myserver:1234/mydatabase"
+        )
+        assert get_params_from_connection_uri(result) == params
+
     def test_remove_driver_from_uri(self):
         db_uri_nd = remove_driver_from_uri(TestDBURI.connection_uri_pg)
         assert db_uri_nd == TestDBURI.connection_uri_pg_no_driver

{hdx_python_database-1.3.9 → hdx_python_database-1.4.0}/tests/hdx/database/test_postgresql.py

@@ -1,5 +1,8 @@
 """PostgreSQL Utility Tests"""
 
+import time
+
+import psycopg
 import pytest
 
 from . import PsycopgConnection
@@ -20,6 +23,15 @@ class TestPostgreSQL:
         wait_for_postgresql(db_uri)
         assert PsycopgConnection.connected is True
 
+    def test_wait_for_postgresql_timeout(self, db_uri, monkeypatch):
+        def connect(*args, **kwargs):
+            raise psycopg.OperationalError
+
+        monkeypatch.setattr(psycopg, "connect", connect)
+        monkeypatch.setattr(time, "sleep", lambda _: None)
+        with pytest.raises(PostgresError):
+            wait_for_postgresql(db_uri, timeout=0)
+
     def test_restore_from_pgfile(self, db_uri, mock_subprocess, mock_engine):
         pg_restore_file = "test.pg_restore"
         assert restore_from_pgfile(db_uri, pg_restore_file) == "WORKED!"

{hdx_python_database-1.3.9 → hdx_python_database-1.4.0}/tests/hdx/database/test_ssh.py

@@ -20,6 +20,7 @@ class TestSSH:
 
     def test_get_session_ssh(
         self,
+        caplog,
         mock_psycopg,
         mock_SSHTunnelForwarder,
     ):
@@ -60,3 +61,11 @@ class TestSSH:
                 == "postgresql+psycopg://myuser@0.0.0.0:12345/mydatabase"
             )
             assert dbdatabase._base == NoTZBase
+        assert "ssh_host_key not supplied" in caplog.text
+
+        caplog.clear()
+        with Database(
+            ssh_host="mysshhost", ssh_host_key="AAAAkey", **params
+        ) as dbdatabase:
+            assert dbdatabase._base == NoTZBase
+        assert "ssh_host_key not supplied" not in caplog.text

hdx_python_database-1.4.0/tests/hdx/database/test_views.py

@@ -0,0 +1,23 @@
+"""View Tests"""
+
+from sqlalchemy import column, select, table
+from sqlalchemy.dialects import postgresql, sqlite
+
+from hdx.database.views import CreateView, DropView
+
+
+class TestViews:
+    selectable = select(table("t", column("a")))
+
+    def test_view_name_quoted(self):
+        for dialect in (sqlite.dialect(), postgresql.dialect()):
+            sql = str(CreateView("date_view", self.selectable).compile(dialect=dialect))
+            assert sql.startswith("CREATE VIEW date_view AS SELECT")
+            sql = str(DropView("date_view").compile(dialect=dialect))
+            assert sql == "DROP VIEW date_view"
+
+            name = 'x"; DROP TABLE t; --'
+            sql = str(CreateView(name, self.selectable).compile(dialect=dialect))
+            assert sql.startswith('CREATE VIEW "x""; DROP TABLE t; --" AS SELECT')
+            sql = str(DropView(name).compile(dialect=dialect))
+            assert sql == 'DROP VIEW "x""; DROP TABLE t; --"'