hdx-python-database 1.3.8__py3-none-any.whl → 1.4.0__py3-none-any.whl

hdx/database/_version.py

@@ -1,5 +1,6 @@
-# file generated by setuptools-scm
+# file generated by vcs-versioning
 # don't change, don't track in version control
+from __future__ import annotations
 
 __all__ = [
     "__version__",
@@ -10,25 +11,14 @@ __all__ = [
     "commit_id",
 ]
 
-TYPE_CHECKING = False
-if TYPE_CHECKING:
-    from typing import Tuple
-    from typing import Union
-
-    VERSION_TUPLE = Tuple[Union[int, str], ...]
-    COMMIT_ID = Union[str, None]
-else:
-    VERSION_TUPLE = object
-    COMMIT_ID = object
-
 version: str
 __version__: str
-__version_tuple__: VERSION_TUPLE
-version_tuple: VERSION_TUPLE
-commit_id: COMMIT_ID
-__commit_id__: COMMIT_ID
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
 
-__version__ = version = '1.3.8'
-__version_tuple__ = version_tuple = (1, 3, 8)
+__version__ = version = '1.4.0'
+__version_tuple__ = version_tuple = (1, 4, 0)
 
 __commit_id__ = commit_id = None

hdx/database/database.py

@@ -1,7 +1,7 @@
 """Database utilities"""
 
 import logging
-from typing import Any, Dict, List, Optional, Tuple, Type, Union
+from typing import Any
 
 from sqlalchemy import Engine, TableClause, create_engine, insert
 from sqlalchemy.exc import SQLAlchemyError
@@ -64,7 +64,8 @@ class Database:
         table_base (Optional[Type[DeclarativeBase]]): Override table base. Defaults to None.
         reflect (bool): Whether to reflect existing tables. Defaults to False.
         **kwargs: See below
-        pg_restore_file (str): Restore database from pg_restore file
+        pg_restore_file (Path | str): Restore database from pg_restore file
+        pg_timeout (int): Seconds to wait for PostgreSQL. Defaults to None (wait forever).
         recreate_schema (bool): Whether to recreate schema
         schema_name (str): Database schema name. Defaults to "public".
         prepare_fn (Callable[[], None]]): Function to call before Base.metadata.create_all.
@@ -73,23 +74,24 @@ class Database:
         ssh_username (str): SSH username
         ssh_password (str): SSH password
         ssh_private_key: Location of SSH private key (instead of ssh_password)
+        ssh_host_key (str): SSH server host key to verify against. Strongly recommended.
         For more advanced usage, see SSHTunnelForwarder documentation.
 
     """
 
     def __init__(
         self,
-        engine: Optional[Engine] = None,
-        db_uri: Optional[str] = None,
-        database: Optional[str] = None,
-        host: Optional[str] = None,
-        port: Union[int, str, None] = None,
-        username: Optional[str] = None,
-        password: Optional[str] = None,
+        engine: Engine | None = None,
+        db_uri: str | None = None,
+        database: str | None = None,
+        host: str | None = None,
+        port: int | str | None = None,
+        username: str | None = None,
+        password: str | None = None,
         dialect: str = "postgresql",
-        driver: Optional[str] = None,
+        driver: str | None = None,
         db_has_tz: bool = False,
-        table_base: Optional[Type[DeclarativeBase]] = None,
+        table_base: type[DeclarativeBase] | None = None,
         reflect: bool = False,
         **kwargs: Any,
     ) -> None:
@@ -98,10 +100,12 @@ class Database:
         schema_name = None
         if len(kwargs) == 0:
             pg_restore_file = None
+            pg_timeout = None
             recreate_schema = False
             prepare_fn = do_nothing_fn
         else:
             pg_restore_file = kwargs.pop("pg_restore_file", None)
+            pg_timeout = kwargs.pop("pg_timeout", None)
             recreate_schema = kwargs.pop("recreate_schema", False)
             schema_name = kwargs.pop("schema", "public")
             prepare_fn = kwargs.pop("prepare_fn", do_nothing_fn)
@@ -116,6 +120,10 @@ class Database:
                 raise
             ssh_host = kwargs["ssh_host"]
             del kwargs["ssh_host"]
+            if "ssh_host_key" not in kwargs:
+                logger.warning(
+                    "ssh_host_key not supplied: SSH server identity will not be verified."
+                )
             ssh_port = kwargs.get("ssh_port")
             if ssh_port is not None:
                 ssh_port = int(ssh_port)
@@ -147,7 +155,7 @@ class Database:
                     "No engine, database uri or connection parameters provided!"
                 )
         if not engine and dialect == "postgresql":
-            wait_for_postgresql(db_uri)
+            wait_for_postgresql(db_uri, pg_timeout)
         if pg_restore_file and db_uri:
             restore_from_pgfile(db_uri, pg_restore_file)
         if not table_base:
@@ -230,8 +238,8 @@ class Database:
 
     def batch_populate(
         self,
-        rows: List[Dict],
-        dbtable: Type[DeclarativeBase],
+        rows: list[dict],
+        dbtable: type[DeclarativeBase],
         batch_size: int = 1000,
     ) -> None:
         """Batch populate database table.
@@ -252,11 +260,11 @@ class Database:
 
     @staticmethod
     def create_session(
-        engine: Optional[Engine] = None,
-        db_uri: Optional[str] = None,
-        table_base: Type[DeclarativeBase] = NoTZBase,
+        engine: Engine | None = None,
+        db_uri: str | None = None,
+        table_base: type[DeclarativeBase] = NoTZBase,
         reflect: bool = False,
-    ) -> Tuple[Session, Any]:
+    ) -> tuple[Session, Any]:
         """Creates SQLAlchemy session given SQLAlchemy engine or database uri
         (one of which must be supplied). Tables must inherit from Base in
         hdx.utilities.database unless base is defined. If reflect is True,
@@ -332,7 +340,7 @@ class Database:
             return False
 
     @staticmethod
-    def prepare_view(view_params: Dict) -> TableClause:
+    def prepare_view(view_params: dict) -> TableClause:
         """Prepare SQLAlchemy view from dictionary with keys: name, metadata and
         selectable. Must be run before Base.metadata.create_all.
 
@@ -345,7 +353,7 @@ class Database:
         return view(**view_params)
 
     @classmethod
-    def prepare_views(cls, view_params_list: List[Dict]) -> List[TableClause]:
+    def prepare_views(cls, view_params_list: list[dict]) -> list[TableClause]:
         """Prepare SQLAlchemy views from a list of dictionaries with keys: name,
         metadata and selectable. Must be run before Base.metadata.create_all.
 

hdx/database/dburi.py

@@ -1,13 +1,13 @@
 """Connection URI utilities"""
 
-from typing import Any, Dict, Optional, Union
-from urllib.parse import urlsplit
+from typing import Any
+from urllib.parse import quote_plus, unquote, urlsplit
 
 
 def get_params_from_connection_uri(
     db_uri: str,
     include_driver: bool = True,
-) -> Dict[str, Any]:
+) -> dict[str, Any]:
     """Gets PostgreSQL database connection parameters from connection URI
 
     Args:
@@ -28,8 +28,8 @@ def get_params_from_connection_uri(
         "database": result.path[1:],
         "host": result.hostname,
         "port": result.port,
-        "username": result.username,
-        "password": result.password,
+        "username": unquote(result.username) if result.username else None,
+        "password": unquote(result.password) if result.password else None,
         "dialect": dialect,
     }
     if include_driver:
@@ -38,13 +38,13 @@ def get_params_from_connection_uri(
 
 
 def get_connection_uri(
-    database: Optional[str] = None,
-    host: Optional[str] = None,
-    port: Union[int, str, None] = None,
-    username: Optional[str] = None,
-    password: Optional[str] = None,
+    database: str | None = None,
+    host: str | None = None,
+    port: int | str | None = None,
+    username: str | None = None,
+    password: str | None = None,
     dialect: str = "postgresql",
-    driver: Optional[str] = None,
+    driver: str | None = None,
     include_driver: bool = True,
 ) -> str:
     """Gets connection uri from database connection parameters
@@ -72,9 +72,9 @@ def get_connection_uri(
             strings.append(f"+{driver}")
     strings.append("://")
     if username:
-        strings.append(username)
+        strings.append(quote_plus(username))
         if password:
-            strings.append(f":{password}@")
+            strings.append(f":{quote_plus(password)}@")
         else:
             strings.append("@")
     if host:

hdx/database/no_timezone.py

@@ -1,7 +1,6 @@
 """Utilities for database datetime columns without timezone"""
 
-from datetime import datetime, timezone
-from typing import Optional
+from datetime import UTC, datetime
 
 from sqlalchemy import DateTime, TypeDecorator
 from sqlalchemy.orm import DeclarativeBase, declared_attr
@@ -16,11 +15,11 @@ class ConversionNoTZ(TypeDecorator):
     impl = DateTime
     cache_ok = True
 
-    def process_bind_param(self, value: Optional[datetime], _):
+    def process_bind_param(self, value: datetime | None, _):
         if value is None:
             return value
         if value.tzinfo:
-            value = value.astimezone(timezone.utc)
+            value = value.astimezone(UTC)
 
         return value.replace(tzinfo=None)
 
@@ -28,9 +27,9 @@ class ConversionNoTZ(TypeDecorator):
         if value is None:
             return value
         if value.tzinfo is None:
-            return value.replace(tzinfo=timezone.utc)
+            return value.replace(tzinfo=UTC)
 
-        return value.astimezone(timezone.utc)
+        return value.astimezone(UTC)
 
 
 class Base(DeclarativeBase):

hdx/database/postgresql.py

@@ -4,6 +4,7 @@ import logging
 import subprocess
 import time
 from os import environ
+from pathlib import Path
 
 from .dburi import get_params_from_connection_uri, remove_driver_from_uri
 
@@ -25,17 +26,19 @@ class PostgresError(Exception):
     pass
 
 
-def wait_for_postgresql(db_uri: str) -> None:
+def wait_for_postgresql(db_uri: str, timeout: int | None = None) -> None:
     """Waits for PostgreSQL database to be up
 
     Args:
         db_uri (str): Connection URI
+        timeout (Optional[int]): Seconds to wait. Defaults to None (wait forever).
 
     Returns:
         None
     """
     connecting_string = "Checking for PostgreSQL..."
     db_uri_nd = remove_driver_from_uri(db_uri)
+    start = time.monotonic()
     while True:
         try:
             logger.info(connecting_string)
@@ -44,22 +47,25 @@ def wait_for_postgresql(db_uri: str) -> None:
             logger.info("PostgreSQL is running!")
             break
         except psycopg.OperationalError:
+            if timeout is not None and time.monotonic() - start > timeout:
+                raise PostgresError(f"PostgreSQL not available after {timeout}s")
             time.sleep(1)
 
 
-def restore_from_pgfile(db_uri: str, pg_restore_file: str) -> str:
+def restore_from_pgfile(db_uri: str, pg_restore_file: Path | str) -> str:
     """Restore database from a pg_restore file created by pg_backup using the
     pg_restore command.
 
     Args:
         db_uri (str): Connection URI
-        pg_restore_file (str): Path to the pg_restore database file
+        pg_restore_file (Path | str): Path to the pg_restore database file
 
     Returns:
         str: Output from the pg_restore command
     """
+    pg_restore_file = Path(pg_restore_file).absolute()
     db_params = get_params_from_connection_uri(db_uri)
-    if pg_restore_file[-3:] == ".xz":
+    if pg_restore_file.suffix == ".xz":
         decompress = subprocess.Popen(
             ("unxz", "-c", "-d", pg_restore_file), stdout=subprocess.PIPE
         ).stdout
@@ -81,7 +87,7 @@ def restore_from_pgfile(db_uri: str, pg_restore_file: str) -> str:
         subprocess_params.append(f"{value}")
 
     if not decompress:
-        subprocess_params.append(pg_restore_file)
+        subprocess_params.append(str(pg_restore_file))
     env = environ.copy()
     password = db_params.get("password")
     if password:

hdx/database/views.py

@@ -24,15 +24,13 @@ class DropView(DDLElement):
 
 @compiler.compiles(CreateView)
 def _create_view(element, compiler, **kw):
-    return "CREATE VIEW %s AS %s" % (
-        element.name,
-        compiler.sql_compiler.process(element.selectable, literal_binds=True),
-    )
+    name = compiler.preparer.quote(element.name)
+    return f"CREATE VIEW {name} AS {compiler.sql_compiler.process(element.selectable, literal_binds=True)}"
 
 
 @compiler.compiles(DropView)
 def _drop_view(element, compiler, **kw):
-    return "DROP VIEW %s" % (element.name)
+    return f"DROP VIEW {compiler.preparer.quote(element.name)}"
 
 
 def view_exists(ddl, target, connection, **kw):

{hdx_python_database-1.3.8.dist-info → hdx_python_database-1.4.0.dist-info}/METADATA

@@ -1,9 +1,9 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: hdx-python-database
-Version: 1.3.8
+Version: 1.4.0
 Summary: HDX Python database utilities
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-database
-Author-email: Michael Rans <rans@email.com>
+Author-email: Michael Rans <rans@un.org>
 License: MIT
 License-File: LICENSE
 Keywords: HDX,database,postgresql
@@ -16,29 +16,16 @@ Classifier: Operating System :: Microsoft :: Windows
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: Unix
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Requires-Dist: sqlalchemy>=2
-Provides-Extra: dev
-Requires-Dist: pre-commit; extra == 'dev'
 Provides-Extra: docs
 Requires-Dist: mkapi; extra == 'docs'
 Provides-Extra: postgresql
 Requires-Dist: psycopg[binary]; extra == 'postgresql'
 Provides-Extra: sshtunnel
-Requires-Dist: sshtunnel; extra == 'sshtunnel'
-Provides-Extra: test
-Requires-Dist: psycopg[binary]; extra == 'test'
-Requires-Dist: pytest; extra == 'test'
-Requires-Dist: pytest-cov; extra == 'test'
-Requires-Dist: sshtunnel; extra == 'test'
+Requires-Dist: cryptography>=48.0.1; extra == 'sshtunnel'
+Requires-Dist: sshtunnel>=0.5.1; extra == 'sshtunnel'
 Description-Content-Type: text/markdown
 
 [![Build Status](https://github.com/OCHA-DAP/hdx-python-database/actions/workflows/run-python-tests.yaml/badge.svg)](https://github.com/OCHA-DAP/hdx-python-database/actions/workflows/run-python-tests.yaml)
@@ -53,3 +40,78 @@ For more information, please read the [documentation](https://hdx-python-databas
 
 This library is part of the [Humanitarian Data Exchange](https://data.humdata.org/) (HDX) project. If you have
 humanitarian related data, please upload your datasets to HDX.
+
+# Development
+
+## Environment
+
+Development is currently done using Python 3.13. The environment can be created with:
+
+```shell
+    uv sync
+```
+
+This creates a .venv folder with the versions specified in the project's uv.lock file.
+
+### Pre-commit
+
+pre-commit will be installed when syncing uv. It is run every time you make a git
+commit if you call it like this:
+
+```shell
+    pre-commit install
+```
+
+With pre-commit, all code is formatted according to
+[ruff](https://docs.astral.sh/ruff/) guidelines.
+
+To check if your changes pass pre-commit without committing, run:
+
+```shell
+    pre-commit run --all-files
+```
+
+## Packages
+
+[uv](https://github.com/astral-sh/uv) is used for package management. If
+you've introduced a new package to the source code (i.e. anywhere in `src/`),
+please add it to the `project.dependencies` section of `pyproject.toml` with
+any known version constraints.
+
+To add packages required only for testing, add them to the
+`[dependency-groups]`.
+
+Any changes to the dependencies will be automatically reflected in
+`uv.lock` with `pre-commit`, but you can re-generate the files without committing by
+executing:
+
+```shell
+    uv lock --upgrade
+```
+
+## Project
+
+[uv](https://github.com/astral-sh/uv) is used for project management. The project can be
+built using:
+
+```shell
+    uv build
+```
+
+Linting and syntax checking can be run with:
+
+```shell
+    uv run ruff check
+```
+
+To run the tests and view coverage, execute:
+
+```shell
+    uv run pytest
+```
+
+## Documentation
+
+The documentation, including API documentation, is generated using ReadtheDocs and
+MkDocs with Material. As you change the source code, remember to update the
+documentation at `documentation/index.md`.

hdx_python_database-1.4.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+hdx/database/__init__.py,sha256=sG2OYV5LHskgiDkB4IeVeQyPHL22Ta1kuxf-xN92-5E,155
+hdx/database/_version.py,sha256=u2oUj6dCZIqQ271Pm6INHvXKeiux4BSapl6g0c_tllg,520
+hdx/database/database.py,sha256=SA3XwmWMg7VIrUaWWRXVLMxXtKV4F9gz-jK5CJEPI18,13320
+hdx/database/dburi.py,sha256=5PbvVe56MglpHxflAFeRY0TXli3XThZrDphPWmBp4ho,3055
+hdx/database/no_timezone.py,sha256=MG4Z2t7SwjPKDnwDD0CVftoIZC0hxpSg-C-1FyEy0R0,1067
+hdx/database/postgresql.py,sha256=JSLWQrAYps7Dxkb0En0kndr-9lNQhPpaY5UvxydTmb8,3321
+hdx/database/utils.py,sha256=nzChgDj2_Mq8GS6xWaNmqOgpAiwK9tGYV_skKbErEVA,401
+hdx/database/views.py,sha256=3ZQT5jS-q3w30zKypnyuy8-2BKwkyYC0It9om9hxmCs,1669
+hdx/database/with_timezone.py,sha256=GUUylmTwQHp22yedmCVKpVJzvjWgdS9lg9KeqciNJkY,235
+hdx_python_database-1.4.0.dist-info/METADATA,sha256=ezN2GRZ808XVFheZ6d7DFaWqsX-am5kQ8jLvbrVAegQ,3985
+hdx_python_database-1.4.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+hdx_python_database-1.4.0.dist-info/licenses/LICENSE,sha256=wc-4GpMn-ODs-U_bTe1YCiPVgvcjzrpYOx2wPuyAeII,1079
+hdx_python_database-1.4.0.dist-info/RECORD,,

{hdx_python_database-1.3.8.dist-info → hdx_python_database-1.4.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.32.4
 Root-Is-Purelib: true
 Tag: py3-none-any

hdx_python_database-1.3.8.dist-info/RECORD

@@ -1,13 +0,0 @@
-hdx/database/__init__.py,sha256=sG2OYV5LHskgiDkB4IeVeQyPHL22Ta1kuxf-xN92-5E,155
-hdx/database/_version.py,sha256=_hPMnsWQOeGw9lqV0fueN3y4Gw7vLcBaoT0Bio9pJWo,704
-hdx/database/database.py,sha256=pOmshEfhkvlVN-cWJ9LX8ZL99TyZnwPAHOC-v0nF1pM,12926
-hdx/database/dburi.py,sha256=rw8RuLpU85vxYl5-Zk1SdZRAz-tqre_ot5aEnkjhxAQ,2977
-hdx/database/no_timezone.py,sha256=Lw087G3qOf8H70o8Hq_VvOMZ3ANei-lIA_sUlpFU_Uo,1130
-hdx/database/postgresql.py,sha256=5xmGFbzjTPTr9clQw4rOD1ZQEzYuY25Ho2hhz-WdCC8,2923
-hdx/database/utils.py,sha256=nzChgDj2_Mq8GS6xWaNmqOgpAiwK9tGYV_skKbErEVA,401
-hdx/database/views.py,sha256=U4BN9tbjlr93qKkeNbZNo_PCIwGsng09mQ2xFqh5CH0,1636
-hdx/database/with_timezone.py,sha256=GUUylmTwQHp22yedmCVKpVJzvjWgdS9lg9KeqciNJkY,235
-hdx_python_database-1.3.8.dist-info/METADATA,sha256=fC99IRhtjoty5SpmkVDobyMMYS3SYhs68ftDUoxOrZM,2841
-hdx_python_database-1.3.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hdx_python_database-1.3.8.dist-info/licenses/LICENSE,sha256=wc-4GpMn-ODs-U_bTe1YCiPVgvcjzrpYOx2wPuyAeII,1079
-hdx_python_database-1.3.8.dist-info/RECORD,,