hdx-python-database 1.3.4__tar.gz → 1.3.6__tar.gz

{hdx_python_database-1.3.4 → hdx_python_database-1.3.6}/.config/pre-commit-config.yaml

@@ -2,13 +2,13 @@ default_language_version:
     python: python3.12
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.9
+    rev: v0.9.10
     hooks:
       # Run the linter.
       - id: ruff
@@ -17,7 +17,7 @@ repos:
       - id: ruff-format
         args: [--config, .config/ruff.toml]
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.4.18
+    rev: 0.6.5
     hooks:
       # Run the pip compile
       - id: pip-compile

{hdx_python_database-1.3.4 → hdx_python_database-1.3.6}/.github/workflows/publish.yaml

@@ -8,6 +8,13 @@ jobs:
   publish:
     runs-on: ubuntu-latest
 
+    environment:
+      name: pypi
+      url: https://pypi.org/p/hdx-python-database
+
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
     steps:
     - uses: actions/checkout@v4
     - name: Get history and tags for versioning to work
@@ -26,9 +33,5 @@ jobs:
     - name: Build with hatch
       run: |
         hatch build
-    - name: Publish with hatch
-      env:
-        HATCH_INDEX_USER: ${{secrets.HATCH_INDEX_USER}}
-        HATCH_INDEX_AUTH: ${{secrets.HATCH_INDEX_AUTH}}
-      run: |
-        hatch publish
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

{hdx_python_database-1.3.4 → hdx_python_database-1.3.6}/.github/workflows/run-python-tests.yaml

@@ -17,6 +17,10 @@ jobs:
   build:
     runs-on: ubuntu-latest
 
+    permissions:
+      checks: write
+      pull-requests: write
+
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python

{hdx_python_database-1.3.4 → hdx_python_database-1.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: hdx-python-database
-Version: 1.3.4
+Version: 1.3.6
 Summary: HDX Python database utilities
 Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-database
 Author-email: Michael Rans <rans@email.com>
@@ -28,6 +28,8 @@ Requires-Python: >=3.8
 Requires-Dist: sqlalchemy>=2
 Provides-Extra: dev
 Requires-Dist: pre-commit; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: mkapi; extra == 'docs'
 Provides-Extra: postgresql
 Requires-Dist: psycopg[binary]; extra == 'postgresql'
 Provides-Extra: sshtunnel

hdx_python_database-1.3.6/documentation/.readthedocs.yaml

@@ -0,0 +1,14 @@
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-24.04
+  tools:
+    python: "3.12"
+  jobs:
+    pre_build:
+      - pip install .[docs]
+
+mkdocs:
+  configuration: documentation/mkdocs.yaml

hdx_python_database-1.3.4/documentation/main.md → hdx_python_database-1.3.6/documentation/index.md

@@ -61,6 +61,7 @@ Versions from 1.0.6 no longer support Python 2.7.
 A fresh schema can be created using the function `recreate_schema`. Any
 existing schema with that name will be dropped eg.
 
+    from hdx.database import Database
     Database.recreate_schema(engine, db_uri)
 
 Your SQLAlchemy database tables must inherit from `Base` in
@@ -117,6 +118,16 @@ If a prepare function is supplied in prepare_fn, it will be executed before
 Base.metadata.create_all and the results of it returned in instance variable
 prepare_results.
 
+Batch population of rows (list of dictionaries) can be achieved with the
+batch_populate method of Database, for example:
+
+        now = datetime(2023, 10, 20, 22, 35, 55, tzinfo=timezone.utc)
+        rows = [{"test_date": now}]
+        dbdatabase.batch_populate(rows, DBTestDate)
+        dbtestdate = (
+            dbsession.execute(select(DBTestDate)).all()[1][0].test_date
+        )
+
 
 ## Connection URI
 

hdx_python_database-1.3.6/documentation/mkdocs.yaml

@@ -0,0 +1,16 @@
+site_name: HDX Python Database
+repo_url: https://github.com/OCHA-DAP/hdx-python-database/
+repo_name: OCHA-DAP/hdx-python-database
+docs_dir: .
+site_dir: ../site
+theme:
+  name: material
+  highlightjs: true
+plugins:
+  - search
+  - mkapi
+nav:
+  - Home: index.md
+  - API Documentation:
+    - Database: $src/hdx.database.*
+    - PostgreSQL specific: $src/hdx.database.postgresql.*

{hdx_python_database-1.3.4 → hdx_python_database-1.3.6}/pyproject.toml

@@ -50,6 +50,7 @@ sshtunnel = ["sshtunnel"]
 postgresql = ["psycopg[binary]"]
 test = ["pytest", "pytest-cov"]
 dev = ["pre-commit"]
+docs = ["mkapi"]
 
 
 #########
@@ -88,6 +89,9 @@ run = """
        --cov-report=lcov --cov-report=term-missing
        """
 
+[tool.hatch.envs.hatch-static-analysis]
+dependencies = ["ruff==0.9.10"]
+
 [tool.hatch.envs.hatch-static-analysis.scripts]
 format-check = ["ruff format --config .config/ruff.toml --check --diff {args:.}",]
 format-fix = ["ruff format --config .config/ruff.toml {args:.}",]

hdx_python_database-1.3.6/requirements.txt

@@ -0,0 +1,148 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
+astdoc==1.1.5
+    # via mkapi
+babel==2.17.0
+    # via mkdocs-material
+backrefs==5.8
+    # via mkdocs-material
+bcrypt==4.3.0
+    # via paramiko
+certifi==2025.1.31
+    # via requests
+cffi==1.17.1
+    # via
+    #   cryptography
+    #   pynacl
+cfgv==3.4.0
+    # via pre-commit
+charset-normalizer==3.4.1
+    # via requests
+click==8.1.8
+    # via mkdocs
+colorama==0.4.6
+    # via mkdocs-material
+coverage==7.7.0
+    # via pytest-cov
+cryptography==44.0.2
+    # via paramiko
+distlib==0.3.9
+    # via virtualenv
+filelock==3.18.0
+    # via virtualenv
+ghp-import==2.1.0
+    # via mkdocs
+greenlet==3.1.1
+    # via sqlalchemy
+identify==2.6.9
+    # via pre-commit
+idna==3.10
+    # via requests
+iniconfig==2.0.0
+    # via pytest
+jinja2==3.1.6
+    # via
+    #   mkapi
+    #   mkdocs
+    #   mkdocs-material
+markdown==3.7
+    # via
+    #   astdoc
+    #   mkdocs
+    #   mkdocs-material
+    #   pymdown-extensions
+markdown-it-py==3.0.0
+    # via rich
+markupsafe==3.0.2
+    # via
+    #   jinja2
+    #   mkdocs
+mdurl==0.1.2
+    # via markdown-it-py
+mergedeep==1.3.4
+    # via
+    #   mkdocs
+    #   mkdocs-get-deps
+mkapi==4.1.3
+    # via hdx-python-database (pyproject.toml)
+mkdocs==1.6.1
+    # via
+    #   mkapi
+    #   mkdocs-material
+mkdocs-get-deps==0.2.0
+    # via mkdocs
+mkdocs-material==9.6.8
+    # via mkapi
+mkdocs-material-extensions==1.3.1
+    # via mkdocs-material
+nodeenv==1.9.1
+    # via pre-commit
+packaging==24.2
+    # via
+    #   mkdocs
+    #   pytest
+paginate==0.5.7
+    # via mkdocs-material
+paramiko==3.5.1
+    # via sshtunnel
+pathspec==0.12.1
+    # via mkdocs
+platformdirs==4.3.6
+    # via
+    #   mkdocs-get-deps
+    #   virtualenv
+pluggy==1.5.0
+    # via pytest
+pre-commit==4.1.0
+    # via hdx-python-database (pyproject.toml)
+psycopg==3.2.6
+    # via hdx-python-database (pyproject.toml)
+psycopg-binary==3.2.6
+    # via psycopg
+pycparser==2.22
+    # via cffi
+pygments==2.19.1
+    # via
+    #   mkdocs-material
+    #   rich
+pymdown-extensions==10.14.3
+    # via mkdocs-material
+pynacl==1.5.0
+    # via paramiko
+pytest==8.3.5
+    # via
+    #   hdx-python-database (pyproject.toml)
+    #   pytest-cov
+pytest-cov==6.0.0
+    # via hdx-python-database (pyproject.toml)
+python-dateutil==2.9.0.post0
+    # via ghp-import
+pyyaml==6.0.2
+    # via
+    #   mkdocs
+    #   mkdocs-get-deps
+    #   pre-commit
+    #   pymdown-extensions
+    #   pyyaml-env-tag
+pyyaml-env-tag==0.1
+    # via mkdocs
+requests==2.32.3
+    # via mkdocs-material
+rich==13.9.4
+    # via mkapi
+six==1.17.0
+    # via python-dateutil
+sqlalchemy==2.0.39
+    # via hdx-python-database (pyproject.toml)
+sshtunnel==0.4.0
+    # via hdx-python-database (pyproject.toml)
+typing-extensions==4.12.2
+    # via
+    #   psycopg
+    #   sqlalchemy
+urllib3==2.3.0
+    # via requests
+virtualenv==20.29.3
+    # via pre-commit
+watchdog==6.0.0
+    # via mkdocs

hdx_python_database-1.3.6/src/hdx/database/__init__.py

@@ -0,0 +1,3 @@
+from ._version import version as __version__  # noqa: F401
+from .database import Database as Database
+from .database import DatabaseError as DatabaseError

{hdx_python_database-1.3.4 → hdx_python_database-1.3.6}/src/hdx/database/_version.py

@@ -1,8 +1,13 @@
-# file generated by setuptools_scm
+# file generated by setuptools-scm
 # don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
 TYPE_CHECKING = False
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Tuple
+    from typing import Union
+
     VERSION_TUPLE = Tuple[Union[int, str], ...]
 else:
     VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.3.4'
-__version_tuple__ = version_tuple = (1, 3, 4)
+__version__ = version = '1.3.6'
+__version_tuple__ = version_tuple = (1, 3, 6)

hdx_python_database-1.3.4/src/hdx/database/__init__.py → hdx_python_database-1.3.6/src/hdx/database/database.py

@@ -3,14 +3,13 @@
 import logging
 from typing import Any, Dict, List, Optional, Tuple, Type, Union
 
-from sqlalchemy import Engine, TableClause, create_engine
+from sqlalchemy import Engine, TableClause, create_engine, insert
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.ext.automap import automap_base
 from sqlalchemy.orm import DeclarativeBase, Session
 from sqlalchemy.pool import NullPool
 from sqlalchemy.sql.ddl import CreateSchema, DropSchema
 
-from ._version import version as __version__  # noqa: F401
 from .dburi import get_connection_uri
 from .no_timezone import Base as NoTZBase
 from .postgresql import restore_from_pgfile, wait_for_postgresql
@@ -123,16 +122,16 @@ class Database:
                 del kwargs["ssh_port"]
             else:
                 ssh_port = 22
-            self.server = sshtunnel.SSHTunnelForwarder(
+            self._server = sshtunnel.SSHTunnelForwarder(
                 (ssh_host, ssh_port),
                 remote_bind_address=(host, port),
                 **kwargs,
             )
-            self.server.start()
-            host = self.server.local_bind_host
-            port = self.server.local_bind_port
+            self._server.start()
+            host = self._server.local_bind_host
+            port = self._server.local_bind_port
         else:
-            self.server = None
+            self._server = None
         if not engine and not db_uri:
             db_uri = get_connection_uri(
                 database,
@@ -158,19 +157,19 @@ class Database:
                 table_base = NoTZBase
         if not engine:
             engine = create_engine(db_uri, poolclass=NullPool, echo=False)
-        self.engine: Engine = engine
+        self._engine: Engine = engine
         if recreate_schema:
             self.recreate_schema(engine, schema_name)
-        self.prepare_results = prepare_fn()
-        self.session, self.base = self.create_session(
+        self._prepare_results = prepare_fn()
+        self._session, self._base = self.create_session(
             engine,
             table_base=table_base,
             reflect=reflect,
         )
         if reflect:
-            self.reflected_classes = self.base.classes
+            self._reflected_classes = self._base.classes
         else:
-            self.reflected_classes = None
+            self._reflected_classes = None
 
     def cleanup(self) -> None:
         """Cleanup SQLAlchemy.
@@ -178,10 +177,10 @@ class Database:
         Returns:
             sqlalchemy.Engine: SQLAlchemy engine
         """
-        self.session.close()
-        self.engine.dispose()
-        if self.server is not None:
-            self.server.stop()
+        self._session.close()
+        self._engine.dispose()
+        if self._server is not None:
+            self._server.stop()
 
     def __enter__(self) -> "Database":
         return self
@@ -195,7 +194,7 @@ class Database:
         Returns:
             None
         """
-        self.base.metadata.drop_all(self.engine)
+        self._base.metadata.drop_all(self._engine)
 
     def get_engine(self) -> Engine:
         """Returns SQLAlchemy engine.
@@ -203,7 +202,7 @@ class Database:
         Returns:
             sqlalchemy.Engine: SQLAlchemy engine
         """
-        return self.engine
+        return self._engine
 
     def get_session(self) -> Session:
         """Returns SQLAlchemy session.
@@ -211,7 +210,7 @@ class Database:
         Returns:
             sqlalchemy.orm.Session: SQLAlchemy session
         """
-        return self.session
+        return self._session
 
     def get_reflected_classes(self) -> Any:
         """Gets reflected classes.
@@ -219,7 +218,7 @@ class Database:
         Returns:
             Any: Reflected classes
         """
-        return self.reflected_classes
+        return self._reflected_classes
 
     def get_prepare_results(self) -> Any:
         """Returns results from prepare function.
@@ -227,7 +226,31 @@ class Database:
         Returns:
             Any: Results from prepare function
         """
-        return self.prepare_results
+        return self._prepare_results
+
+    def batch_populate(
+        self,
+        rows: List[Dict],
+        dbtable: Type[DeclarativeBase],
+        batch_size: int = 1000,
+    ) -> None:
+        """Batch populate database table.
+
+        Args:
+            rows (List[Dict]): List of rows
+            dbtable (Type[DeclarativeBase]): Database table
+            batch_size (int): Batch size. Defaults to 1000.
+
+        Returns:
+            Tuple[Session, Any]: (SQLAlchemy session, base)
+        """
+        batches = range(len(rows) // batch_size + 1)
+        for batch in batches:
+            start_row = batch * batch_size
+            end_row = start_row + batch_size
+            batch_rows = rows[start_row:end_row]
+            self._session.execute(insert(dbtable), batch_rows)
+        self._session.commit()
 
     @staticmethod
     def create_session(

{hdx_python_database-1.3.4 → hdx_python_database-1.3.6}/src/hdx/database/views.py

@@ -7,10 +7,8 @@ https://github.com/sqlalchemy/sqlalchemy/wiki/Views#sqlalchemy-14-20-version
 """
 
 import sqlalchemy as sa
-from sqlalchemy import MetaData, Selectable, TableClause
 from sqlalchemy.ext import compiler
 from sqlalchemy.schema import DDLElement
-from sqlalchemy.sql import table
 
 
 class CreateView(DDLElement):
@@ -45,22 +43,15 @@ def view_doesnt_exist(ddl, target, connection, **kw):
     return not view_exists(ddl, target, connection, **kw)
 
 
-def view(name: str, metadata: MetaData, selectable: Selectable) -> TableClause:
-    """Prepare SQLAlchemy view. Must be run before Base.metadata.create_all.
-
-    Args:
-        name (str): View name
-        metadata (MetaData): Base metadata
-        selectable (Selectable): SQLAlchemy select statement
-
-    Returns:
-        TableClause: SQLAlchemy View
-    """
-    t = table(name)
-
-    t._columns._populate_separate_keys(
-        col._make_proxy(t) for col in selectable.selected_columns
+def view(name, metadata, selectable):
+    t = sa.table(
+        name,
+        *(
+            sa.Column(c.name, c.type, primary_key=c.primary_key)
+            for c in selectable.selected_columns
+        ),
     )
+    t.primary_key.update(c for c in t.c if c.primary_key)
 
     sa.event.listen(
         metadata,

{hdx_python_database-1.3.4 → hdx_python_database-1.3.6}/tests/hdx/database/test_database.py

@@ -31,10 +31,9 @@ class TestDatabase:
         ) as dbdatabase:
             dbsession = dbdatabase.get_session()
             assert str(dbsession.bind.engine.url) == f"sqlite:///{dbpath}"
-            assert dbdatabase.base == NoTZBase
+            assert dbdatabase._base == NoTZBase
             now = datetime(2022, 10, 20, 22, 35, 55, tzinfo=timezone.utc)
-            input_dbtestdate = DBTestDate()
-            input_dbtestdate.test_date = now
+            input_dbtestdate = DBTestDate(test_date=now)
             dbsession.add(input_dbtestdate)
             dbsession.commit()
             dbtestdate = dbsession.execute(select(DBTestDate)).scalar_one()
@@ -42,6 +41,15 @@ class TestDatabase:
             date_view = dbdatabase.get_prepare_results()[0]
             dbtestdate = dbsession.execute(select(date_view)).scalar_one()
             assert dbtestdate == now
+
+            now = datetime(2023, 10, 20, 22, 35, 55, tzinfo=timezone.utc)
+            rows = [{"test_date": now}]
+            dbdatabase.batch_populate(rows, DBTestDate)
+            dbtestdate = (
+                dbsession.execute(select(DBTestDate)).all()[1][0].test_date
+            )
+            assert dbtestdate == now
+
         remove(dbpath)
 
     def test_errors(self):

{hdx_python_database-1.3.4 → hdx_python_database-1.3.6}/tests/hdx/database/test_reflect.py

@@ -27,7 +27,7 @@ class TestReflect:
             dburi = f"sqlite:///{dbpath}"
             assert str(dbengine.url) == dburi
             dbsession = dbdatabase.get_session()
-            assert isinstance(dbdatabase.base, DeclarativeAttributeIntercept)
+            assert isinstance(dbdatabase._base, DeclarativeAttributeIntercept)
             assert str(dbsession.bind.engine.url) == dburi
             Table1 = dbdatabase.get_reflected_classes().table1
             row = dbsession.execute(select(Table1)).scalar_one()

{hdx_python_database-1.3.4 → hdx_python_database-1.3.6}/tests/hdx/database/test_ssh.py

@@ -33,7 +33,7 @@ class TestSSH:
                 str(dbsession.bind.engine.url)
                 == "postgresql+psycopg://myuser@0.0.0.0:12345/mydatabase"
             )
-            assert dbdatabase.base == TZBase
+            assert dbdatabase._base == TZBase
 
         with Database(
             ssh_host="mysshhost", ssh_port=25, table_base=TZBase, **params
@@ -43,7 +43,7 @@ class TestSSH:
                 str(dbsession.bind.engine.url)
                 == "postgresql+psycopg://myuser@0.0.0.0:12345/mydatabase"
             )
-            assert dbdatabase.base == TZBase
+            assert dbdatabase._base == TZBase
 
         with Database(ssh_host="mysshhost", **self.params_pg) as dbdatabase:
             dbsession = dbdatabase.get_session()
@@ -51,7 +51,7 @@ class TestSSH:
                 str(dbsession.bind.engine.url)
                 == "postgresql+psycopg://myuser:***@0.0.0.0:12345/mydatabase"
             )
-            assert dbdatabase.base == NoTZBase
+            assert dbdatabase._base == NoTZBase
 
         with Database(
             ssh_host="mysshhost", ssh_port=25, **params
@@ -61,4 +61,4 @@ class TestSSH:
                 str(dbsession.bind.engine.url)
                 == "postgresql+psycopg://myuser@0.0.0.0:12345/mydatabase"
             )
-            assert dbdatabase.base == NoTZBase
+            assert dbdatabase._base == NoTZBase

hdx_python_database-1.3.4/documentation/.readthedocs.yaml

@@ -1,16 +0,0 @@
-version: 2
-
-mkdocs:
-  configuration: docs/mkdocs.yml
-  fail_on_warning: false
-
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.11"
-  jobs:
-    pre_build:
-      - pip install --upgrade mkdocs
-      - pip install mkdocs-material
-      - pip install pydoc-markdown
-      - pydoc-markdown documentation/pydoc-markdown.yaml

hdx_python_database-1.3.4/documentation/pydoc-markdown.yaml

@@ -1,29 +0,0 @@
-loaders:
-  - type: python
-    search_path:
-      - ../src
-    packages:
-      - hdx.database
-renderer:
-  type: mkdocs
-  output_directory: docs
-  mkdocs_config:
-    site_name: HDX Python Database
-    theme: material
-    repo_url: "https://github.com/OCHA-DAP/hdx-python-database"
-  markdown:
-    source_linker:
-      type: github
-      repo: OCHA-DAP/hdx-python-database
-  pages:
-    - title: Home
-      name: index
-      source: main.md
-    - title: API Documentation
-      children:
-        - title: Database
-          contents:
-            - hdx.database.Database.*
-        - title: PostgreSQL specific
-          contents:
-            - hdx.database.postgresql.*

hdx_python_database-1.3.4/requirements.txt

@@ -1,62 +0,0 @@
-# This file was autogenerated by uv via the following command:
-#    uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
-bcrypt==4.2.0
-    # via paramiko
-cffi==1.17.1
-    # via
-    #   cryptography
-    #   pynacl
-cfgv==3.4.0
-    # via pre-commit
-coverage==7.6.3
-    # via pytest-cov
-cryptography==43.0.1
-    # via paramiko
-distlib==0.3.9
-    # via virtualenv
-filelock==3.16.1
-    # via virtualenv
-greenlet==3.1.1
-    # via sqlalchemy
-identify==2.6.1
-    # via pre-commit
-iniconfig==2.0.0
-    # via pytest
-nodeenv==1.9.1
-    # via pre-commit
-packaging==24.1
-    # via pytest
-paramiko==3.5.0
-    # via sshtunnel
-platformdirs==4.3.6
-    # via virtualenv
-pluggy==1.5.0
-    # via pytest
-pre-commit==4.0.1
-    # via hdx-python-database (pyproject.toml)
-psycopg==3.2.3
-    # via hdx-python-database (pyproject.toml)
-psycopg-binary==3.2.3
-    # via psycopg
-pycparser==2.22
-    # via cffi
-pynacl==1.5.0
-    # via paramiko
-pytest==8.3.3
-    # via
-    #   hdx-python-database (pyproject.toml)
-    #   pytest-cov
-pytest-cov==5.0.0
-    # via hdx-python-database (pyproject.toml)
-pyyaml==6.0.2
-    # via pre-commit
-sqlalchemy==2.0.35
-    # via hdx-python-database (pyproject.toml)
-sshtunnel==0.4.0
-    # via hdx-python-database (pyproject.toml)
-typing-extensions==4.12.2
-    # via
-    #   psycopg
-    #   sqlalchemy
-virtualenv==20.26.6
-    # via pre-commit