hdx-python-database 1.1.7__tar.gz → 1.2.2__tar.gz

{hdx-python-database-1.1.7 → hdx-python-database-1.2.2}/.github/workflows/publish.yml

@@ -1,4 +1,4 @@
-name: publish
+name: Publish to PyPI
 
 on:
   release:
@@ -9,13 +9,13 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Get history and tags for SCM versioning to work
       run: |
         git fetch --prune --unshallow
         git fetch --depth=1 origin +refs/tags/*:refs/tags/*
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: '3.x'
     - name: Install dependencies

{hdx-python-database-1.1.7 → hdx-python-database-1.2.2}/.github/workflows/run-python-tests.yml

@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, lint and run tests
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: build
+name: Run tests
 
 on:
   push:
@@ -16,9 +16,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: '3.x'
     - name: Install dependencies
@@ -29,11 +29,11 @@ jobs:
       run: |
         tox
     - name: Publish Unit Test Results
-      uses: EnricoMi/publish-unit-test-result-action@v1
+      uses: EnricoMi/publish-unit-test-result-action@v2
       if: always()
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
-        files: .tox/*.xml
+        junit_files: .tox/*.xml
     - name: Upload Coverage Results
-      uses: codecov/codecov-action@v2
+      uses: codecov/codecov-action@v3
       if: success()

{hdx-python-database-1.1.7 → hdx-python-database-1.2.2}/.readthedocs.yml

@@ -5,13 +5,13 @@ mkdocs:
   fail_on_warning: false
 
 build:
-  os: ubuntu-20.04
+  os: ubuntu-22.04
   tools:
-    python: "3.10"
+    python: "3.11"
   jobs:
     pre_build:
-      - pip install pydoc-markdown==4.5.1
-      - pip install mkdocs==1.3.0
+      - pip install --upgrade mkdocs
+      - pip install --upgrade pydoc-markdown
       - pydoc-markdown
       - mv docs/content/* docs
       - rmdir docs/content

{hdx-python-database-1.1.7 → hdx-python-database-1.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hdx-python-database
-Version: 1.1.7
+Version: 1.2.2
 Summary: HDX Python database utilities
 Home-page: https://github.com/OCHA-DAP/hdx-python-database
 Author: Michael Rans
@@ -13,12 +13,11 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Natural Language :: English
@@ -26,12 +25,12 @@ Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: Unix
 Classifier: Operating System :: MacOS
 Classifier: Operating System :: Microsoft :: Windows
-Requires-Python: >=3.6
+Requires-Python: >=3.8
 Description-Content-Type: text/markdown; charset=UTF-8
 Provides-Extra: postgresql
 License-File: LICENSE
 
-[![Build Status](https://github.com/OCHA-DAP/hdx-python-database/workflows/build/badge.svg)](https://github.com/OCHA-DAP/hdx-python-database/actions?query=workflow%3Abuild)
+[![Build Status](https://github.com/OCHA-DAP/hdx-python-database/actions/workflows/run-python-tests.yml/badge.svg)](https://github.com/OCHA-DAP/hdx-python-database/actions/workflows/run-python-tests.yml)
 [![Coverage Status](https://codecov.io/gh/OCHA-DAP/hdx-python-database/branch/main/graph/badge.svg?token=JpWZc5js4y)](https://codecov.io/gh/OCHA-DAP/hdx-python-database)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)

{hdx-python-database-1.1.7 → hdx-python-database-1.2.2}/README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://github.com/OCHA-DAP/hdx-python-database/workflows/build/badge.svg)](https://github.com/OCHA-DAP/hdx-python-database/actions?query=workflow%3Abuild)
+[![Build Status](https://github.com/OCHA-DAP/hdx-python-database/actions/workflows/run-python-tests.yml/badge.svg)](https://github.com/OCHA-DAP/hdx-python-database/actions/workflows/run-python-tests.yml)
 [![Coverage Status](https://codecov.io/gh/OCHA-DAP/hdx-python-database/branch/main/graph/badge.svg?token=JpWZc5js4y)](https://codecov.io/gh/OCHA-DAP/hdx-python-database)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)

hdx-python-database-1.2.2/doc/main.md

@@ -0,0 +1,99 @@
+# Summary
+
+The HDX Python Database Library provides utilities for connecting to databases in a standard way including
+through an ssh tunnel if needed. It is built on top of SQLAlchemy and simplifies its setup.
+
+# Information
+
+This library is part of the [Humanitarian Data Exchange](https://data.humdata.org/) (HDX) project. If you have 
+humanitarian related data, please upload your datasets to HDX.
+
+The code for the library is [here](https://github.com/OCHA-DAP/hdx-python-database).
+The library has detailed API documentation which can be found in the menu at the top. 
+
+Additional postgresql functionality is available if this library is installed with extra "postgresql":
+
+    pip install hdx-python-database[postgresql]
+
+## Breaking changes
+
+From 1.2.2, database datetime columns are assumed to be timezoneless unless
+db_has_tz is set to True.
+
+From 1.2.1, wait_for_postgresql takes connection URI not database parameters, 
+get_params_from_sqlalchemy_url renamed to get_params_from_connection_uri
+and moved to dburi module, get_sqlalchemy_url renamed to get_connection_uri and 
+moved to dburi module. New function remove_driver_from_uri in dburi module.
+Parameter driver replaced by dialect+driver. Supports Python 3.8 and later.
+
+From 1.1.2, the postgres module is renamed postgresql and function wait_for_postgres
+is renamed wait_for_postgresql.
+
+Versions from 1.0.8 support Python 3.6 and later.
+
+Versions from 1.0.6 no longer support Python 2.7. 
+
+# Description of Utilities
+
+## Database
+
+Your SQLAlchemy database tables must inherit from `Base` in
+`hdx.database` or `hdx.database.database_notz` eg. :
+
+    from hdx.database import Base
+    class MyTable(Base):
+        my_col = Column(Integer, ForeignKey(MyTable2.col2), primary_key=True)
+
+Then a connection can be made to a database as follows including through an SSH
+tunnel:
+
+    # Get SQLAlchemy session object given database parameters and
+    # if needed SSH parameters. If database is PostgreSQL, will poll
+    # till it is up.
+    from hdx.database import Database
+    with Database(database="db", host="1.2.3.4", username="user", 
+                  password="pass", dialect="dialect", driver="driver", 
+                  ssh_host="5.6.7.8", ssh_port=2222, ssh_username="sshuser", 
+                  ssh_private_key="path_to_key", db_has_tz=True) as session:
+        session.query(...)
+
+`db_has_tz` which defaults to `False` indicates whether database datetime 
+columns have timezones. If not, conversion occurs between Python datetimes
+with timezones to timezoneless database columns.   
+
+## Connection URI
+
+There are functions to handle converting from connection URIs to parameters and
+vice-versa as well as a function to remove the driver string from a connection 
+URI that contains both dialect and driver.
+
+    # Extract dictionary of parameters from database connection URI
+    result = get_params_from_connection_uri(
+        "postgresql+psycopg://myuser:mypass@myserver:1234/mydatabase"
+    )
+
+    # Build database connection URI from dictionary of parameters
+    params_pg = {
+        "database": "mydatabase",
+        "host": "myserver",
+        "port": 1234,
+        "username": "myuser",
+        "password": "mypass",
+        "dialect": "postgresql",
+        "driver": "psycopg",
+    }
+    result = get_connection_uri(**params_pg)
+
+    db_uri_nd = remove_driver_from_uri(
+        "postgresql+psycopg://myuser:mypass@myserver:1234/mydatabase"
+    )
+
+## PostgreSQL specific
+
+There is a PostgreSQL specific call that only returns when the PostgreSQL database
+is available:
+
+    # Wait until PostgreSQL is up
+    # Library should be installed with hdx-python-database[postgresql]
+    wait_for_postgresql("mydatabase", "myserver", 5432, "myuser", "mypass")
+

{hdx-python-database-1.1.7 → hdx-python-database-1.2.2}/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = [ "setuptools>=45", "setuptools_scm[toml]>=6.2", "wheel"]
+requires = [ "setuptools>=65", "setuptools_scm[toml]>=7.1", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]
@@ -15,22 +15,12 @@ profile = "black"
 line_length = 79
 multi_line_output = 3
 
-[tool.flakeheaven]
-# make output nice
-format = "grouped"
-max_line_length = 79
-# show line of source code in output
-show_source = true
+[tool.flake8]
 exclude = ["_version.py"]
-
-    # list of plugins and rules for them
-    [tool.flakeheaven.plugins]
-    mccabe = ["+*"]
-    pycodestyle = ["+*", "-E203", "-E501", "-W503"]
-    pyflakes = ["+*"]
-    pylint = ["+*"]
-    flake8-isort = ["+*"]
-    flake8-black = ["+*"]
+ignore = ["E203", "E501", "W503"]
+max-line-length = 79
+count = true
+show_source = true
 
 [tool.pytest.ini_options]
 pythonpath = "src"
@@ -97,45 +87,41 @@ output_directory = "docs"
 [tool.tox]
 legacy_tox_ini = """
 [tox]
-isolated_build = true
-envlist = py310, lint
+envlist = py311, lint
 
 [gh-actions]
 python =
-    3: py310, lint
+    3: py311, lint
 
 [testenv]
-recreate = true
-wheel = true
-wheel_pep517 = build
+package = wheel
+wheel_build_env = .pkg
 deps =
     -r test-requirements.txt
 commands =
     pytest --cov=hdx --no-cov-on-fail --junitxml=.tox/test-results.xml --cov-report=xml --cov-report=term-missing
 
 [testenv:lint]
-wheel_build_env = py310
 deps =
-    flakeheaven
     flake8
     flake8-isort
     flake8-black
+    flake8-pyproject
 commands =
-    flakeheaven lint src tests
+    flake8 --color=always src tests
 
 [testenv:docs]
-wheel_build_env = py310
 deps =
     pydoc-markdown
 commands =
     pydoc-markdown
 
 [testenv:publish]
-wheel_build_env = py310
-passenv = SSH_AUTH_SOCK TWINE_USERNAME TWINE_PASSWORD
+package = sdist
+pass_env = SSH_AUTH_SOCK, TWINE_USERNAME, TWINE_PASSWORD
 deps =
     twine
 
 commands =
-    twine upload {distdir}/*
+    twine upload {work_dir}/{package_env}/dist/*
 """

hdx-python-database-1.2.2/requirements.txt

@@ -0,0 +1,3 @@
+psycopg[binary]==3.1.8
+sqlalchemy==2.0.4
+sshtunnel==0.4.0

{hdx-python-database-1.1.7 → hdx-python-database-1.2.2}/setup.cfg

@@ -17,12 +17,11 @@ classifiers =
 	Programming Language :: Python
 	Programming Language :: Python :: 3
 	Programming Language :: Python :: 3 :: Only
-	Programming Language :: Python :: 3.6
-	Programming Language :: Python :: 3.7
 	Programming Language :: Python :: 3.8
 	Programming Language :: Python :: 3.9
 	Programming Language :: Python :: 3.10
 	Programming Language :: Python :: 3.11
+	Programming Language :: Python :: 3.12
 	Intended Audience :: Developers
 	License :: OSI Approved :: MIT License
 	Natural Language :: English
@@ -37,13 +36,13 @@ packages = find_namespace:
 include_package_data = True
 package_dir = 
 	=src
-python_requires = >=3.6
+python_requires = >=3.8
 install_requires = 
-	sqlalchemy
+	sqlalchemy>=2
 	sshtunnel
 
 [options.extras_require]
-postgresql = psycopg2-binary
+postgresql = psycopg[binary]
 
 [options.packages.find]
 where = src

hdx-python-database-1.2.2/src/hdx/database/__init__.py

@@ -0,0 +1,127 @@
+from ._version import version as __version__  # noqa: F401
+from .dburi import get_connection_uri
+
+"""Database utilities"""
+import logging
+from typing import Any, Optional, Type, Union
+
+from sqlalchemy import create_engine
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
+from sqlalchemy.pool import NullPool
+from sshtunnel import SSHTunnelForwarder
+
+from .no_timezone import Base as NoTZBase
+from .postgresql import wait_for_postgresql
+
+logger = logging.getLogger(__name__)
+
+
+class Base(DeclarativeBase):
+    @declared_attr.directive
+    def __tablename__(cls):
+        return f"{cls.__name__.lower()}s"
+
+
+class Database:
+    """Database helper class to handle ssh tunnels, waiting for PostgreSQL to
+    be up etc. Can be used in a with statement returning a Session object.
+    db_has_tz which defaults to False indicates whether database datetime
+    columns have timezones. If not, conversion occurs between Python datetimes
+    with timezones to timezoneless database columns.
+
+    Args:
+        database (Optional[str]): Database name
+        host (Optional[str]): Host where database is located
+        port (Union[int, str, None]): Database port
+        username (Optional[str]): Username to log into database
+        password (Optional[str]): Password to log into database
+        dialect (str): Database dialect. Defaults to "postgresql".
+        driver (Optional[str]): Database driver. Defaults to None (psycopg if postgresql or None)
+        db_has_tz (bool): True if db datetime columns have timezone. Defaults to False.
+        **kwargs: See below
+        ssh_host (str): SSH host (the server to connect to)
+        ssh_port (int): SSH port. Defaults to 22.
+        ssh_username (str): SSH username
+        ssh_password (str): SSH password
+        ssh_private_key: Location of SSH private key (instead of ssh_password)
+        For more advanced usage, see SSHTunnelForwarder documentation.
+
+    """
+
+    def __init__(
+        self,
+        database: Optional[str] = None,
+        host: Optional[str] = None,
+        port: Union[int, str, None] = None,
+        username: Optional[str] = None,
+        password: Optional[str] = None,
+        dialect: str = "postgresql",
+        driver: Optional[str] = None,
+        db_has_tz: bool = False,
+        **kwargs: Any,
+    ) -> None:
+        if port is not None:
+            port = int(port)
+        if len(kwargs) != 0:
+            ssh_host = kwargs["ssh_host"]
+            del kwargs["ssh_host"]
+            ssh_port = kwargs.get("ssh_port")
+            if ssh_port is not None:
+                ssh_port = int(ssh_port)
+                del kwargs["ssh_port"]
+            else:
+                ssh_port = 22
+            self.server = SSHTunnelForwarder(
+                (ssh_host, ssh_port),
+                remote_bind_address=(host, port),
+                **kwargs,
+            )
+            self.server.start()
+            host = self.server.local_bind_host
+            port = self.server.local_bind_port
+        else:
+            self.server = None
+        db_uri = get_connection_uri(
+            database,
+            host,
+            port,
+            username,
+            password,
+            dialect=dialect,
+            driver=driver,
+        )
+        if dialect == "postgresql":
+            wait_for_postgresql(db_uri)
+        if db_has_tz:
+            table_base = Base
+        else:
+            table_base = NoTZBase
+        self.session = self.get_session(db_uri, table_base=table_base)
+
+    def __enter__(self) -> Session:
+        return self.session
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        self.session.close()
+        if self.server is not None:
+            self.server.stop()
+
+    @staticmethod
+    def get_session(
+        db_uri: str, table_base: Type[DeclarativeBase] = NoTZBase
+    ) -> Session:
+        """Gets SQLAlchemy session given url. Tables must inherit from Base in
+        hdx.utilities.database unless base is defined.
+
+        Args:
+            db_uri (str): Connection URI
+            table_base (Type[DeclarativeBase]): Base database table class. Defaults to NoTZBase.
+
+        Returns:
+            sqlalchemy.orm.Session: SQLAlchemy session
+        """
+        engine = create_engine(db_uri, poolclass=NullPool, echo=False)
+        Session = sessionmaker(bind=engine)
+        table_base.metadata.create_all(engine)
+        return Session()

hdx-python-database-1.2.2/src/hdx/database/_version.py

@@ -0,0 +1,4 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+__version__ = version = '1.2.2'
+__version_tuple__ = version_tuple = (1, 2, 2)

hdx-python-database-1.2.2/src/hdx/database/dburi.py

@@ -0,0 +1,99 @@
+"""Connection URI utilities"""
+
+from typing import Any, Dict, Optional, Union
+from urllib.parse import urlsplit
+
+
+def get_params_from_connection_uri(
+    db_uri: str,
+    include_driver: bool = True,
+) -> Dict[str, Any]:
+    """Gets PostgreSQL database connection parameters from connection URI
+
+    Args:
+        db_uri (str): Connection URI
+        include_driver (bool): Whether to include driver in params. Defaults to True.
+
+    Returns:
+        Dict[str,Any]: Dictionary of database connection parameters
+    """
+    result = urlsplit(db_uri)
+    dialectdriver = result.scheme.split("+")
+    if len(dialectdriver) == 1:
+        dialect = dialectdriver[0]
+        driver = None
+    else:
+        dialect, driver = dialectdriver
+    result = {
+        "database": result.path[1:],
+        "host": result.hostname,
+        "port": result.port,
+        "username": result.username,
+        "password": result.password,
+        "dialect": dialect,
+    }
+    if include_driver:
+        result["driver"] = driver
+    return result
+
+
+def get_connection_uri(
+    database: Optional[str] = None,
+    host: Optional[str] = None,
+    port: Union[int, str, None] = None,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    dialect: str = "postgresql",
+    driver: Optional[str] = None,
+    include_driver: bool = True,
+) -> str:
+    """Gets connection uri from database connection parameters
+
+    Args:
+        database (Optional[str]): Database name
+        host (Optional[str]): Host where database is located
+        port (Union[int, str, None]): Database port
+        username (Optional[str]): Username to log into database
+        password (Optional[str]): Password to log into database
+        dialect (str): Database dialect. Defaults to "postgresql".
+        driver (Optional[str]): Database driver. Defaults to None (psycopg if postgresql or None)
+        include_driver (bool): Whether to include driver in uri. Defaults to True.
+
+    Returns:
+        str: Connection URI
+    """
+    strings = [dialect]
+    if include_driver:
+        if dialect == "postgresql" and driver is None:
+            driver = "psycopg"
+        if driver:
+            strings.append(f"+{driver}")
+    strings.append("://")
+    if username:
+        strings.append(username)
+        if password:
+            strings.append(f":{password}@")
+        else:
+            strings.append("@")
+    if host:
+        strings.append(host)
+    if port is not None:
+        strings.append(f":{port}")
+    if database:
+        strings.append(f"/{database}")
+    return "".join(strings)
+
+
+def remove_driver_from_uri(db_uri: str) -> str:
+    """Remove driver from connection URI
+
+    Args:
+        db_uri (str): Connection URI
+
+    Returns:
+        str: Connection URI with driver removed
+    """
+    result = urlsplit(db_uri)
+    dialectdriver = result.scheme
+    dialect = dialectdriver.split("+")[0]
+    return db_uri.replace(dialectdriver, dialect)

hdx-python-database-1.2.2/src/hdx/database/no_timezone.py

@@ -0,0 +1,40 @@
+"""Utilities for database datetime columns without timezone"""
+from datetime import datetime, timezone
+from typing import Optional
+
+from sqlalchemy import DateTime, TypeDecorator
+from sqlalchemy.orm import DeclarativeBase, declared_attr
+
+
+class ConversionNoTZ(TypeDecorator):
+    """Convert from/to datetime with timezone from database columns that don't
+    use timezone"""
+
+    impl = DateTime
+    cache_ok = True
+
+    def process_bind_param(self, value: Optional[datetime], _):
+        if value is None:
+            return value
+        if value.tzinfo:
+            value = value.astimezone(timezone.utc)
+
+        return value.replace(tzinfo=None)
+
+    def process_result_value(self, value, _):
+        if value is None:
+            return value
+        if value.tzinfo is None:
+            return value.replace(tzinfo=timezone.utc)
+
+        return value.astimezone(timezone.utc)
+
+
+class Base(DeclarativeBase):
+    type_annotation_map = {
+        datetime: ConversionNoTZ,
+    }
+
+    @declared_attr.directive
+    def __tablename__(cls):
+        return f"{cls.__name__.lower()}s"

hdx-python-database-1.2.2/src/hdx/database/postgresql.py

@@ -0,0 +1,40 @@
+"""PostgreSQL specific utilities"""
+import logging
+import time
+
+from .dburi import remove_driver_from_uri
+
+try:
+    import psycopg
+except ImportError:
+    psycopg = None
+    # dependency missing, issue a warning
+    import warnings
+
+    warnings.warn(
+        "psycopg not found! Please install hdx-python-database[postgresql] to enable."
+    )
+
+logger = logging.getLogger(__name__)
+
+
+def wait_for_postgresql(db_uri: str) -> None:
+    """Waits for PostgreSQL database to be up
+
+    Args:
+        db_uri (str): Connection URI
+
+    Returns:
+        None
+    """
+    connecting_string = "Checking for PostgreSQL..."
+    db_uri_nd = remove_driver_from_uri(db_uri)
+    while True:
+        try:
+            logger.info(connecting_string)
+            connection = psycopg.connect(db_uri_nd, connect_timeout=3)
+            connection.close()
+            logger.info("PostgreSQL is running!")
+            break
+        except psycopg.OperationalError:
+            time.sleep(1)