hdx-python-database 1.3.5__tar.gz → 1.3.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/.github/workflows/publish.yaml +9 -6
  2. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/.github/workflows/run-python-tests.yaml +4 -0
  3. hdx_python_database-1.3.5/.config/pre-commit-config.yaml → hdx_python_database-1.3.7/.pre-commit-config.yaml +5 -6
  4. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/PKG-INFO +6 -2
  5. hdx_python_database-1.3.7/documentation/.readthedocs.yaml +14 -0
  6. hdx_python_database-1.3.5/documentation/main.md → hdx_python_database-1.3.7/documentation/index.md +11 -0
  7. hdx_python_database-1.3.7/documentation/mkdocs.yaml +18 -0
  8. hdx_python_database-1.3.7/hatch.toml +34 -0
  9. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/pyproject.toml +3 -48
  10. {hdx_python_database-1.3.5/.config → hdx_python_database-1.3.7}/pytest.ini +1 -1
  11. hdx_python_database-1.3.7/requirements.txt +141 -0
  12. {hdx_python_database-1.3.5/.config → hdx_python_database-1.3.7}/ruff.toml +0 -1
  13. hdx_python_database-1.3.7/src/hdx/database/__init__.py +3 -0
  14. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/src/hdx/database/_version.py +9 -4
  15. hdx_python_database-1.3.5/src/hdx/database/__init__.py → hdx_python_database-1.3.7/src/hdx/database/database.py +68 -27
  16. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/src/hdx/database/postgresql.py +13 -2
  17. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/src/hdx/database/utils.py +1 -3
  18. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/tests/hdx/database/test_database.py +9 -3
  19. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/tests/hdx/database/test_dburi.py +4 -12
  20. hdx_python_database-1.3.5/tests/hdx/database/test_recreate.py → hdx_python_database-1.3.7/tests/hdx/database/test_drop_recreate.py +7 -0
  21. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/tests/hdx/database/test_reflect.py +1 -1
  22. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/tests/hdx/database/test_ssh.py +5 -7
  23. hdx_python_database-1.3.5/documentation/.readthedocs.yaml +0 -16
  24. hdx_python_database-1.3.5/documentation/pydoc-markdown.yaml +0 -29
  25. hdx_python_database-1.3.5/requirements.txt +0 -62
  26. /hdx_python_database-1.3.5/.config/coveragerc → /hdx_python_database-1.3.7/.coveragerc +0 -0
  27. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/.gitignore +0 -0
  28. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/CONTRIBUTING.md +0 -0
  29. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/LICENSE +0 -0
  30. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/README.md +0 -0
  31. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/src/hdx/database/dburi.py +0 -0
  32. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/src/hdx/database/no_timezone.py +0 -0
  33. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/src/hdx/database/views.py +0 -0
  34. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/src/hdx/database/with_timezone.py +0 -0
  35. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/tests/fixtures/test.db +0 -0
  36. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/tests/hdx/database/__init__.py +0 -0
  37. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/tests/hdx/database/conftest.py +0 -0
  38. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/tests/hdx/database/dbtestdate.py +0 -0
  39. {hdx_python_database-1.3.5 → hdx_python_database-1.3.7}/tests/hdx/database/test_postgresql.py +0 -0
@@ -8,6 +8,13 @@ jobs:
8
8
  publish:
9
9
  runs-on: ubuntu-latest
10
10
 
11
+ environment:
12
+ name: pypi
13
+ url: https://pypi.org/p/hdx-python-database
14
+
15
+ permissions:
16
+ id-token: write # IMPORTANT: mandatory for trusted publishing
17
+
11
18
  steps:
12
19
  - uses: actions/checkout@v4
13
20
  - name: Get history and tags for versioning to work
@@ -26,9 +33,5 @@ jobs:
26
33
  - name: Build with hatch
27
34
  run: |
28
35
  hatch build
29
- - name: Publish with hatch
30
- env:
31
- HATCH_INDEX_USER: ${{secrets.HATCH_INDEX_USER}}
32
- HATCH_INDEX_AUTH: ${{secrets.HATCH_INDEX_AUTH}}
33
- run: |
34
- hatch publish
36
+ - name: Publish distribution 📦 to PyPI
37
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -17,6 +17,10 @@ jobs:
17
17
  build:
18
18
  runs-on: ubuntu-latest
19
19
 
20
+ permissions:
21
+ checks: write
22
+ pull-requests: write
23
+
20
24
  steps:
21
25
  - uses: actions/checkout@v4
22
26
  - name: Set up Python
@@ -1,5 +1,5 @@
1
1
  default_language_version:
2
- python: python3.12
2
+ python: python3.13
3
3
  repos:
4
4
  - repo: https://github.com/pre-commit/pre-commit-hooks
5
5
  rev: v5.0.0
@@ -8,16 +8,15 @@ repos:
8
8
  - id: end-of-file-fixer
9
9
  - id: check-ast
10
10
  - repo: https://github.com/astral-sh/ruff-pre-commit
11
- rev: v0.9.6
11
+ rev: v0.12.0
12
12
  hooks:
13
13
  # Run the linter.
14
- - id: ruff
15
- args: [--config, .config/ruff.toml, --fix]
14
+ - id: ruff-check
15
+ args: [ --fix ]
16
16
  # Run the formatter.
17
17
  - id: ruff-format
18
- args: [--config, .config/ruff.toml]
19
18
  - repo: https://github.com/astral-sh/uv-pre-commit
20
- rev: 0.5.29
19
+ rev: 0.7.14
21
20
  hooks:
22
21
  # Run the pip compile
23
22
  - id: pip-compile
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: hdx-python-database
3
- Version: 1.3.5
3
+ Version: 1.3.7
4
4
  Summary: HDX Python database utilities
5
5
  Project-URL: Homepage, https://github.com/OCHA-DAP/hdx-python-database
6
6
  Author-email: Michael Rans <rans@email.com>
@@ -24,17 +24,21 @@ Classifier: Programming Language :: Python :: 3.10
24
24
  Classifier: Programming Language :: Python :: 3.11
25
25
  Classifier: Programming Language :: Python :: 3.12
26
26
  Classifier: Topic :: Software Development :: Libraries :: Python Modules
27
- Requires-Python: >=3.8
27
+ Requires-Python: >=3.10
28
28
  Requires-Dist: sqlalchemy>=2
29
29
  Provides-Extra: dev
30
30
  Requires-Dist: pre-commit; extra == 'dev'
31
+ Provides-Extra: docs
32
+ Requires-Dist: mkapi; extra == 'docs'
31
33
  Provides-Extra: postgresql
32
34
  Requires-Dist: psycopg[binary]; extra == 'postgresql'
33
35
  Provides-Extra: sshtunnel
34
36
  Requires-Dist: sshtunnel; extra == 'sshtunnel'
35
37
  Provides-Extra: test
38
+ Requires-Dist: psycopg[binary]; extra == 'test'
36
39
  Requires-Dist: pytest; extra == 'test'
37
40
  Requires-Dist: pytest-cov; extra == 'test'
41
+ Requires-Dist: sshtunnel; extra == 'test'
38
42
  Description-Content-Type: text/markdown
39
43
 
40
44
  [![Build Status](https://github.com/OCHA-DAP/hdx-python-database/actions/workflows/run-python-tests.yaml/badge.svg)](https://github.com/OCHA-DAP/hdx-python-database/actions/workflows/run-python-tests.yaml)
@@ -0,0 +1,14 @@
1
+ # Required
2
+ version: 2
3
+
4
+ # Set the version of Python and other tools you might need
5
+ build:
6
+ os: ubuntu-24.04
7
+ tools:
8
+ python: "3.12"
9
+ jobs:
10
+ pre_build:
11
+ - pip install .[docs]
12
+
13
+ mkdocs:
14
+ configuration: documentation/mkdocs.yaml
@@ -61,6 +61,7 @@ Versions from 1.0.6 no longer support Python 2.7.
61
61
  A fresh schema can be created using the function `recreate_schema`. Any
62
62
  existing schema with that name will be dropped eg.
63
63
 
64
+ from hdx.database import Database
64
65
  Database.recreate_schema(engine, db_uri)
65
66
 
66
67
  Your SQLAlchemy database tables must inherit from `Base` in
@@ -117,6 +118,16 @@ If a prepare function is supplied in prepare_fn, it will be executed before
117
118
  Base.metadata.create_all and the results of it returned in instance variable
118
119
  prepare_results.
119
120
 
121
+ Batch population of rows (list of dictionaries) can be achieved with the
122
+ batch_populate method of Database, for example:
123
+
124
+ now = datetime(2023, 10, 20, 22, 35, 55, tzinfo=timezone.utc)
125
+ rows = [{"test_date": now}]
126
+ dbdatabase.batch_populate(rows, DBTestDate)
127
+ dbtestdate = (
128
+ dbsession.execute(select(DBTestDate)).all()[1][0].test_date
129
+ )
130
+
120
131
 
121
132
  ## Connection URI
122
133
 
@@ -0,0 +1,18 @@
1
+ site_name: HDX Python Database
2
+ repo_url: https://github.com/OCHA-DAP/hdx-python-database/
3
+ repo_name: OCHA-DAP/hdx-python-database
4
+ docs_dir: .
5
+ site_dir: ../site
6
+ theme:
7
+ name: material
8
+ highlightjs: true
9
+ plugins:
10
+ - search
11
+ - mkapi
12
+ nav:
13
+ - Home: index.md
14
+ - API Documentation:
15
+ - Database:
16
+ - $src/hdx.database.Database
17
+ - $src/hdx.database.DatabaseError
18
+ - PostgreSQL specific: $src/hdx.database.postgresql.*
@@ -0,0 +1,34 @@
1
+ # Build
2
+
3
+ [build.targets.wheel]
4
+ packages = ["src/hdx"]
5
+
6
+ [build.hooks.vcs]
7
+ version-file = "src/hdx/database/_version.py"
8
+
9
+ # Versioning
10
+
11
+ [version]
12
+ source = "vcs"
13
+
14
+ [version.raw-options]
15
+ local_scheme = "no-local-version"
16
+ version_scheme = "python-simplified-semver"
17
+
18
+ # Tests
19
+
20
+ [envs.hatch-test]
21
+ features = ["test"]
22
+
23
+ [[envs.hatch-test.matrix]]
24
+ python = ["3.13"]
25
+
26
+ [envs.hatch-test.scripts]
27
+ run = """
28
+ pytest --rootdir=. --junitxml=test-results.xml --cov --no-cov-on-fail \
29
+ --cov-report=lcov --cov-report=term-missing
30
+ """
31
+
32
+ [envs.hatch-static-analysis]
33
+ config-path = "none"
34
+ dependencies = ["ruff==0.12.0"]
@@ -31,7 +31,7 @@ classifiers = [
31
31
  "Operating System :: MacOS",
32
32
  "Operating System :: Microsoft :: Windows",
33
33
  ]
34
- requires-python = ">=3.8"
34
+ requires-python = ">=3.10"
35
35
 
36
36
  dependencies = [
37
37
  "sqlalchemy>=2",
@@ -48,51 +48,6 @@ Homepage = "https://github.com/OCHA-DAP/hdx-python-database"
48
48
  [project.optional-dependencies]
49
49
  sshtunnel = ["sshtunnel"]
50
50
  postgresql = ["psycopg[binary]"]
51
- test = ["pytest", "pytest-cov"]
51
+ test = ["sshtunnel", "psycopg[binary]", "pytest", "pytest-cov"]
52
52
  dev = ["pre-commit"]
53
-
54
-
55
- #########
56
- # Hatch #
57
- #########
58
-
59
- # Build
60
-
61
- [tool.hatch.build.targets.wheel]
62
- packages = ["src/hdx"]
63
-
64
- [tool.hatch.build.hooks.vcs]
65
- version-file = "src/hdx/database/_version.py"
66
-
67
- # Versioning
68
-
69
- [tool.hatch.version]
70
- source = "vcs"
71
-
72
- [tool.hatch.version.raw-options]
73
- local_scheme = "no-local-version"
74
- version_scheme = "python-simplified-semver"
75
-
76
- # Tests
77
-
78
- [tool.hatch.envs.hatch-test]
79
- features = ["sshtunnel", "postgresql", "test"]
80
-
81
- [[tool.hatch.envs.hatch-test.matrix]]
82
- python = ["3.12"]
83
-
84
- [tool.hatch.envs.hatch-test.scripts]
85
- run = """
86
- pytest -c .config/pytest.ini --rootdir=. --junitxml=test-results.xml \
87
- --cov --cov-config=.config/coveragerc --no-cov-on-fail \
88
- --cov-report=lcov --cov-report=term-missing
89
- """
90
-
91
- [tool.hatch.envs.hatch-static-analysis]
92
- dependencies = ["ruff==0.9.6"]
93
-
94
- [tool.hatch.envs.hatch-static-analysis.scripts]
95
- format-check = ["ruff format --config .config/ruff.toml --check --diff {args:.}",]
96
- format-fix = ["ruff format --config .config/ruff.toml {args:.}",]
97
- lint-check = ["ruff check --config .config/ruff.toml {args:.}",]
98
- lint-fix = ["ruff check --config .config/ruff.toml --fix {args:.}",]
53
+ docs = ["mkapi"]
@@ -1,4 +1,4 @@
1
1
  [pytest]
2
- pythonpath = ../src
2
+ pythonpath = src
3
3
  addopts = "--color=yes"
4
4
  log_cli = 1
@@ -0,0 +1,141 @@
1
+ # This file was autogenerated by uv via the following command:
2
+ # uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
3
+ astdoc==1.3.0
4
+ # via mkapi
5
+ babel==2.17.0
6
+ # via mkdocs-material
7
+ backrefs==5.9
8
+ # via mkdocs-material
9
+ bcrypt==4.3.0
10
+ # via paramiko
11
+ certifi==2025.7.14
12
+ # via requests
13
+ cffi==1.17.1
14
+ # via
15
+ # cryptography
16
+ # pynacl
17
+ cfgv==3.4.0
18
+ # via pre-commit
19
+ charset-normalizer==3.4.2
20
+ # via requests
21
+ click==8.2.1
22
+ # via mkdocs
23
+ colorama==0.4.6
24
+ # via mkdocs-material
25
+ coverage==7.9.2
26
+ # via pytest-cov
27
+ cryptography==45.0.5
28
+ # via paramiko
29
+ distlib==0.4.0
30
+ # via virtualenv
31
+ filelock==3.18.0
32
+ # via virtualenv
33
+ ghp-import==2.1.0
34
+ # via mkdocs
35
+ greenlet==3.2.3
36
+ # via sqlalchemy
37
+ identify==2.6.12
38
+ # via pre-commit
39
+ idna==3.10
40
+ # via requests
41
+ iniconfig==2.1.0
42
+ # via pytest
43
+ jinja2==3.1.6
44
+ # via
45
+ # mkapi
46
+ # mkdocs
47
+ # mkdocs-material
48
+ markdown==3.8.2
49
+ # via
50
+ # mkdocs
51
+ # mkdocs-material
52
+ # pymdown-extensions
53
+ markupsafe==3.0.2
54
+ # via
55
+ # jinja2
56
+ # mkdocs
57
+ mergedeep==1.3.4
58
+ # via
59
+ # mkdocs
60
+ # mkdocs-get-deps
61
+ mkapi==4.4.4
62
+ # via hdx-python-database (pyproject.toml)
63
+ mkdocs==1.6.1
64
+ # via
65
+ # mkapi
66
+ # mkdocs-material
67
+ mkdocs-get-deps==0.2.0
68
+ # via mkdocs
69
+ mkdocs-material==9.6.15
70
+ # via mkapi
71
+ mkdocs-material-extensions==1.3.1
72
+ # via mkdocs-material
73
+ nodeenv==1.9.1
74
+ # via pre-commit
75
+ packaging==25.0
76
+ # via
77
+ # mkdocs
78
+ # pytest
79
+ paginate==0.5.7
80
+ # via mkdocs-material
81
+ paramiko==3.5.1
82
+ # via sshtunnel
83
+ pathspec==0.12.1
84
+ # via mkdocs
85
+ platformdirs==4.3.8
86
+ # via
87
+ # mkdocs-get-deps
88
+ # virtualenv
89
+ pluggy==1.6.0
90
+ # via
91
+ # pytest
92
+ # pytest-cov
93
+ pre-commit==4.2.0
94
+ # via hdx-python-database (pyproject.toml)
95
+ psycopg==3.2.9
96
+ # via hdx-python-database (pyproject.toml)
97
+ psycopg-binary==3.2.9
98
+ # via psycopg
99
+ pycparser==2.22
100
+ # via cffi
101
+ pygments==2.19.2
102
+ # via
103
+ # mkdocs-material
104
+ # pytest
105
+ pymdown-extensions==10.16
106
+ # via mkdocs-material
107
+ pynacl==1.5.0
108
+ # via paramiko
109
+ pytest==8.4.1
110
+ # via
111
+ # hdx-python-database (pyproject.toml)
112
+ # pytest-cov
113
+ pytest-cov==6.2.1
114
+ # via hdx-python-database (pyproject.toml)
115
+ python-dateutil==2.9.0.post0
116
+ # via ghp-import
117
+ pyyaml==6.0.2
118
+ # via
119
+ # mkdocs
120
+ # mkdocs-get-deps
121
+ # pre-commit
122
+ # pymdown-extensions
123
+ # pyyaml-env-tag
124
+ pyyaml-env-tag==1.1
125
+ # via mkdocs
126
+ requests==2.32.4
127
+ # via mkdocs-material
128
+ six==1.17.0
129
+ # via python-dateutil
130
+ sqlalchemy==2.0.41
131
+ # via hdx-python-database (pyproject.toml)
132
+ sshtunnel==0.4.0
133
+ # via hdx-python-database (pyproject.toml)
134
+ typing-extensions==4.14.1
135
+ # via sqlalchemy
136
+ urllib3==2.5.0
137
+ # via requests
138
+ virtualenv==20.32.0
139
+ # via pre-commit
140
+ watchdog==6.0.0
141
+ # via mkdocs
@@ -1,4 +1,3 @@
1
- line-length = 79
2
1
  exclude = ["_version.py"]
3
2
 
4
3
  [lint]
@@ -0,0 +1,3 @@
1
+ from ._version import version as __version__ # noqa: F401
2
+ from .database import Database as Database
3
+ from .database import DatabaseError as DatabaseError
@@ -1,8 +1,13 @@
1
- # file generated by setuptools_scm
1
+ # file generated by setuptools-scm
2
2
  # don't change, don't track in version control
3
+
4
+ __all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
5
+
3
6
  TYPE_CHECKING = False
4
7
  if TYPE_CHECKING:
5
- from typing import Tuple, Union
8
+ from typing import Tuple
9
+ from typing import Union
10
+
6
11
  VERSION_TUPLE = Tuple[Union[int, str], ...]
7
12
  else:
8
13
  VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
12
17
  __version_tuple__: VERSION_TUPLE
13
18
  version_tuple: VERSION_TUPLE
14
19
 
15
- __version__ = version = '1.3.5'
16
- __version_tuple__ = version_tuple = (1, 3, 5)
20
+ __version__ = version = '1.3.7'
21
+ __version_tuple__ = version_tuple = (1, 3, 7)
@@ -3,14 +3,13 @@
3
3
  import logging
4
4
  from typing import Any, Dict, List, Optional, Tuple, Type, Union
5
5
 
6
- from sqlalchemy import Engine, TableClause, create_engine
6
+ from sqlalchemy import Engine, TableClause, create_engine, insert
7
7
  from sqlalchemy.exc import SQLAlchemyError
8
8
  from sqlalchemy.ext.automap import automap_base
9
9
  from sqlalchemy.orm import DeclarativeBase, Session
10
10
  from sqlalchemy.pool import NullPool
11
11
  from sqlalchemy.sql.ddl import CreateSchema, DropSchema
12
12
 
13
- from ._version import version as __version__ # noqa: F401
14
13
  from .dburi import get_connection_uri
15
14
  from .no_timezone import Base as NoTZBase
16
15
  from .postgresql import restore_from_pgfile, wait_for_postgresql
@@ -123,16 +122,16 @@ class Database:
123
122
  del kwargs["ssh_port"]
124
123
  else:
125
124
  ssh_port = 22
126
- self.server = sshtunnel.SSHTunnelForwarder(
125
+ self._server = sshtunnel.SSHTunnelForwarder(
127
126
  (ssh_host, ssh_port),
128
127
  remote_bind_address=(host, port),
129
128
  **kwargs,
130
129
  )
131
- self.server.start()
132
- host = self.server.local_bind_host
133
- port = self.server.local_bind_port
130
+ self._server.start()
131
+ host = self._server.local_bind_host
132
+ port = self._server.local_bind_port
134
133
  else:
135
- self.server = None
134
+ self._server = None
136
135
  if not engine and not db_uri:
137
136
  db_uri = get_connection_uri(
138
137
  database,
@@ -158,19 +157,19 @@ class Database:
158
157
  table_base = NoTZBase
159
158
  if not engine:
160
159
  engine = create_engine(db_uri, poolclass=NullPool, echo=False)
161
- self.engine: Engine = engine
160
+ self._engine: Engine = engine
162
161
  if recreate_schema:
163
162
  self.recreate_schema(engine, schema_name)
164
- self.prepare_results = prepare_fn()
165
- self.session, self.base = self.create_session(
163
+ self._prepare_results = prepare_fn()
164
+ self._session, self._base = self.create_session(
166
165
  engine,
167
166
  table_base=table_base,
168
167
  reflect=reflect,
169
168
  )
170
169
  if reflect:
171
- self.reflected_classes = self.base.classes
170
+ self._reflected_classes = self._base.classes
172
171
  else:
173
- self.reflected_classes = None
172
+ self._reflected_classes = None
174
173
 
175
174
  def cleanup(self) -> None:
176
175
  """Cleanup SQLAlchemy.
@@ -178,10 +177,10 @@ class Database:
178
177
  Returns:
179
178
  sqlalchemy.Engine: SQLAlchemy engine
180
179
  """
181
- self.session.close()
182
- self.engine.dispose()
183
- if self.server is not None:
184
- self.server.stop()
180
+ self._session.close()
181
+ self._engine.dispose()
182
+ if self._server is not None:
183
+ self._server.stop()
185
184
 
186
185
  def __enter__(self) -> "Database":
187
186
  return self
@@ -195,7 +194,7 @@ class Database:
195
194
  Returns:
196
195
  None
197
196
  """
198
- self.base.metadata.drop_all(self.engine)
197
+ self._base.metadata.drop_all(self._engine)
199
198
 
200
199
  def get_engine(self) -> Engine:
201
200
  """Returns SQLAlchemy engine.
@@ -203,7 +202,7 @@ class Database:
203
202
  Returns:
204
203
  sqlalchemy.Engine: SQLAlchemy engine
205
204
  """
206
- return self.engine
205
+ return self._engine
207
206
 
208
207
  def get_session(self) -> Session:
209
208
  """Returns SQLAlchemy session.
@@ -211,7 +210,7 @@ class Database:
211
210
  Returns:
212
211
  sqlalchemy.orm.Session: SQLAlchemy session
213
212
  """
214
- return self.session
213
+ return self._session
215
214
 
216
215
  def get_reflected_classes(self) -> Any:
217
216
  """Gets reflected classes.
@@ -219,7 +218,7 @@ class Database:
219
218
  Returns:
220
219
  Any: Reflected classes
221
220
  """
222
- return self.reflected_classes
221
+ return self._reflected_classes
223
222
 
224
223
  def get_prepare_results(self) -> Any:
225
224
  """Returns results from prepare function.
@@ -227,7 +226,31 @@ class Database:
227
226
  Returns:
228
227
  Any: Results from prepare function
229
228
  """
230
- return self.prepare_results
229
+ return self._prepare_results
230
+
231
+ def batch_populate(
232
+ self,
233
+ rows: List[Dict],
234
+ dbtable: Type[DeclarativeBase],
235
+ batch_size: int = 1000,
236
+ ) -> None:
237
+ """Batch populate database table.
238
+
239
+ Args:
240
+ rows (List[Dict]): List of rows
241
+ dbtable (Type[DeclarativeBase]): Database table
242
+ batch_size (int): Batch size. Defaults to 1000.
243
+
244
+ Returns:
245
+ Tuple[Session, Any]: (SQLAlchemy session, base)
246
+ """
247
+ batches = range(len(rows) // batch_size + 1)
248
+ for batch in batches:
249
+ start_row = batch * batch_size
250
+ end_row = start_row + batch_size
251
+ batch_rows = rows[start_row:end_row]
252
+ self._session.execute(insert(dbtable), batch_rows)
253
+ self._session.commit()
231
254
 
232
255
  @staticmethod
233
256
  def create_session(
@@ -258,17 +281,15 @@ class Database:
258
281
  engine = create_engine(db_uri, poolclass=NullPool, echo=False)
259
282
  if reflect:
260
283
  Base = automap_base(declarative_base=table_base)
261
- Base.prepare(
262
- autoload_with=engine, reflection_options={"views": True}
263
- )
284
+ Base.prepare(autoload_with=engine, reflection_options={"views": True})
264
285
  table_base = Base
265
286
  else:
266
287
  table_base.metadata.create_all(engine)
267
288
  return Session(engine), table_base
268
289
 
269
290
  @staticmethod
270
- def recreate_schema(engine: Engine, schema_name: str = "public") -> bool:
271
- """Wipe and create empty schema in database using SQLAlchemy.
291
+ def drop_schema(engine: Engine, schema_name: str = "public") -> bool:
292
+ """Wipe schema in database using SQLAlchemy.
272
293
 
273
294
  Args:
274
295
  engine (Engine): SQLAlchemy engine to use.
@@ -284,10 +305,30 @@ class Database:
284
305
  DropSchema(schema_name, cascade=True, if_exists=True)
285
306
  )
286
307
  connection.commit()
308
+ return True
309
+ except SQLAlchemyError:
310
+ return False
311
+
312
+ @staticmethod
313
+ def recreate_schema(engine: Engine, schema_name: str = "public") -> bool:
314
+ """Wipe and create empty schema in database using SQLAlchemy.
315
+
316
+ Args:
317
+ engine (Engine): SQLAlchemy engine to use.
318
+ schema_name (str): Schema name. Defaults to "public".
319
+
320
+ Returns:
321
+ bool: True if all successful, False if not
322
+ """
323
+ # Wipe and create an empty schema
324
+ try:
325
+ with engine.connect() as connection:
287
326
  connection.execute(
288
- CreateSchema(schema_name, if_not_exists=True)
327
+ DropSchema(schema_name, cascade=True, if_exists=True)
289
328
  )
290
329
  connection.commit()
330
+ connection.execute(CreateSchema(schema_name, if_not_exists=True))
331
+ connection.commit()
291
332
  return True
292
333
  except SQLAlchemyError:
293
334
  return False
@@ -59,6 +59,12 @@ def restore_from_pgfile(db_uri: str, pg_restore_file: str) -> str:
59
59
  str: Output from the pg_restore command
60
60
  """
61
61
  db_params = get_params_from_connection_uri(db_uri)
62
+ if pg_restore_file[-3:] == ".xz":
63
+ decompress = subprocess.Popen(
64
+ ("unxz", "-c", "-d", pg_restore_file), stdout=subprocess.PIPE
65
+ ).stdout
66
+ else:
67
+ decompress = None
62
68
  subprocess_params = ["pg_restore", "-c"]
63
69
  for key, value in db_params.items():
64
70
  match key:
@@ -74,13 +80,18 @@ def restore_from_pgfile(db_uri: str, pg_restore_file: str) -> str:
74
80
  continue
75
81
  subprocess_params.append(f"{value}")
76
82
 
77
- subprocess_params.append(f"{pg_restore_file}")
83
+ if not decompress:
84
+ subprocess_params.append(pg_restore_file)
78
85
  env = environ.copy()
79
86
  password = db_params.get("password")
80
87
  if password:
81
88
  env["PGPASSWORD"] = password
82
89
  process = subprocess.run(
83
- subprocess_params, env=env, capture_output=True, encoding="utf-8"
90
+ subprocess_params,
91
+ env=env,
92
+ capture_output=True,
93
+ encoding="utf-8",
94
+ stdin=decompress,
84
95
  )
85
96
  try:
86
97
  process.check_returncode()
@@ -13,7 +13,5 @@ def camel_to_snake_case(string: str) -> str:
13
13
  Returns:
14
14
  str: String in snake case
15
15
  """
16
- string = re.sub(
17
- r"((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))", r"_\1", string
18
- )
16
+ string = re.sub(r"((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))", r"_\1", string)
19
17
  return string.lower().lstrip("_")
@@ -31,10 +31,9 @@ class TestDatabase:
31
31
  ) as dbdatabase:
32
32
  dbsession = dbdatabase.get_session()
33
33
  assert str(dbsession.bind.engine.url) == f"sqlite:///{dbpath}"
34
- assert dbdatabase.base == NoTZBase
34
+ assert dbdatabase._base == NoTZBase
35
35
  now = datetime(2022, 10, 20, 22, 35, 55, tzinfo=timezone.utc)
36
- input_dbtestdate = DBTestDate()
37
- input_dbtestdate.test_date = now
36
+ input_dbtestdate = DBTestDate(test_date=now)
38
37
  dbsession.add(input_dbtestdate)
39
38
  dbsession.commit()
40
39
  dbtestdate = dbsession.execute(select(DBTestDate)).scalar_one()
@@ -42,6 +41,13 @@ class TestDatabase:
42
41
  date_view = dbdatabase.get_prepare_results()[0]
43
42
  dbtestdate = dbsession.execute(select(date_view)).scalar_one()
44
43
  assert dbtestdate == now
44
+
45
+ now = datetime(2023, 10, 20, 22, 35, 55, tzinfo=timezone.utc)
46
+ rows = [{"test_date": now}]
47
+ dbdatabase.batch_populate(rows, DBTestDate)
48
+ dbtestdate = dbsession.execute(select(DBTestDate)).all()[1][0].test_date
49
+ assert dbtestdate == now
50
+
45
51
  remove(dbpath)
46
52
 
47
53
  def test_errors(self):
@@ -17,9 +17,7 @@ class TestDBURI:
17
17
  "dialect": "postgresql",
18
18
  "driver": "psycopg",
19
19
  }
20
- connection_uri_pg = (
21
- "postgresql+psycopg://myuser:mypass@myserver:1234/mydatabase"
22
- )
20
+ connection_uri_pg = "postgresql+psycopg://myuser:mypass@myserver:1234/mydatabase"
23
21
  params_pg_no_driver = {
24
22
  "database": "mydatabase",
25
23
  "host": "myserver",
@@ -37,9 +35,7 @@ class TestDBURI:
37
35
  "dialect": "postgresql",
38
36
  "driver": None,
39
37
  }
40
- connection_uri_pg_no_driver = (
41
- "postgresql://myuser:mypass@myserver:1234/mydatabase"
42
- )
38
+ connection_uri_pg_no_driver = "postgresql://myuser:mypass@myserver:1234/mydatabase"
43
39
  params_sq_no_driver = {
44
40
  "database": "mydatabase",
45
41
  "host": "myserver",
@@ -67,9 +63,7 @@ class TestDBURI:
67
63
  include_driver=False,
68
64
  )
69
65
  assert result == TestDBURI.params_pg_no_driver
70
- result = get_params_from_connection_uri(
71
- TestDBURI.connection_uri_pg_no_driver
72
- )
66
+ result = get_params_from_connection_uri(TestDBURI.connection_uri_pg_no_driver)
73
67
  assert result == TestDBURI.params_pg_driver_none
74
68
  result = get_params_from_connection_uri(TestDBURI.connection_uri_sq)
75
69
  assert result == TestDBURI.params_sq_driver_none
@@ -77,9 +71,7 @@ class TestDBURI:
77
71
  def test_get_connection_uri(self):
78
72
  result = get_connection_uri(**TestDBURI.params_pg)
79
73
  assert result == TestDBURI.connection_uri_pg
80
- result = get_connection_uri(
81
- **TestDBURI.params_pg, include_driver=False
82
- )
74
+ result = get_connection_uri(**TestDBURI.params_pg, include_driver=False)
83
75
  assert result == TestDBURI.connection_uri_pg_no_driver
84
76
  result = get_connection_uri(**TestDBURI.params_pg_no_driver)
85
77
  assert result == TestDBURI.connection_uri_pg
@@ -1,6 +1,13 @@
1
1
  from hdx.database import Database
2
2
 
3
3
 
4
+ def test_drop_schema(mock_engine):
5
+ db_uri = "postgresql+psycopg://myuser:mypass@0.0.0.0:12345/mydatabase"
6
+ assert Database.drop_schema(mock_engine, db_uri) is True
7
+ db_uri = "Error"
8
+ assert Database.drop_schema(mock_engine, db_uri) is False
9
+
10
+
4
11
  def test_recreate_schema(mock_engine):
5
12
  db_uri = "postgresql+psycopg://myuser:mypass@0.0.0.0:12345/mydatabase"
6
13
  assert Database.recreate_schema(mock_engine, db_uri) is True
@@ -27,7 +27,7 @@ class TestReflect:
27
27
  dburi = f"sqlite:///{dbpath}"
28
28
  assert str(dbengine.url) == dburi
29
29
  dbsession = dbdatabase.get_session()
30
- assert isinstance(dbdatabase.base, DeclarativeAttributeIntercept)
30
+ assert isinstance(dbdatabase._base, DeclarativeAttributeIntercept)
31
31
  assert str(dbsession.bind.engine.url) == dburi
32
32
  Table1 = dbdatabase.get_reflected_classes().table1
33
33
  row = dbsession.execute(select(Table1)).scalar_one()
@@ -33,7 +33,7 @@ class TestSSH:
33
33
  str(dbsession.bind.engine.url)
34
34
  == "postgresql+psycopg://myuser@0.0.0.0:12345/mydatabase"
35
35
  )
36
- assert dbdatabase.base == TZBase
36
+ assert dbdatabase._base == TZBase
37
37
 
38
38
  with Database(
39
39
  ssh_host="mysshhost", ssh_port=25, table_base=TZBase, **params
@@ -43,7 +43,7 @@ class TestSSH:
43
43
  str(dbsession.bind.engine.url)
44
44
  == "postgresql+psycopg://myuser@0.0.0.0:12345/mydatabase"
45
45
  )
46
- assert dbdatabase.base == TZBase
46
+ assert dbdatabase._base == TZBase
47
47
 
48
48
  with Database(ssh_host="mysshhost", **self.params_pg) as dbdatabase:
49
49
  dbsession = dbdatabase.get_session()
@@ -51,14 +51,12 @@ class TestSSH:
51
51
  str(dbsession.bind.engine.url)
52
52
  == "postgresql+psycopg://myuser:***@0.0.0.0:12345/mydatabase"
53
53
  )
54
- assert dbdatabase.base == NoTZBase
54
+ assert dbdatabase._base == NoTZBase
55
55
 
56
- with Database(
57
- ssh_host="mysshhost", ssh_port=25, **params
58
- ) as dbdatabase:
56
+ with Database(ssh_host="mysshhost", ssh_port=25, **params) as dbdatabase:
59
57
  dbsession = dbdatabase.get_session()
60
58
  assert (
61
59
  str(dbsession.bind.engine.url)
62
60
  == "postgresql+psycopg://myuser@0.0.0.0:12345/mydatabase"
63
61
  )
64
- assert dbdatabase.base == NoTZBase
62
+ assert dbdatabase._base == NoTZBase
@@ -1,16 +0,0 @@
1
- version: 2
2
-
3
- mkdocs:
4
- configuration: docs/mkdocs.yml
5
- fail_on_warning: false
6
-
7
- build:
8
- os: ubuntu-22.04
9
- tools:
10
- python: "3.11"
11
- jobs:
12
- pre_build:
13
- - pip install --upgrade mkdocs
14
- - pip install mkdocs-material
15
- - pip install pydoc-markdown
16
- - pydoc-markdown documentation/pydoc-markdown.yaml
@@ -1,29 +0,0 @@
1
- loaders:
2
- - type: python
3
- search_path:
4
- - ../src
5
- packages:
6
- - hdx.database
7
- renderer:
8
- type: mkdocs
9
- output_directory: docs
10
- mkdocs_config:
11
- site_name: HDX Python Database
12
- theme: material
13
- repo_url: "https://github.com/OCHA-DAP/hdx-python-database"
14
- markdown:
15
- source_linker:
16
- type: github
17
- repo: OCHA-DAP/hdx-python-database
18
- pages:
19
- - title: Home
20
- name: index
21
- source: main.md
22
- - title: API Documentation
23
- children:
24
- - title: Database
25
- contents:
26
- - hdx.database.Database.*
27
- - title: PostgreSQL specific
28
- contents:
29
- - hdx.database.postgresql.*
@@ -1,62 +0,0 @@
1
- # This file was autogenerated by uv via the following command:
2
- # uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
3
- bcrypt==4.2.1
4
- # via paramiko
5
- cffi==1.17.1
6
- # via
7
- # cryptography
8
- # pynacl
9
- cfgv==3.4.0
10
- # via pre-commit
11
- coverage==7.6.11
12
- # via pytest-cov
13
- cryptography==44.0.0
14
- # via paramiko
15
- distlib==0.3.9
16
- # via virtualenv
17
- filelock==3.17.0
18
- # via virtualenv
19
- greenlet==3.1.1
20
- # via sqlalchemy
21
- identify==2.6.7
22
- # via pre-commit
23
- iniconfig==2.0.0
24
- # via pytest
25
- nodeenv==1.9.1
26
- # via pre-commit
27
- packaging==24.2
28
- # via pytest
29
- paramiko==3.5.1
30
- # via sshtunnel
31
- platformdirs==4.3.6
32
- # via virtualenv
33
- pluggy==1.5.0
34
- # via pytest
35
- pre-commit==4.1.0
36
- # via hdx-python-database (pyproject.toml)
37
- psycopg==3.2.4
38
- # via hdx-python-database (pyproject.toml)
39
- psycopg-binary==3.2.4
40
- # via psycopg
41
- pycparser==2.22
42
- # via cffi
43
- pynacl==1.5.0
44
- # via paramiko
45
- pytest==8.3.4
46
- # via
47
- # hdx-python-database (pyproject.toml)
48
- # pytest-cov
49
- pytest-cov==6.0.0
50
- # via hdx-python-database (pyproject.toml)
51
- pyyaml==6.0.2
52
- # via pre-commit
53
- sqlalchemy==2.0.38
54
- # via hdx-python-database (pyproject.toml)
55
- sshtunnel==0.4.0
56
- # via hdx-python-database (pyproject.toml)
57
- typing-extensions==4.12.2
58
- # via
59
- # psycopg
60
- # sqlalchemy
61
- virtualenv==20.29.1
62
- # via pre-commit