bcdata 0.17.0__tar.gz → 0.17.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bcdata-0.17.2/.github/workflows/release.yml +100 -0
- bcdata-0.17.2/.github/workflows/tests.yml +73 -0
- bcdata-0.17.2/.gitignore +66 -0
- bcdata-0.17.2/.pre-commit-config.yaml +12 -0
- bcdata-0.17.2/CODE_OF_CONDUCT.md +74 -0
- bcdata-0.17.2/Manifest.in +5 -0
- {bcdata-0.17.0 → bcdata-0.17.2}/PKG-INFO +8 -5
- {bcdata-0.17.0 → bcdata-0.17.2}/README.md +6 -4
- bcdata-0.17.2/data/README.md +9 -0
- bcdata-0.17.2/data/primary_keys.json +56 -0
- bcdata-0.17.2/data/validate.py +30 -0
- bcdata-0.17.2/docker-compose.yml +15 -0
- bcdata-0.17.2/ogr2ogr.md +58 -0
- {bcdata-0.17.0 → bcdata-0.17.2}/pyproject.toml +4 -3
- bcdata-0.17.2/release_checklist.md +13 -0
- bcdata-0.17.2/scripts/document_cli.sh +39 -0
- {bcdata-0.17.0 → bcdata-0.17.2}/src/bcdata/__init__.py +1 -2
- bcdata-0.17.2/src/bcdata/_version.py +24 -0
- {bcdata-0.17.0 → bcdata-0.17.2}/src/bcdata/bc2pg.py +2 -2
- {bcdata-0.17.0 → bcdata-0.17.2}/src/bcdata/cli.py +1 -1
- {bcdata-0.17.0 → bcdata-0.17.2}/src/bcdata/wfs.py +9 -2
- {bcdata-0.17.0 → bcdata-0.17.2}/src/bcdata.egg-info/PKG-INFO +8 -5
- {bcdata-0.17.0 → bcdata-0.17.2}/src/bcdata.egg-info/SOURCES.txt +18 -0
- {bcdata-0.17.0 → bcdata-0.17.2}/src/bcdata.egg-info/requires.txt +1 -0
- bcdata-0.17.2/src/bcdata.egg-info/scm_file_list.json +32 -0
- bcdata-0.17.2/src/bcdata.egg-info/scm_version.json +8 -0
- bcdata-0.17.2/tests/conftest.py +20 -0
- {bcdata-0.17.0 → bcdata-0.17.2}/tests/test_bc2pg.py +10 -2
- bcdata-0.17.2/uv.lock +1606 -0
- {bcdata-0.17.0 → bcdata-0.17.2}/LICENSE.txt +0 -0
- {bcdata-0.17.0 → bcdata-0.17.2}/setup.cfg +0 -0
- {bcdata-0.17.0 → bcdata-0.17.2}/src/bcdata/bcdc.py +0 -0
- {bcdata-0.17.0 → bcdata-0.17.2}/src/bcdata/database.py +0 -0
- {bcdata-0.17.0 → bcdata-0.17.2}/src/bcdata.egg-info/dependency_links.txt +0 -0
- {bcdata-0.17.0 → bcdata-0.17.2}/src/bcdata.egg-info/entry_points.txt +0 -0
- {bcdata-0.17.0 → bcdata-0.17.2}/src/bcdata.egg-info/top_level.txt +0 -0
- {bcdata-0.17.0 → bcdata-0.17.2}/tests/test_bcdc.py +0 -0
- {bcdata-0.17.0 → bcdata-0.17.2}/tests/test_cli.py +0 -0
- {bcdata-0.17.0 → bcdata-0.17.2}/tests/test_wfs.py +0 -0
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
name: Publish to PyPI and create release on GitHub
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- 'v*'
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
build:
|
|
10
|
+
name: Build distribution 📦
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
|
|
13
|
+
permissions:
|
|
14
|
+
contents: read
|
|
15
|
+
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v7
|
|
18
|
+
with:
|
|
19
|
+
fetch-depth: 0 # setuptools_scm needs full history/tags to derive the version
|
|
20
|
+
- name: Install uv
|
|
21
|
+
uses: astral-sh/setup-uv@v7
|
|
22
|
+
- name: Build a binary wheel and a source tarball
|
|
23
|
+
run: uv build
|
|
24
|
+
- name: Store the distribution packages
|
|
25
|
+
uses: actions/upload-artifact@v7
|
|
26
|
+
with:
|
|
27
|
+
name: python-package-distributions
|
|
28
|
+
path: dist/
|
|
29
|
+
|
|
30
|
+
publish-to-pypi:
|
|
31
|
+
name: >-
|
|
32
|
+
Publish Python 🐍 distribution 📦 to PyPI
|
|
33
|
+
needs:
|
|
34
|
+
- build
|
|
35
|
+
runs-on: ubuntu-latest
|
|
36
|
+
environment:
|
|
37
|
+
name: pypi
|
|
38
|
+
url: https://pypi.org/p/bcdata
|
|
39
|
+
permissions:
|
|
40
|
+
contents: read
|
|
41
|
+
id-token: write # IMPORTANT: mandatory for trusted publishing
|
|
42
|
+
|
|
43
|
+
steps:
|
|
44
|
+
- name: Download all the dists
|
|
45
|
+
uses: actions/download-artifact@v8
|
|
46
|
+
with:
|
|
47
|
+
name: python-package-distributions
|
|
48
|
+
path: dist/
|
|
49
|
+
- name: Publish distribution 📦 to PyPI
|
|
50
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
51
|
+
|
|
52
|
+
github-release:
|
|
53
|
+
name: >-
|
|
54
|
+
Sign the Python 🐍 distribution 📦 with Sigstore
|
|
55
|
+
and upload them to GitHub Release
|
|
56
|
+
needs:
|
|
57
|
+
- publish-to-pypi
|
|
58
|
+
runs-on: ubuntu-latest
|
|
59
|
+
|
|
60
|
+
permissions:
|
|
61
|
+
contents: write # IMPORTANT: mandatory for making GitHub Releases
|
|
62
|
+
id-token: write # IMPORTANT: mandatory for sigstore
|
|
63
|
+
|
|
64
|
+
steps:
|
|
65
|
+
- name: Download all the dists
|
|
66
|
+
uses: actions/download-artifact@v8
|
|
67
|
+
with:
|
|
68
|
+
name: python-package-distributions
|
|
69
|
+
path: dist/
|
|
70
|
+
- name: Create GitHub Release
|
|
71
|
+
env:
|
|
72
|
+
GITHUB_TOKEN: ${{ github.token }}
|
|
73
|
+
run: >-
|
|
74
|
+
gh release create
|
|
75
|
+
'${{ github.ref_name }}'
|
|
76
|
+
--repo '${{ github.repository }}'
|
|
77
|
+
--generate-notes
|
|
78
|
+
# Signing is best-effort: it depends on Sigstore's public TUF trust-root
|
|
79
|
+
# infrastructure, which can fail for reasons unrelated to this release
|
|
80
|
+
# (eg tuf.api.exceptions.UnsignedMetadataError). Don't let that block
|
|
81
|
+
# the release itself.
|
|
82
|
+
- name: Sign the dists with Sigstore
|
|
83
|
+
id: sign
|
|
84
|
+
continue-on-error: true
|
|
85
|
+
uses: sigstore/gh-action-sigstore-python@v3.5.0
|
|
86
|
+
with:
|
|
87
|
+
inputs: >-
|
|
88
|
+
./dist/*.tar.gz
|
|
89
|
+
./dist/*.whl
|
|
90
|
+
- name: Upload artifact signatures to GitHub Release
|
|
91
|
+
if: steps.sign.outcome == 'success'
|
|
92
|
+
env:
|
|
93
|
+
GITHUB_TOKEN: ${{ github.token }}
|
|
94
|
+
# Upload to GitHub Release using the `gh` CLI.
|
|
95
|
+
# `dist/` contains the built packages, and the
|
|
96
|
+
# sigstore-produced signatures and certificates.
|
|
97
|
+
run: >-
|
|
98
|
+
gh release upload
|
|
99
|
+
'${{ github.ref_name }}' dist/**
|
|
100
|
+
--repo '${{ github.repository }}'
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
name: Tests
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_dispatch:
|
|
5
|
+
push:
|
|
6
|
+
branches: [ main]
|
|
7
|
+
paths:
|
|
8
|
+
- '.github/workflows/tests.yml'
|
|
9
|
+
- 'uv.lock'
|
|
10
|
+
- 'MANIFEST.in'
|
|
11
|
+
- 'pyproject.toml'
|
|
12
|
+
- 'src/bcdata/**'
|
|
13
|
+
- 'tests/**'
|
|
14
|
+
pull_request:
|
|
15
|
+
branches: [ main ]
|
|
16
|
+
paths:
|
|
17
|
+
- '.github/workflows/tests.yml'
|
|
18
|
+
- 'uv.lock'
|
|
19
|
+
- 'MANIFEST.in'
|
|
20
|
+
- 'pyproject.toml'
|
|
21
|
+
- 'src/bcdata/**'
|
|
22
|
+
- 'tests/**'
|
|
23
|
+
|
|
24
|
+
permissions:
|
|
25
|
+
contents: read
|
|
26
|
+
|
|
27
|
+
jobs:
|
|
28
|
+
lint:
|
|
29
|
+
runs-on: ubuntu-latest
|
|
30
|
+
name: Lint with ruff
|
|
31
|
+
steps:
|
|
32
|
+
- uses: actions/checkout@v7
|
|
33
|
+
- name: Install uv
|
|
34
|
+
uses: astral-sh/setup-uv@v3
|
|
35
|
+
# ruff version pinned here should match the rev in .pre-commit-config.yaml
|
|
36
|
+
- name: Check lint rules
|
|
37
|
+
run: uvx ruff@0.16.3 check .
|
|
38
|
+
- name: Check formatting
|
|
39
|
+
run: uvx ruff@0.16.3 format --check .
|
|
40
|
+
|
|
41
|
+
test:
|
|
42
|
+
runs-on: ubuntu-latest
|
|
43
|
+
name: test
|
|
44
|
+
services:
|
|
45
|
+
postgres:
|
|
46
|
+
image: postgis/postgis:18-3.6
|
|
47
|
+
env:
|
|
48
|
+
POSTGRES_PASSWORD: postgres
|
|
49
|
+
ports:
|
|
50
|
+
- 5432:5432
|
|
51
|
+
# Set health checks to wait until postgres has started
|
|
52
|
+
options: >-
|
|
53
|
+
--health-cmd pg_isready
|
|
54
|
+
--health-interval 10s
|
|
55
|
+
--health-timeout 5s
|
|
56
|
+
--health-retries 5
|
|
57
|
+
steps:
|
|
58
|
+
- uses: actions/checkout@v7
|
|
59
|
+
with:
|
|
60
|
+
fetch-depth: 0 # setuptools_scm needs full history/tags to derive the version
|
|
61
|
+
|
|
62
|
+
- name: Install uv
|
|
63
|
+
uses: astral-sh/setup-uv@v3
|
|
64
|
+
with:
|
|
65
|
+
python-version: '3.13'
|
|
66
|
+
|
|
67
|
+
- name: Install dependencies
|
|
68
|
+
run: uv sync --extra test
|
|
69
|
+
|
|
70
|
+
- name: Run tests
|
|
71
|
+
run: uv run pytest -v -rxXs -x --reruns 2 --reruns-delay 15
|
|
72
|
+
env:
|
|
73
|
+
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
|
bcdata-0.17.2/.gitignore
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
|
|
5
|
+
# C extensions
|
|
6
|
+
*.so
|
|
7
|
+
|
|
8
|
+
# Distribution / packaging
|
|
9
|
+
.Python
|
|
10
|
+
env/
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
lib/
|
|
17
|
+
lib64/
|
|
18
|
+
parts/
|
|
19
|
+
sdist/
|
|
20
|
+
var/
|
|
21
|
+
*.egg-info/
|
|
22
|
+
.installed.cfg
|
|
23
|
+
*.egg
|
|
24
|
+
|
|
25
|
+
# PyInstaller
|
|
26
|
+
# Usually these files are written by a python script from a template
|
|
27
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
28
|
+
*.manifest
|
|
29
|
+
*.spec
|
|
30
|
+
|
|
31
|
+
# Installer logs
|
|
32
|
+
pip-log.txt
|
|
33
|
+
pip-delete-this-directory.txt
|
|
34
|
+
|
|
35
|
+
# Unit test / coverage reports
|
|
36
|
+
htmlcov/
|
|
37
|
+
.tox/
|
|
38
|
+
.coverage
|
|
39
|
+
.cache
|
|
40
|
+
nosetests.xml
|
|
41
|
+
coverage.xml
|
|
42
|
+
|
|
43
|
+
# Translations
|
|
44
|
+
*.mo
|
|
45
|
+
*.pot
|
|
46
|
+
|
|
47
|
+
# Django stuff:
|
|
48
|
+
*.log
|
|
49
|
+
|
|
50
|
+
# Sphinx documentation
|
|
51
|
+
docs/_build/
|
|
52
|
+
|
|
53
|
+
# PyBuilder
|
|
54
|
+
target/
|
|
55
|
+
|
|
56
|
+
# OS X
|
|
57
|
+
.DS_Store
|
|
58
|
+
|
|
59
|
+
# uv-managed virtual environment
|
|
60
|
+
.venv/
|
|
61
|
+
|
|
62
|
+
# ignore generated file that gets manually copied into README
|
|
63
|
+
cli.md
|
|
64
|
+
|
|
65
|
+
# generated by setuptools_scm
|
|
66
|
+
src/bcdata/_version.py
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
repos:
|
|
2
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
3
|
+
rev: v0.16.3
|
|
4
|
+
hooks:
|
|
5
|
+
- id: ruff-check
|
|
6
|
+
name: lint with ruff
|
|
7
|
+
args: [--fix]
|
|
8
|
+
- id: ruff-check
|
|
9
|
+
name: sort imports with ruff
|
|
10
|
+
args: [--select, I, --fix]
|
|
11
|
+
- id: ruff-format
|
|
12
|
+
name: format with ruff
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# Contributor Covenant Code of Conduct
|
|
2
|
+
|
|
3
|
+
## Our Pledge
|
|
4
|
+
|
|
5
|
+
In the interest of fostering an open and welcoming environment, we as
|
|
6
|
+
contributors and maintainers pledge to making participation in our project and
|
|
7
|
+
our community a harassment-free experience for everyone, regardless of age, body
|
|
8
|
+
size, disability, ethnicity, gender identity and expression, level of experience,
|
|
9
|
+
nationality, personal appearance, race, religion, or sexual identity and
|
|
10
|
+
orientation.
|
|
11
|
+
|
|
12
|
+
## Our Standards
|
|
13
|
+
|
|
14
|
+
Examples of behavior that contributes to creating a positive environment
|
|
15
|
+
include:
|
|
16
|
+
|
|
17
|
+
* Using welcoming and inclusive language
|
|
18
|
+
* Being respectful of differing viewpoints and experiences
|
|
19
|
+
* Gracefully accepting constructive criticism
|
|
20
|
+
* Focusing on what is best for the community
|
|
21
|
+
* Showing empathy towards other community members
|
|
22
|
+
|
|
23
|
+
Examples of unacceptable behavior by participants include:
|
|
24
|
+
|
|
25
|
+
* The use of sexualized language or imagery and unwelcome sexual attention or
|
|
26
|
+
advances
|
|
27
|
+
* Trolling, insulting/derogatory comments, and personal or political attacks
|
|
28
|
+
* Public or private harassment
|
|
29
|
+
* Publishing others' private information, such as a physical or electronic
|
|
30
|
+
address, without explicit permission
|
|
31
|
+
* Other conduct which could reasonably be considered inappropriate in a
|
|
32
|
+
professional setting
|
|
33
|
+
|
|
34
|
+
## Our Responsibilities
|
|
35
|
+
|
|
36
|
+
Project maintainers are responsible for clarifying the standards of acceptable
|
|
37
|
+
behavior and are expected to take appropriate and fair corrective action in
|
|
38
|
+
response to any instances of unacceptable behavior.
|
|
39
|
+
|
|
40
|
+
Project maintainers have the right and responsibility to remove, edit, or
|
|
41
|
+
reject comments, commits, code, wiki edits, issues, and other contributions
|
|
42
|
+
that are not aligned to this Code of Conduct, or to ban temporarily or
|
|
43
|
+
permanently any contributor for other behaviors that they deem inappropriate,
|
|
44
|
+
threatening, offensive, or harmful.
|
|
45
|
+
|
|
46
|
+
## Scope
|
|
47
|
+
|
|
48
|
+
This Code of Conduct applies both within project spaces and in public spaces
|
|
49
|
+
when an individual is representing the project or its community. Examples of
|
|
50
|
+
representing a project or community include using an official project e-mail
|
|
51
|
+
address, posting via an official social media account, or acting as an appointed
|
|
52
|
+
representative at an online or offline event. Representation of a project may be
|
|
53
|
+
further defined and clarified by project maintainers.
|
|
54
|
+
|
|
55
|
+
## Enforcement
|
|
56
|
+
|
|
57
|
+
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
|
58
|
+
reported by contacting the project team at todd.wilson@gov.bc.ca. All
|
|
59
|
+
complaints will be reviewed and investigated and will result in a response that
|
|
60
|
+
is deemed necessary and appropriate to the circumstances. The project team is
|
|
61
|
+
obligated to maintain confidentiality with regard to the reporter of an incident.
|
|
62
|
+
Further details of specific enforcement policies may be posted separately.
|
|
63
|
+
|
|
64
|
+
Project maintainers who do not follow or enforce the Code of Conduct in good
|
|
65
|
+
faith may face temporary or permanent repercussions as determined by other
|
|
66
|
+
members of the project's leadership.
|
|
67
|
+
|
|
68
|
+
## Attribution
|
|
69
|
+
|
|
70
|
+
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
|
71
|
+
available at [http://contributor-covenant.org/version/1/4][version]
|
|
72
|
+
|
|
73
|
+
[homepage]: http://contributor-covenant.org
|
|
74
|
+
[version]: http://contributor-covenant.org/version/1/4/
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: bcdata
|
|
3
|
-
Version: 0.17.
|
|
3
|
+
Version: 0.17.2
|
|
4
4
|
Summary: Download open data, monitor and report on changes
|
|
5
5
|
Author-email: Simon Norris <snorris@hillcrestgeo.ca>
|
|
6
6
|
Project-URL: Homepage, https://github.com/smnorris/bcdata
|
|
@@ -26,6 +26,7 @@ Requires-Dist: sqlalchemy
|
|
|
26
26
|
Requires-Dist: stamina
|
|
27
27
|
Provides-Extra: test
|
|
28
28
|
Requires-Dist: pytest; extra == "test"
|
|
29
|
+
Requires-Dist: pytest-rerunfailures; extra == "test"
|
|
29
30
|
Requires-Dist: pre-commit; extra == "test"
|
|
30
31
|
Requires-Dist: requests-mock; extra == "test"
|
|
31
32
|
Dynamic: license-file
|
|
@@ -43,8 +44,6 @@ but most sources are available only via WFS - and the syntax to download WFS dat
|
|
|
43
44
|
|
|
44
45
|
This tool attempts to simplify downloads of BC geographic data and smoothly integrate with PostGIS and Python GIS tools like `geopandas` and `fiona`. The tool only accesses geographic data [avialable via WFS](https://bcgov.github.io/data-publication/pages/tips_tricks_webservices.html) - for other BC open data, download the files directly with `requests` / `curl` / `ogr2ogr` / etc (or the [bcdata R package](https://github.com/bcgov/bcdata)).
|
|
45
46
|
|
|
46
|
-
If downloads are failing, consult the [BCGov Data Systems and Services uptime monitor](https://uptime.com/statuspage/bcgov-dss) - this tool depends on BC Open Geospatial Web Services API at [openmaps.gov.bc.ca](https://openmaps.gov.bc.ca/geo/pub/ows?service=WFS&request=Getcapabilities) and the [BC Data Catalogue API](https://catalogue.data.gov.bc.ca/dataset/bc-data-catalogue-api).
|
|
47
|
-
|
|
48
47
|
**Disclaimer**
|
|
49
48
|
*Data are generally licensed as [OGL-BC](http://www2.gov.bc.ca/gov/content/governments/about-the-bc-government/databc/open-data/open-government-license-bc), but it is the user's responsibility to check the licensing for any downloads.*
|
|
50
49
|
|
|
@@ -363,10 +362,14 @@ Clone the repo:
|
|
|
363
362
|
$ git clone git@github.com:bcgov/bcdata_py.git
|
|
364
363
|
$ cd bcdata_py
|
|
365
364
|
|
|
366
|
-
`bc2pg` tests require a PostGIS database, referenced via the `
|
|
365
|
+
`bc2pg` tests require a PostGIS database, referenced via the `BCDATA_TEST_DATABASE_URL` environment variable.
|
|
366
|
+
Start one locally with docker (after editing the `docker-compose.yml` as needed):
|
|
367
367
|
|
|
368
|
+
$ docker compose build # default to building the postgis image since official pre-built images are not available for all platforms
|
|
368
369
|
$ docker compose up -d
|
|
369
|
-
$ export
|
|
370
|
+
$ export BCDATA_TEST_DATABASE_URL=postgresql://postgres:postgres@localhost:8080/postgres
|
|
371
|
+
|
|
372
|
+
|
|
370
373
|
|
|
371
374
|
install `bcdata` in development mode with [`uv`](https://docs.astral.sh/uv/):
|
|
372
375
|
|
|
@@ -11,8 +11,6 @@ but most sources are available only via WFS - and the syntax to download WFS dat
|
|
|
11
11
|
|
|
12
12
|
This tool attempts to simplify downloads of BC geographic data and smoothly integrate with PostGIS and Python GIS tools like `geopandas` and `fiona`. The tool only accesses geographic data [avialable via WFS](https://bcgov.github.io/data-publication/pages/tips_tricks_webservices.html) - for other BC open data, download the files directly with `requests` / `curl` / `ogr2ogr` / etc (or the [bcdata R package](https://github.com/bcgov/bcdata)).
|
|
13
13
|
|
|
14
|
-
If downloads are failing, consult the [BCGov Data Systems and Services uptime monitor](https://uptime.com/statuspage/bcgov-dss) - this tool depends on BC Open Geospatial Web Services API at [openmaps.gov.bc.ca](https://openmaps.gov.bc.ca/geo/pub/ows?service=WFS&request=Getcapabilities) and the [BC Data Catalogue API](https://catalogue.data.gov.bc.ca/dataset/bc-data-catalogue-api).
|
|
15
|
-
|
|
16
14
|
**Disclaimer**
|
|
17
15
|
*Data are generally licensed as [OGL-BC](http://www2.gov.bc.ca/gov/content/governments/about-the-bc-government/databc/open-data/open-government-license-bc), but it is the user's responsibility to check the licensing for any downloads.*
|
|
18
16
|
|
|
@@ -331,10 +329,14 @@ Clone the repo:
|
|
|
331
329
|
$ git clone git@github.com:bcgov/bcdata_py.git
|
|
332
330
|
$ cd bcdata_py
|
|
333
331
|
|
|
334
|
-
`bc2pg` tests require a PostGIS database, referenced via the `
|
|
332
|
+
`bc2pg` tests require a PostGIS database, referenced via the `BCDATA_TEST_DATABASE_URL` environment variable.
|
|
333
|
+
Start one locally with docker (after editing the `docker-compose.yml` as needed):
|
|
335
334
|
|
|
335
|
+
$ docker compose build # default to building the postgis image since official pre-built images are not available for all platforms
|
|
336
336
|
$ docker compose up -d
|
|
337
|
-
$ export
|
|
337
|
+
$ export BCDATA_TEST_DATABASE_URL=postgresql://postgres:postgres@localhost:8080/postgres
|
|
338
|
+
|
|
339
|
+
|
|
338
340
|
|
|
339
341
|
install `bcdata` in development mode with [`uv`](https://docs.astral.sh/uv/):
|
|
340
342
|
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# primary_keys.json
|
|
2
|
+
|
|
3
|
+
Record the bcdata table primary keys in one spot (where available).
|
|
4
|
+
|
|
5
|
+
When finished adding a key/value pair, be sure to validate:
|
|
6
|
+
|
|
7
|
+
python validate.py
|
|
8
|
+
|
|
9
|
+
The validation script iterates through each `table`: `column` key/value pair in `primary_keys.json`, confirming that the table exists and the given column is present within the table. Note that the script does not examine values of the column to confirm uniqueness.
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
{
|
|
2
|
+
"whse_admin_boundaries.clab_indian_reserves": "clab_id",
|
|
3
|
+
"whse_admin_boundaries.clab_national_parks": "national_park_id",
|
|
4
|
+
"whse_admin_boundaries.fadm_designated_areas": "feature_id",
|
|
5
|
+
"whse_admin_boundaries.fadm_special_protection_area": "feature_id",
|
|
6
|
+
"whse_admin_boundaries.fadm_tfl_all_sp": "tfl_all_sysid",
|
|
7
|
+
"whse_basemapping.bcgs_20k_grid": "map_tile",
|
|
8
|
+
"whse_basemapping.dbm_mof_50k_grid": "map_tile",
|
|
9
|
+
"whse_basemapping.gba_local_reg_greenspaces_sp": "local_reg_greenspace_id",
|
|
10
|
+
"whse_basemapping.gba_railway_structure_lines_sp": "railway_structure_line_id",
|
|
11
|
+
"whse_basemapping.gba_railway_tracks_sp": "railway_track_id",
|
|
12
|
+
"whse_basemapping.gba_transmission_lines_sp": "transmission_line_id",
|
|
13
|
+
"whse_basemapping.gns_geographical_names_sp": "geographical_names_id",
|
|
14
|
+
"whse_basemapping.nts_250k_grid": "map_tile",
|
|
15
|
+
"whse_basemapping.trim_cultural_lines": "objectid",
|
|
16
|
+
"whse_basemapping.trim_cultural_points": "objectid",
|
|
17
|
+
"whse_basemapping.trim_ebm_airfields": "objectid",
|
|
18
|
+
"whse_basemapping.trim_ebm_ocean": "objectid",
|
|
19
|
+
"whse_basemapping.utmg_utm_zones_sp": "utm_zone",
|
|
20
|
+
"whse_cadastre.pmbc_parcel_fabric_poly_svw": "parcel_fabric_poly_id",
|
|
21
|
+
"whse_environmental_monitoring.envcan_hydrometric_stn_sp": "hydrometric_station_id",
|
|
22
|
+
"whse_fish.fiss_stream_sample_sites_sp": "stream_sample_site_id",
|
|
23
|
+
"whse_fish.pscis_assessment_svw": "stream_crossing_id",
|
|
24
|
+
"whse_fish.pscis_design_proposal_svw": "stream_crossing_id",
|
|
25
|
+
"whse_fish.pscis_habitat_confirmation_svw": "stream_crossing_id",
|
|
26
|
+
"whse_fish.pscis_remediation_svw": "stream_crossing_id",
|
|
27
|
+
"whse_forest_tenure.ften_managed_licence_poly_svw": "objectid",
|
|
28
|
+
"whse_forest_tenure.ften_range_poly_svw": "objectid",
|
|
29
|
+
"whse_forest_tenure.ften_recreation_poly_svw": "rmf_skey",
|
|
30
|
+
"whse_forest_vegetation.ogsr_priority_def_area_cur_sp": "ogsr_pdac_sysid",
|
|
31
|
+
"whse_forest_vegetation.rec_visual_landscape_inventory": "vli_polygon_no",
|
|
32
|
+
"whse_forest_vegetation.veg_comp_lyr_r1_poly": "feature_id",
|
|
33
|
+
"whse_forest_vegetation.veg_consolidated_cut_blocks_sp": "vccb_sysid",
|
|
34
|
+
"whse_human_cultural_economic.hist_historic_environmnt_pa_sv": "hist_hef_sysid",
|
|
35
|
+
"whse_imagery_and_base_maps.mot_road_structure_sp": "hwy_structure_class_id",
|
|
36
|
+
"whse_land_use_planning.rmp_landscape_rsrv_design_sp": "rmp_lrd_sysid",
|
|
37
|
+
"whse_land_use_planning.rmp_ogma_legal_current_svw": "legal_ogma_internal_id",
|
|
38
|
+
"whse_land_use_planning.rmp_ogma_non_legal_current_svw": "non_legal_ogma_internal_id",
|
|
39
|
+
"whse_land_use_planning.rmp_plan_legal_poly_svw": "legal_feat_id",
|
|
40
|
+
"whse_land_use_planning.rmp_plan_non_legal_poly_svw": "non_legal_feat_id",
|
|
41
|
+
"whse_land_use_planning.rmp_strgc_land_rsrce_plan_svw": "strgc_land_rsrce_plan_id",
|
|
42
|
+
"whse_legal_admin_boundaries.abms_municipalities_sp": "lgl_admin_area_id",
|
|
43
|
+
"whse_legal_admin_boundaries.wcl_conservation_areas_ngo_sp": "conservation_areas_ngo_id",
|
|
44
|
+
"whse_legal_admin_boundaries.wcl_conservation_lands_sp": "conservation_land_id",
|
|
45
|
+
"whse_mineral_tenure.mta_acquired_tenure_svw": "tenure_number_id",
|
|
46
|
+
"whse_mineral_tenure.og_petrlm_dev_rds_pre06_pub_sp": "og_petrlm_dev_rd_pre06_pub_id",
|
|
47
|
+
"whse_mineral_tenure.og_road_segment_permit_sp": "og_road_segment_permit_id",
|
|
48
|
+
"whse_tantalis.ta_conservancy_areas_svw": "admin_area_sid",
|
|
49
|
+
"whse_tantalis.ta_crown_tenures_svw": "objectid",
|
|
50
|
+
"whse_tantalis.ta_park_ecores_pa_svw": "admin_area_sid",
|
|
51
|
+
"whse_tantalis.ta_wildlife_mgmt_areas_svw": "admin_area_sid",
|
|
52
|
+
"whse_water_management.wls_community_ws_pub_svw": "wls_cw_sysid",
|
|
53
|
+
"whse_wildlife_management.wcp_fish_sensitive_ws_poly": "fish_sensitive_ws_poly_id",
|
|
54
|
+
"whse_wildlife_management.wcp_ungulate_winter_range_sp": "ungulate_winter_range_id",
|
|
55
|
+
"whse_wildlife_management.wcp_wildlife_habitat_area_poly": "habitat_area_id"
|
|
56
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
import bcdata
|
|
6
|
+
|
|
7
|
+
LOG_FORMAT = "%(asctime)s:%(levelname)s:%(name)s: %(message)s"
|
|
8
|
+
|
|
9
|
+
with open("primary_keys.json", "r") as file:
|
|
10
|
+
"""validate pk database"""
|
|
11
|
+
logging.basicConfig(stream=sys.stderr, level=20, format=LOG_FORMAT)
|
|
12
|
+
log = logging.getLogger(__name__)
|
|
13
|
+
primary_keys = json.load(file)
|
|
14
|
+
pk_db_tables = set(primary_keys.keys())
|
|
15
|
+
bcdata_tables = {t.lower() for t in bcdata.list_tables()}
|
|
16
|
+
if pk_db_tables.issubset(bcdata_tables):
|
|
17
|
+
log.info("Table names in primary_keys.json are valid")
|
|
18
|
+
for table in primary_keys:
|
|
19
|
+
column = primary_keys[table]
|
|
20
|
+
schema = bcdata.get_table_definition(table)["schema"]
|
|
21
|
+
if column not in [c["column_name"].lower() for c in schema]:
|
|
22
|
+
raise ValueError(f"Column {column} not found in {table}")
|
|
23
|
+
log.info(
|
|
24
|
+
"Validation successful - columns listed in primary_keys.json are present in listed tables"
|
|
25
|
+
)
|
|
26
|
+
else:
|
|
27
|
+
invalid_keys = list(pk_db_tables - bcdata_tables)
|
|
28
|
+
for k in invalid_keys:
|
|
29
|
+
log.error(f"{k}")
|
|
30
|
+
raise ValueError("Invalid table name(s)")
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
services:
|
|
2
|
+
postgres:
|
|
3
|
+
image: postgis/postgis:18-3.6-alpine
|
|
4
|
+
pull_policy: never
|
|
5
|
+
build:
|
|
6
|
+
context: https://github.com/postgis/docker-postgis.git#master:18-3.6/alpine
|
|
7
|
+
environment:
|
|
8
|
+
POSTGRES_PASSWORD: postgres
|
|
9
|
+
ports:
|
|
10
|
+
- "8080:5432"
|
|
11
|
+
healthcheck:
|
|
12
|
+
test: ["CMD", "pg_isready", "-U", "postgres"]
|
|
13
|
+
interval: 10s
|
|
14
|
+
timeout: 5s
|
|
15
|
+
retries: 5
|
bcdata-0.17.2/ogr2ogr.md
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# WFS via ogr2ogr/curl
|
|
2
|
+
|
|
3
|
+
Making WFS requests and dumping to file/postgres is fairly straightforward with existing command line tools:
|
|
4
|
+
|
|
5
|
+
- list all layers:
|
|
6
|
+
|
|
7
|
+
$ ogrinfo WFS:http://openmaps.gov.bc.ca/geo/ows
|
|
8
|
+
|
|
9
|
+
- define a data request url:
|
|
10
|
+
|
|
11
|
+
$ airports_url="https://openmaps.gov.bc.ca/geo/pub/ows?service=WFS&version=2.0.0&request=GetFeature&typeName=WHSE_IMAGERY_AND_BASE_MAPS.GSR_AIRPORTS_SVW&outputFormat=json&SRSNAME=epsg%3A3005"
|
|
12
|
+
|
|
13
|
+
- describe the data:
|
|
14
|
+
|
|
15
|
+
$ ogrinfo -so $airports_url OGRGeoJSON
|
|
16
|
+
|
|
17
|
+
- making the request and dumping to file can be done different ways:
|
|
18
|
+
|
|
19
|
+
$ ogr2ogr \
|
|
20
|
+
-f GeoJSON \
|
|
21
|
+
airports.geojson \
|
|
22
|
+
$airports_url
|
|
23
|
+
|
|
24
|
+
$ curl $airports_url > airports.geojson
|
|
25
|
+
|
|
26
|
+
- request a bigger dataset - gdal docs say [paging is automatically supported for WFS 2.0.0](https://gdal.org/drivers/vector/wfs.html#paging-with-wfs-2-0):
|
|
27
|
+
|
|
28
|
+
$ uwr="https://openmaps.gov.bc.ca/geo/pub/ows?service=WFS&version=2.0.0&request=GetFeature&typeName=WHSE_WILDLIFE_MANAGEMENT.WCP_UNGULATE_WINTER_RANGE_SP&outputFormat=json&SRSNAME=epsg%3A3005"
|
|
29
|
+
$ curl $uwr > uwr.geojson
|
|
30
|
+
$ ogrinfo uwr.geojson uwr -so
|
|
31
|
+
|
|
32
|
+
INFO: Open of `uwr.geojson'
|
|
33
|
+
using driver `GeoJSON' successful.
|
|
34
|
+
|
|
35
|
+
Layer name: uwr
|
|
36
|
+
Geometry: Unknown (any)
|
|
37
|
+
Feature Count: 10000
|
|
38
|
+
|
|
39
|
+
only 10k features... a client needs to request the total number of features and define the per-page requests
|
|
40
|
+
|
|
41
|
+
- load to postgres:
|
|
42
|
+
|
|
43
|
+
$ ogr2ogr \
|
|
44
|
+
-f PostgreSQL \
|
|
45
|
+
PG:"host=localhost user=postgres dbname=postgis password=postgres" \
|
|
46
|
+
-lco SCHEMA=whse_imagery_and_base_maps \
|
|
47
|
+
-lco GEOMETRY_NAME=geom \
|
|
48
|
+
-nln gsr_airports_svw \
|
|
49
|
+
$airports_url
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
`bcdata` smooths the above processes by:
|
|
53
|
+
|
|
54
|
+
- providing Python and CLI interfaces for typical request options
|
|
55
|
+
- generating paged requests
|
|
56
|
+
- reducing server load by caching getcapabilites responses
|
|
57
|
+
- handling typical minor data issues (eg mixed geometry types)
|
|
58
|
+
- creating postgres tables that mirror the table schema as defined in BCGW (rather than using ogr default types)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
[build-system]
|
|
2
|
-
requires = ["setuptools"]
|
|
2
|
+
requires = ["setuptools>=64", "setuptools_scm>=8"]
|
|
3
3
|
build-backend = "setuptools.build_meta"
|
|
4
4
|
|
|
5
5
|
[project]
|
|
@@ -36,6 +36,7 @@ dependencies = [
|
|
|
36
36
|
[project.optional-dependencies]
|
|
37
37
|
test = [
|
|
38
38
|
"pytest",
|
|
39
|
+
"pytest-rerunfailures",
|
|
39
40
|
"pre-commit",
|
|
40
41
|
"requests-mock"
|
|
41
42
|
]
|
|
@@ -47,8 +48,8 @@ bcdata = "bcdata.cli:cli"
|
|
|
47
48
|
Homepage = "https://github.com/smnorris/bcdata"
|
|
48
49
|
Issues = "https://github.com/smnorris/bcdata"
|
|
49
50
|
|
|
50
|
-
[tool.
|
|
51
|
-
|
|
51
|
+
[tool.setuptools_scm]
|
|
52
|
+
write_to = "src/bcdata/_version.py"
|
|
52
53
|
|
|
53
54
|
[tool.ruff]
|
|
54
55
|
line-length = 100
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Release checklist
|
|
2
|
+
|
|
3
|
+
- run tests (`uv run pytest`)
|
|
4
|
+
- confirm dependencies/classifiers in `pyproject.toml` are current
|
|
5
|
+
- run `uv lock` to update `uv.lock`
|
|
6
|
+
- commit changes on dev branch, create pr, merge pr to main
|
|
7
|
+
- switch to main branch locally, pull latest, create tag & push:
|
|
8
|
+
|
|
9
|
+
git checkout main
|
|
10
|
+
git pull
|
|
11
|
+
git tag -a <version> -m <version> && git push
|
|
12
|
+
|
|
13
|
+
Note that releases are automatically published (to pypi & github) on tag via `.github/workflows/release.yml`
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
rm -rf cli.md
|
|
2
|
+
printf '### CLI' > cli.md
|
|
3
|
+
printf '\n' >> cli.md
|
|
4
|
+
|
|
5
|
+
printf 'Commands available via the bcdata command line interface are documented with the --help option\n' >> cli.md
|
|
6
|
+
printf '\n' >> cli.md
|
|
7
|
+
printf '```\n\n$ bcdata --help\n\n' >> cli.md
|
|
8
|
+
bcdata --help >> cli.md
|
|
9
|
+
printf '```\n' >> cli.md
|
|
10
|
+
|
|
11
|
+
printf '\n' >> cli.md
|
|
12
|
+
printf '#### bc2pg\n\n' >> cli.md
|
|
13
|
+
printf '```\n$ bcdata bc2pg --help\n\n' >> cli.md
|
|
14
|
+
bcdata bc2pg --help >> cli.md
|
|
15
|
+
printf '```\n' >> cli.md
|
|
16
|
+
|
|
17
|
+
printf '\n' >> cli.md
|
|
18
|
+
printf '#### cat\n\n' >> cli.md
|
|
19
|
+
printf '```\n$ bcdata cat --help\n\n' >> cli.md
|
|
20
|
+
bcdata cat --help >> cli.md
|
|
21
|
+
printf '```\n' >> cli.md
|
|
22
|
+
|
|
23
|
+
printf '\n' >> cli.md
|
|
24
|
+
printf '#### dump \n\n' >> cli.md
|
|
25
|
+
printf '```\n$ bcdata dump --help\n\n' >> cli.md
|
|
26
|
+
bcdata dump --help >> cli.md
|
|
27
|
+
printf '```\n' >> cli.md
|
|
28
|
+
|
|
29
|
+
printf '\n' >> cli.md
|
|
30
|
+
printf '#### info\n\n' >> cli.md
|
|
31
|
+
printf '```\n$ bcdata info --help\n\n' >> cli.md
|
|
32
|
+
bcdata info --help >> cli.md
|
|
33
|
+
printf '```\n' >> cli.md
|
|
34
|
+
|
|
35
|
+
printf '\n' >> cli.md
|
|
36
|
+
printf '#### list\n\n' >> cli.md
|
|
37
|
+
printf '```\n$ bcdata list --help\n\n' >> cli.md
|
|
38
|
+
bcdata list --help >> cli.md
|
|
39
|
+
printf '```\n' >> cli.md
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import requests
|
|
2
2
|
|
|
3
|
+
from ._version import __version__ as __version__
|
|
3
4
|
from .bc2pg import bc2pg as bc2pg
|
|
4
5
|
from .bcdc import get_table_definition as get_table_definition
|
|
5
6
|
from .bcdc import get_table_name as get_table_name
|
|
@@ -20,5 +21,3 @@ if response.status_code == 200:
|
|
|
20
21
|
else:
|
|
21
22
|
raise ConnectionError(f"Failed to download primary key database at {PRIMARY_KEY_DB_URL}")
|
|
22
23
|
primary_keys = {}
|
|
23
|
-
|
|
24
|
-
__version__ = "0.17.0"
|