bcdata 0.17.1__tar.gz → 0.17.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. bcdata-0.17.2/.github/workflows/release.yml +100 -0
  2. bcdata-0.17.2/.github/workflows/tests.yml +73 -0
  3. bcdata-0.17.2/.gitignore +66 -0
  4. bcdata-0.17.2/.pre-commit-config.yaml +12 -0
  5. bcdata-0.17.2/CODE_OF_CONDUCT.md +74 -0
  6. bcdata-0.17.2/Manifest.in +5 -0
  7. {bcdata-0.17.1 → bcdata-0.17.2}/PKG-INFO +8 -5
  8. {bcdata-0.17.1 → bcdata-0.17.2}/README.md +6 -4
  9. bcdata-0.17.2/data/README.md +9 -0
  10. bcdata-0.17.2/data/primary_keys.json +56 -0
  11. bcdata-0.17.2/data/validate.py +30 -0
  12. bcdata-0.17.2/docker-compose.yml +15 -0
  13. bcdata-0.17.2/ogr2ogr.md +58 -0
  14. {bcdata-0.17.1 → bcdata-0.17.2}/pyproject.toml +4 -3
  15. bcdata-0.17.2/release_checklist.md +13 -0
  16. bcdata-0.17.2/scripts/document_cli.sh +39 -0
  17. {bcdata-0.17.1 → bcdata-0.17.2}/src/bcdata/__init__.py +1 -2
  18. bcdata-0.17.2/src/bcdata/_version.py +24 -0
  19. {bcdata-0.17.1 → bcdata-0.17.2}/src/bcdata/bc2pg.py +2 -2
  20. {bcdata-0.17.1 → bcdata-0.17.2}/src/bcdata/cli.py +1 -1
  21. {bcdata-0.17.1 → bcdata-0.17.2}/src/bcdata/wfs.py +1 -1
  22. {bcdata-0.17.1 → bcdata-0.17.2}/src/bcdata.egg-info/PKG-INFO +8 -5
  23. {bcdata-0.17.1 → bcdata-0.17.2}/src/bcdata.egg-info/SOURCES.txt +18 -0
  24. {bcdata-0.17.1 → bcdata-0.17.2}/src/bcdata.egg-info/requires.txt +1 -0
  25. bcdata-0.17.2/src/bcdata.egg-info/scm_file_list.json +32 -0
  26. bcdata-0.17.2/src/bcdata.egg-info/scm_version.json +8 -0
  27. bcdata-0.17.2/tests/conftest.py +20 -0
  28. {bcdata-0.17.1 → bcdata-0.17.2}/tests/test_bc2pg.py +10 -2
  29. bcdata-0.17.2/uv.lock +1606 -0
  30. {bcdata-0.17.1 → bcdata-0.17.2}/LICENSE.txt +0 -0
  31. {bcdata-0.17.1 → bcdata-0.17.2}/setup.cfg +0 -0
  32. {bcdata-0.17.1 → bcdata-0.17.2}/src/bcdata/bcdc.py +0 -0
  33. {bcdata-0.17.1 → bcdata-0.17.2}/src/bcdata/database.py +0 -0
  34. {bcdata-0.17.1 → bcdata-0.17.2}/src/bcdata.egg-info/dependency_links.txt +0 -0
  35. {bcdata-0.17.1 → bcdata-0.17.2}/src/bcdata.egg-info/entry_points.txt +0 -0
  36. {bcdata-0.17.1 → bcdata-0.17.2}/src/bcdata.egg-info/top_level.txt +0 -0
  37. {bcdata-0.17.1 → bcdata-0.17.2}/tests/test_bcdc.py +0 -0
  38. {bcdata-0.17.1 → bcdata-0.17.2}/tests/test_cli.py +0 -0
  39. {bcdata-0.17.1 → bcdata-0.17.2}/tests/test_wfs.py +0 -0
@@ -0,0 +1,100 @@
1
+ name: Publish to PyPI and create release on GitHub
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - 'v*'
7
+
8
+ jobs:
9
+ build:
10
+ name: Build distribution 📦
11
+ runs-on: ubuntu-latest
12
+
13
+ permissions:
14
+ contents: read
15
+
16
+ steps:
17
+ - uses: actions/checkout@v7
18
+ with:
19
+ fetch-depth: 0 # setuptools_scm needs full history/tags to derive the version
20
+ - name: Install uv
21
+ uses: astral-sh/setup-uv@v7
22
+ - name: Build a binary wheel and a source tarball
23
+ run: uv build
24
+ - name: Store the distribution packages
25
+ uses: actions/upload-artifact@v7
26
+ with:
27
+ name: python-package-distributions
28
+ path: dist/
29
+
30
+ publish-to-pypi:
31
+ name: >-
32
+ Publish Python 🐍 distribution 📦 to PyPI
33
+ needs:
34
+ - build
35
+ runs-on: ubuntu-latest
36
+ environment:
37
+ name: pypi
38
+ url: https://pypi.org/p/bcdata
39
+ permissions:
40
+ contents: read
41
+ id-token: write # IMPORTANT: mandatory for trusted publishing
42
+
43
+ steps:
44
+ - name: Download all the dists
45
+ uses: actions/download-artifact@v8
46
+ with:
47
+ name: python-package-distributions
48
+ path: dist/
49
+ - name: Publish distribution 📦 to PyPI
50
+ uses: pypa/gh-action-pypi-publish@release/v1
51
+
52
+ github-release:
53
+ name: >-
54
+ Sign the Python 🐍 distribution 📦 with Sigstore
55
+ and upload them to GitHub Release
56
+ needs:
57
+ - publish-to-pypi
58
+ runs-on: ubuntu-latest
59
+
60
+ permissions:
61
+ contents: write # IMPORTANT: mandatory for making GitHub Releases
62
+ id-token: write # IMPORTANT: mandatory for sigstore
63
+
64
+ steps:
65
+ - name: Download all the dists
66
+ uses: actions/download-artifact@v8
67
+ with:
68
+ name: python-package-distributions
69
+ path: dist/
70
+ - name: Create GitHub Release
71
+ env:
72
+ GITHUB_TOKEN: ${{ github.token }}
73
+ run: >-
74
+ gh release create
75
+ '${{ github.ref_name }}'
76
+ --repo '${{ github.repository }}'
77
+ --generate-notes
78
+ # Signing is best-effort: it depends on Sigstore's public TUF trust-root
79
+ # infrastructure, which can fail for reasons unrelated to this release
80
+ # (eg tuf.api.exceptions.UnsignedMetadataError). Don't let that block
81
+ # the release itself.
82
+ - name: Sign the dists with Sigstore
83
+ id: sign
84
+ continue-on-error: true
85
+ uses: sigstore/gh-action-sigstore-python@v3.5.0
86
+ with:
87
+ inputs: >-
88
+ ./dist/*.tar.gz
89
+ ./dist/*.whl
90
+ - name: Upload artifact signatures to GitHub Release
91
+ if: steps.sign.outcome == 'success'
92
+ env:
93
+ GITHUB_TOKEN: ${{ github.token }}
94
+ # Upload to GitHub Release using the `gh` CLI.
95
+ # `dist/` contains the built packages, and the
96
+ # sigstore-produced signatures and certificates.
97
+ run: >-
98
+ gh release upload
99
+ '${{ github.ref_name }}' dist/**
100
+ --repo '${{ github.repository }}'
@@ -0,0 +1,73 @@
1
+ name: Tests
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ push:
6
+ branches: [ main]
7
+ paths:
8
+ - '.github/workflows/tests.yml'
9
+ - 'uv.lock'
10
+ - 'MANIFEST.in'
11
+ - 'pyproject.toml'
12
+ - 'src/bcdata/**'
13
+ - 'tests/**'
14
+ pull_request:
15
+ branches: [ main ]
16
+ paths:
17
+ - '.github/workflows/tests.yml'
18
+ - 'uv.lock'
19
+ - 'MANIFEST.in'
20
+ - 'pyproject.toml'
21
+ - 'src/bcdata/**'
22
+ - 'tests/**'
23
+
24
+ permissions:
25
+ contents: read
26
+
27
+ jobs:
28
+ lint:
29
+ runs-on: ubuntu-latest
30
+ name: Lint with ruff
31
+ steps:
32
+ - uses: actions/checkout@v7
33
+ - name: Install uv
34
+ uses: astral-sh/setup-uv@v3
35
+ # ruff version pinned here should match the rev in .pre-commit-config.yaml
36
+ - name: Check lint rules
37
+ run: uvx ruff@0.16.3 check .
38
+ - name: Check formatting
39
+ run: uvx ruff@0.16.3 format --check .
40
+
41
+ test:
42
+ runs-on: ubuntu-latest
43
+ name: test
44
+ services:
45
+ postgres:
46
+ image: postgis/postgis:18-3.6
47
+ env:
48
+ POSTGRES_PASSWORD: postgres
49
+ ports:
50
+ - 5432:5432
51
+ # Set health checks to wait until postgres has started
52
+ options: >-
53
+ --health-cmd pg_isready
54
+ --health-interval 10s
55
+ --health-timeout 5s
56
+ --health-retries 5
57
+ steps:
58
+ - uses: actions/checkout@v7
59
+ with:
60
+ fetch-depth: 0 # setuptools_scm needs full history/tags to derive the version
61
+
62
+ - name: Install uv
63
+ uses: astral-sh/setup-uv@v3
64
+ with:
65
+ python-version: '3.13'
66
+
67
+ - name: Install dependencies
68
+ run: uv sync --extra test
69
+
70
+ - name: Run tests
71
+ run: uv run pytest -v -rxXs -x --reruns 2 --reruns-delay 15
72
+ env:
73
+ DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
@@ -0,0 +1,66 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+
5
+ # C extensions
6
+ *.so
7
+
8
+ # Distribution / packaging
9
+ .Python
10
+ env/
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ lib/
17
+ lib64/
18
+ parts/
19
+ sdist/
20
+ var/
21
+ *.egg-info/
22
+ .installed.cfg
23
+ *.egg
24
+
25
+ # PyInstaller
26
+ # Usually these files are written by a python script from a template
27
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
28
+ *.manifest
29
+ *.spec
30
+
31
+ # Installer logs
32
+ pip-log.txt
33
+ pip-delete-this-directory.txt
34
+
35
+ # Unit test / coverage reports
36
+ htmlcov/
37
+ .tox/
38
+ .coverage
39
+ .cache
40
+ nosetests.xml
41
+ coverage.xml
42
+
43
+ # Translations
44
+ *.mo
45
+ *.pot
46
+
47
+ # Django stuff:
48
+ *.log
49
+
50
+ # Sphinx documentation
51
+ docs/_build/
52
+
53
+ # PyBuilder
54
+ target/
55
+
56
+ # OS X
57
+ .DS_Store
58
+
59
+ # uv-managed virtual environment
60
+ .venv/
61
+
62
+ # ignore generated file that gets manually copied into README
63
+ cli.md
64
+
65
+ # generated by setuptools_scm
66
+ src/bcdata/_version.py
@@ -0,0 +1,12 @@
1
+ repos:
2
+ - repo: https://github.com/astral-sh/ruff-pre-commit
3
+ rev: v0.16.3
4
+ hooks:
5
+ - id: ruff-check
6
+ name: lint with ruff
7
+ args: [--fix]
8
+ - id: ruff-check
9
+ name: sort imports with ruff
10
+ args: [--select, I, --fix]
11
+ - id: ruff-format
12
+ name: format with ruff
@@ -0,0 +1,74 @@
1
+ # Contributor Covenant Code of Conduct
2
+
3
+ ## Our Pledge
4
+
5
+ In the interest of fostering an open and welcoming environment, we as
6
+ contributors and maintainers pledge to making participation in our project and
7
+ our community a harassment-free experience for everyone, regardless of age, body
8
+ size, disability, ethnicity, gender identity and expression, level of experience,
9
+ nationality, personal appearance, race, religion, or sexual identity and
10
+ orientation.
11
+
12
+ ## Our Standards
13
+
14
+ Examples of behavior that contributes to creating a positive environment
15
+ include:
16
+
17
+ * Using welcoming and inclusive language
18
+ * Being respectful of differing viewpoints and experiences
19
+ * Gracefully accepting constructive criticism
20
+ * Focusing on what is best for the community
21
+ * Showing empathy towards other community members
22
+
23
+ Examples of unacceptable behavior by participants include:
24
+
25
+ * The use of sexualized language or imagery and unwelcome sexual attention or
26
+ advances
27
+ * Trolling, insulting/derogatory comments, and personal or political attacks
28
+ * Public or private harassment
29
+ * Publishing others' private information, such as a physical or electronic
30
+ address, without explicit permission
31
+ * Other conduct which could reasonably be considered inappropriate in a
32
+ professional setting
33
+
34
+ ## Our Responsibilities
35
+
36
+ Project maintainers are responsible for clarifying the standards of acceptable
37
+ behavior and are expected to take appropriate and fair corrective action in
38
+ response to any instances of unacceptable behavior.
39
+
40
+ Project maintainers have the right and responsibility to remove, edit, or
41
+ reject comments, commits, code, wiki edits, issues, and other contributions
42
+ that are not aligned to this Code of Conduct, or to ban temporarily or
43
+ permanently any contributor for other behaviors that they deem inappropriate,
44
+ threatening, offensive, or harmful.
45
+
46
+ ## Scope
47
+
48
+ This Code of Conduct applies both within project spaces and in public spaces
49
+ when an individual is representing the project or its community. Examples of
50
+ representing a project or community include using an official project e-mail
51
+ address, posting via an official social media account, or acting as an appointed
52
+ representative at an online or offline event. Representation of a project may be
53
+ further defined and clarified by project maintainers.
54
+
55
+ ## Enforcement
56
+
57
+ Instances of abusive, harassing, or otherwise unacceptable behavior may be
58
+ reported by contacting the project team at todd.wilson@gov.bc.ca. All
59
+ complaints will be reviewed and investigated and will result in a response that
60
+ is deemed necessary and appropriate to the circumstances. The project team is
61
+ obligated to maintain confidentiality with regard to the reporter of an incident.
62
+ Further details of specific enforcement policies may be posted separately.
63
+
64
+ Project maintainers who do not follow or enforce the Code of Conduct in good
65
+ faith may face temporary or permanent repercussions as determined by other
66
+ members of the project's leadership.
67
+
68
+ ## Attribution
69
+
70
+ This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
71
+ available at [http://contributor-covenant.org/version/1/4][version]
72
+
73
+ [homepage]: http://contributor-covenant.org
74
+ [version]: http://contributor-covenant.org/version/1/4/
@@ -0,0 +1,5 @@
1
+ global-exclude .DS_Store
2
+ global-exclude *.pyc
3
+ exclude MANIFEST.in
4
+ recursive-include tests *.py
5
+ include LICENSE.txt README.md
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: bcdata
3
- Version: 0.17.1
3
+ Version: 0.17.2
4
4
  Summary: Download open data, monitor and report on changes
5
5
  Author-email: Simon Norris <snorris@hillcrestgeo.ca>
6
6
  Project-URL: Homepage, https://github.com/smnorris/bcdata
@@ -26,6 +26,7 @@ Requires-Dist: sqlalchemy
26
26
  Requires-Dist: stamina
27
27
  Provides-Extra: test
28
28
  Requires-Dist: pytest; extra == "test"
29
+ Requires-Dist: pytest-rerunfailures; extra == "test"
29
30
  Requires-Dist: pre-commit; extra == "test"
30
31
  Requires-Dist: requests-mock; extra == "test"
31
32
  Dynamic: license-file
@@ -43,8 +44,6 @@ but most sources are available only via WFS - and the syntax to download WFS dat
43
44
 
44
45
  This tool attempts to simplify downloads of BC geographic data and smoothly integrate with PostGIS and Python GIS tools like `geopandas` and `fiona`. The tool only accesses geographic data [avialable via WFS](https://bcgov.github.io/data-publication/pages/tips_tricks_webservices.html) - for other BC open data, download the files directly with `requests` / `curl` / `ogr2ogr` / etc (or the [bcdata R package](https://github.com/bcgov/bcdata)).
45
46
 
46
- If downloads are failing, consult the [BCGov Data Systems and Services uptime monitor](https://uptime.com/statuspage/bcgov-dss) - this tool depends on BC Open Geospatial Web Services API at [openmaps.gov.bc.ca](https://openmaps.gov.bc.ca/geo/pub/ows?service=WFS&request=Getcapabilities) and the [BC Data Catalogue API](https://catalogue.data.gov.bc.ca/dataset/bc-data-catalogue-api).
47
-
48
47
  **Disclaimer**
49
48
  *Data are generally licensed as [OGL-BC](http://www2.gov.bc.ca/gov/content/governments/about-the-bc-government/databc/open-data/open-government-license-bc), but it is the user's responsibility to check the licensing for any downloads.*
50
49
 
@@ -363,10 +362,14 @@ Clone the repo:
363
362
  $ git clone git@github.com:bcgov/bcdata_py.git
364
363
  $ cd bcdata_py
365
364
 
366
- `bc2pg` tests require a PostGIS database, referenced via the `DATABASE_URL` environment variable (matching the image used in CI, see `.github/workflows/tests.yml`). Start one locally with `docker compose`:
365
+ `bc2pg` tests require a PostGIS database, referenced via the `BCDATA_TEST_DATABASE_URL` environment variable.
366
+ Start one locally with docker (after editing the `docker-compose.yml` as needed):
367
367
 
368
+ $ docker compose build # default to building the postgis image since official pre-built images are not available for all platforms
368
369
  $ docker compose up -d
369
- $ export DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres
370
+ $ export BCDATA_TEST_DATABASE_URL=postgresql://postgres:postgres@localhost:8080/postgres
371
+
372
+
370
373
 
371
374
  install `bcdata` in development mode with [`uv`](https://docs.astral.sh/uv/):
372
375
 
@@ -11,8 +11,6 @@ but most sources are available only via WFS - and the syntax to download WFS dat
11
11
 
12
12
  This tool attempts to simplify downloads of BC geographic data and smoothly integrate with PostGIS and Python GIS tools like `geopandas` and `fiona`. The tool only accesses geographic data [avialable via WFS](https://bcgov.github.io/data-publication/pages/tips_tricks_webservices.html) - for other BC open data, download the files directly with `requests` / `curl` / `ogr2ogr` / etc (or the [bcdata R package](https://github.com/bcgov/bcdata)).
13
13
 
14
- If downloads are failing, consult the [BCGov Data Systems and Services uptime monitor](https://uptime.com/statuspage/bcgov-dss) - this tool depends on BC Open Geospatial Web Services API at [openmaps.gov.bc.ca](https://openmaps.gov.bc.ca/geo/pub/ows?service=WFS&request=Getcapabilities) and the [BC Data Catalogue API](https://catalogue.data.gov.bc.ca/dataset/bc-data-catalogue-api).
15
-
16
14
  **Disclaimer**
17
15
  *Data are generally licensed as [OGL-BC](http://www2.gov.bc.ca/gov/content/governments/about-the-bc-government/databc/open-data/open-government-license-bc), but it is the user's responsibility to check the licensing for any downloads.*
18
16
 
@@ -331,10 +329,14 @@ Clone the repo:
331
329
  $ git clone git@github.com:bcgov/bcdata_py.git
332
330
  $ cd bcdata_py
333
331
 
334
- `bc2pg` tests require a PostGIS database, referenced via the `DATABASE_URL` environment variable (matching the image used in CI, see `.github/workflows/tests.yml`). Start one locally with `docker compose`:
332
+ `bc2pg` tests require a PostGIS database, referenced via the `BCDATA_TEST_DATABASE_URL` environment variable.
333
+ Start one locally with docker (after editing the `docker-compose.yml` as needed):
335
334
 
335
+ $ docker compose build # default to building the postgis image since official pre-built images are not available for all platforms
336
336
  $ docker compose up -d
337
- $ export DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres
337
+ $ export BCDATA_TEST_DATABASE_URL=postgresql://postgres:postgres@localhost:8080/postgres
338
+
339
+
338
340
 
339
341
  install `bcdata` in development mode with [`uv`](https://docs.astral.sh/uv/):
340
342
 
@@ -0,0 +1,9 @@
1
+ # primary_keys.json
2
+
3
+ Record the bcdata table primary keys in one spot (where available).
4
+
5
+ When finished adding a key/value pair, be sure to validate:
6
+
7
+ python validate.py
8
+
9
+ The validation script iterates through each `table`: `column` key/value pair in `primary_keys.json`, confirming that the table exists and the given column is present within the table. Note that the script does not examine values of the column to confirm uniqueness.
@@ -0,0 +1,56 @@
1
+ {
2
+ "whse_admin_boundaries.clab_indian_reserves": "clab_id",
3
+ "whse_admin_boundaries.clab_national_parks": "national_park_id",
4
+ "whse_admin_boundaries.fadm_designated_areas": "feature_id",
5
+ "whse_admin_boundaries.fadm_special_protection_area": "feature_id",
6
+ "whse_admin_boundaries.fadm_tfl_all_sp": "tfl_all_sysid",
7
+ "whse_basemapping.bcgs_20k_grid": "map_tile",
8
+ "whse_basemapping.dbm_mof_50k_grid": "map_tile",
9
+ "whse_basemapping.gba_local_reg_greenspaces_sp": "local_reg_greenspace_id",
10
+ "whse_basemapping.gba_railway_structure_lines_sp": "railway_structure_line_id",
11
+ "whse_basemapping.gba_railway_tracks_sp": "railway_track_id",
12
+ "whse_basemapping.gba_transmission_lines_sp": "transmission_line_id",
13
+ "whse_basemapping.gns_geographical_names_sp": "geographical_names_id",
14
+ "whse_basemapping.nts_250k_grid": "map_tile",
15
+ "whse_basemapping.trim_cultural_lines": "objectid",
16
+ "whse_basemapping.trim_cultural_points": "objectid",
17
+ "whse_basemapping.trim_ebm_airfields": "objectid",
18
+ "whse_basemapping.trim_ebm_ocean": "objectid",
19
+ "whse_basemapping.utmg_utm_zones_sp": "utm_zone",
20
+ "whse_cadastre.pmbc_parcel_fabric_poly_svw": "parcel_fabric_poly_id",
21
+ "whse_environmental_monitoring.envcan_hydrometric_stn_sp": "hydrometric_station_id",
22
+ "whse_fish.fiss_stream_sample_sites_sp": "stream_sample_site_id",
23
+ "whse_fish.pscis_assessment_svw": "stream_crossing_id",
24
+ "whse_fish.pscis_design_proposal_svw": "stream_crossing_id",
25
+ "whse_fish.pscis_habitat_confirmation_svw": "stream_crossing_id",
26
+ "whse_fish.pscis_remediation_svw": "stream_crossing_id",
27
+ "whse_forest_tenure.ften_managed_licence_poly_svw": "objectid",
28
+ "whse_forest_tenure.ften_range_poly_svw": "objectid",
29
+ "whse_forest_tenure.ften_recreation_poly_svw": "rmf_skey",
30
+ "whse_forest_vegetation.ogsr_priority_def_area_cur_sp": "ogsr_pdac_sysid",
31
+ "whse_forest_vegetation.rec_visual_landscape_inventory": "vli_polygon_no",
32
+ "whse_forest_vegetation.veg_comp_lyr_r1_poly": "feature_id",
33
+ "whse_forest_vegetation.veg_consolidated_cut_blocks_sp": "vccb_sysid",
34
+ "whse_human_cultural_economic.hist_historic_environmnt_pa_sv": "hist_hef_sysid",
35
+ "whse_imagery_and_base_maps.mot_road_structure_sp": "hwy_structure_class_id",
36
+ "whse_land_use_planning.rmp_landscape_rsrv_design_sp": "rmp_lrd_sysid",
37
+ "whse_land_use_planning.rmp_ogma_legal_current_svw": "legal_ogma_internal_id",
38
+ "whse_land_use_planning.rmp_ogma_non_legal_current_svw": "non_legal_ogma_internal_id",
39
+ "whse_land_use_planning.rmp_plan_legal_poly_svw": "legal_feat_id",
40
+ "whse_land_use_planning.rmp_plan_non_legal_poly_svw": "non_legal_feat_id",
41
+ "whse_land_use_planning.rmp_strgc_land_rsrce_plan_svw": "strgc_land_rsrce_plan_id",
42
+ "whse_legal_admin_boundaries.abms_municipalities_sp": "lgl_admin_area_id",
43
+ "whse_legal_admin_boundaries.wcl_conservation_areas_ngo_sp": "conservation_areas_ngo_id",
44
+ "whse_legal_admin_boundaries.wcl_conservation_lands_sp": "conservation_land_id",
45
+ "whse_mineral_tenure.mta_acquired_tenure_svw": "tenure_number_id",
46
+ "whse_mineral_tenure.og_petrlm_dev_rds_pre06_pub_sp": "og_petrlm_dev_rd_pre06_pub_id",
47
+ "whse_mineral_tenure.og_road_segment_permit_sp": "og_road_segment_permit_id",
48
+ "whse_tantalis.ta_conservancy_areas_svw": "admin_area_sid",
49
+ "whse_tantalis.ta_crown_tenures_svw": "objectid",
50
+ "whse_tantalis.ta_park_ecores_pa_svw": "admin_area_sid",
51
+ "whse_tantalis.ta_wildlife_mgmt_areas_svw": "admin_area_sid",
52
+ "whse_water_management.wls_community_ws_pub_svw": "wls_cw_sysid",
53
+ "whse_wildlife_management.wcp_fish_sensitive_ws_poly": "fish_sensitive_ws_poly_id",
54
+ "whse_wildlife_management.wcp_ungulate_winter_range_sp": "ungulate_winter_range_id",
55
+ "whse_wildlife_management.wcp_wildlife_habitat_area_poly": "habitat_area_id"
56
+ }
@@ -0,0 +1,30 @@
1
+ import json
2
+ import logging
3
+ import sys
4
+
5
+ import bcdata
6
+
7
+ LOG_FORMAT = "%(asctime)s:%(levelname)s:%(name)s: %(message)s"
8
+
9
+ with open("primary_keys.json", "r") as file:
10
+ """validate pk database"""
11
+ logging.basicConfig(stream=sys.stderr, level=20, format=LOG_FORMAT)
12
+ log = logging.getLogger(__name__)
13
+ primary_keys = json.load(file)
14
+ pk_db_tables = set(primary_keys.keys())
15
+ bcdata_tables = {t.lower() for t in bcdata.list_tables()}
16
+ if pk_db_tables.issubset(bcdata_tables):
17
+ log.info("Table names in primary_keys.json are valid")
18
+ for table in primary_keys:
19
+ column = primary_keys[table]
20
+ schema = bcdata.get_table_definition(table)["schema"]
21
+ if column not in [c["column_name"].lower() for c in schema]:
22
+ raise ValueError(f"Column {column} not found in {table}")
23
+ log.info(
24
+ "Validation successful - columns listed in primary_keys.json are present in listed tables"
25
+ )
26
+ else:
27
+ invalid_keys = list(pk_db_tables - bcdata_tables)
28
+ for k in invalid_keys:
29
+ log.error(f"{k}")
30
+ raise ValueError("Invalid table name(s)")
@@ -0,0 +1,15 @@
1
+ services:
2
+ postgres:
3
+ image: postgis/postgis:18-3.6-alpine
4
+ pull_policy: never
5
+ build:
6
+ context: https://github.com/postgis/docker-postgis.git#master:18-3.6/alpine
7
+ environment:
8
+ POSTGRES_PASSWORD: postgres
9
+ ports:
10
+ - "8080:5432"
11
+ healthcheck:
12
+ test: ["CMD", "pg_isready", "-U", "postgres"]
13
+ interval: 10s
14
+ timeout: 5s
15
+ retries: 5
@@ -0,0 +1,58 @@
1
+ # WFS via ogr2ogr/curl
2
+
3
+ Making WFS requests and dumping to file/postgres is fairly straightforward with existing command line tools:
4
+
5
+ - list all layers:
6
+
7
+ $ ogrinfo WFS:http://openmaps.gov.bc.ca/geo/ows
8
+
9
+ - define a data request url:
10
+
11
+ $ airports_url="https://openmaps.gov.bc.ca/geo/pub/ows?service=WFS&version=2.0.0&request=GetFeature&typeName=WHSE_IMAGERY_AND_BASE_MAPS.GSR_AIRPORTS_SVW&outputFormat=json&SRSNAME=epsg%3A3005"
12
+
13
+ - describe the data:
14
+
15
+ $ ogrinfo -so $airports_url OGRGeoJSON
16
+
17
+ - making the request and dumping to file can be done different ways:
18
+
19
+ $ ogr2ogr \
20
+ -f GeoJSON \
21
+ airports.geojson \
22
+ $airports_url
23
+
24
+ $ curl $airports_url > airports.geojson
25
+
26
+ - request a bigger dataset - gdal docs say [paging is automatically supported for WFS 2.0.0](https://gdal.org/drivers/vector/wfs.html#paging-with-wfs-2-0):
27
+
28
+ $ uwr="https://openmaps.gov.bc.ca/geo/pub/ows?service=WFS&version=2.0.0&request=GetFeature&typeName=WHSE_WILDLIFE_MANAGEMENT.WCP_UNGULATE_WINTER_RANGE_SP&outputFormat=json&SRSNAME=epsg%3A3005"
29
+ $ curl $uwr > uwr.geojson
30
+ $ ogrinfo uwr.geojson uwr -so
31
+
32
+ INFO: Open of `uwr.geojson'
33
+ using driver `GeoJSON' successful.
34
+
35
+ Layer name: uwr
36
+ Geometry: Unknown (any)
37
+ Feature Count: 10000
38
+
39
+ only 10k features... a client needs to request the total number of features and define the per-page requests
40
+
41
+ - load to postgres:
42
+
43
+ $ ogr2ogr \
44
+ -f PostgreSQL \
45
+ PG:"host=localhost user=postgres dbname=postgis password=postgres" \
46
+ -lco SCHEMA=whse_imagery_and_base_maps \
47
+ -lco GEOMETRY_NAME=geom \
48
+ -nln gsr_airports_svw \
49
+ $airports_url
50
+
51
+
52
+ `bcdata` smooths the above processes by:
53
+
54
+ - providing Python and CLI interfaces for typical request options
55
+ - generating paged requests
56
+ - reducing server load by caching getcapabilites responses
57
+ - handling typical minor data issues (eg mixed geometry types)
58
+ - creating postgres tables that mirror the table schema as defined in BCGW (rather than using ogr default types)
@@ -1,5 +1,5 @@
1
1
  [build-system]
2
- requires = ["setuptools"]
2
+ requires = ["setuptools>=64", "setuptools_scm>=8"]
3
3
  build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
@@ -36,6 +36,7 @@ dependencies = [
36
36
  [project.optional-dependencies]
37
37
  test = [
38
38
  "pytest",
39
+ "pytest-rerunfailures",
39
40
  "pre-commit",
40
41
  "requests-mock"
41
42
  ]
@@ -47,8 +48,8 @@ bcdata = "bcdata.cli:cli"
47
48
  Homepage = "https://github.com/smnorris/bcdata"
48
49
  Issues = "https://github.com/smnorris/bcdata"
49
50
 
50
- [tool.setuptools.dynamic]
51
- version = {attr = "bcdata.__version__"}
51
+ [tool.setuptools_scm]
52
+ write_to = "src/bcdata/_version.py"
52
53
 
53
54
  [tool.ruff]
54
55
  line-length = 100
@@ -0,0 +1,13 @@
1
+ # Release checklist
2
+
3
+ - run tests (`uv run pytest`)
4
+ - confirm dependencies/classifiers in `pyproject.toml` are current
5
+ - run `uv lock` to update `uv.lock`
6
+ - commit changes on dev branch, create pr, merge pr to main
7
+ - switch to main branch locally, pull latest, create tag & push:
8
+
9
+ git checkout main
10
+ git pull
11
+ git tag -a <version> -m <version> && git push
12
+
13
+ Note that releases are automatically published (to pypi & github) on tag via `.github/workflows/release.yml`
@@ -0,0 +1,39 @@
1
+ rm -rf cli.md
2
+ printf '### CLI' > cli.md
3
+ printf '\n' >> cli.md
4
+
5
+ printf 'Commands available via the bcdata command line interface are documented with the --help option\n' >> cli.md
6
+ printf '\n' >> cli.md
7
+ printf '```\n\n$ bcdata --help\n\n' >> cli.md
8
+ bcdata --help >> cli.md
9
+ printf '```\n' >> cli.md
10
+
11
+ printf '\n' >> cli.md
12
+ printf '#### bc2pg\n\n' >> cli.md
13
+ printf '```\n$ bcdata bc2pg --help\n\n' >> cli.md
14
+ bcdata bc2pg --help >> cli.md
15
+ printf '```\n' >> cli.md
16
+
17
+ printf '\n' >> cli.md
18
+ printf '#### cat\n\n' >> cli.md
19
+ printf '```\n$ bcdata cat --help\n\n' >> cli.md
20
+ bcdata cat --help >> cli.md
21
+ printf '```\n' >> cli.md
22
+
23
+ printf '\n' >> cli.md
24
+ printf '#### dump \n\n' >> cli.md
25
+ printf '```\n$ bcdata dump --help\n\n' >> cli.md
26
+ bcdata dump --help >> cli.md
27
+ printf '```\n' >> cli.md
28
+
29
+ printf '\n' >> cli.md
30
+ printf '#### info\n\n' >> cli.md
31
+ printf '```\n$ bcdata info --help\n\n' >> cli.md
32
+ bcdata info --help >> cli.md
33
+ printf '```\n' >> cli.md
34
+
35
+ printf '\n' >> cli.md
36
+ printf '#### list\n\n' >> cli.md
37
+ printf '```\n$ bcdata list --help\n\n' >> cli.md
38
+ bcdata list --help >> cli.md
39
+ printf '```\n' >> cli.md
@@ -1,5 +1,6 @@
1
1
  import requests
2
2
 
3
+ from ._version import __version__ as __version__
3
4
  from .bc2pg import bc2pg as bc2pg
4
5
  from .bcdc import get_table_definition as get_table_definition
5
6
  from .bcdc import get_table_name as get_table_name
@@ -20,5 +21,3 @@ if response.status_code == 200:
20
21
  else:
21
22
  raise ConnectionError(f"Failed to download primary key database at {PRIMARY_KEY_DB_URL}")
22
23
  primary_keys = {}
23
-
24
- __version__ = "0.17.1"