bcdata 0.10.2__tar.gz → 0.14.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bcdata-0.10.2/bcdata.egg-info → bcdata-0.14.0}/PKG-INFO +72 -58
- {bcdata-0.10.2 → bcdata-0.14.0}/README.md +52 -36
- bcdata-0.14.0/pyproject.toml +55 -0
- bcdata-0.14.0/src/bcdata/__init__.py +25 -0
- {bcdata-0.10.2 → bcdata-0.14.0/src}/bcdata/bc2pg.py +43 -82
- {bcdata-0.10.2 → bcdata-0.14.0/src}/bcdata/bcdc.py +46 -59
- {bcdata-0.10.2 → bcdata-0.14.0/src}/bcdata/cli.py +81 -42
- {bcdata-0.10.2 → bcdata-0.14.0/src}/bcdata/database.py +25 -16
- {bcdata-0.10.2 → bcdata-0.14.0/src}/bcdata/wcs.py +2 -8
- {bcdata-0.10.2 → bcdata-0.14.0/src}/bcdata/wfs.py +106 -168
- {bcdata-0.10.2 → bcdata-0.14.0/src/bcdata.egg-info}/PKG-INFO +72 -58
- bcdata-0.14.0/src/bcdata.egg-info/SOURCES.txt +21 -0
- bcdata-0.14.0/src/bcdata.egg-info/requires.txt +13 -0
- {bcdata-0.10.2 → bcdata-0.14.0}/tests/test_bc2pg.py +33 -12
- {bcdata-0.10.2 → bcdata-0.14.0}/tests/test_bcdc.py +31 -2
- {bcdata-0.10.2 → bcdata-0.14.0}/tests/test_cli.py +9 -0
- {bcdata-0.10.2 → bcdata-0.14.0}/tests/test_wcs.py +3 -6
- {bcdata-0.10.2 → bcdata-0.14.0}/tests/test_wfs.py +5 -24
- bcdata-0.10.2/CHANGES.txt +0 -298
- bcdata-0.10.2/bcdata/__init__.py +0 -18
- bcdata-0.10.2/bcdata.egg-info/SOURCES.txt +0 -25
- bcdata-0.10.2/bcdata.egg-info/not-zip-safe +0 -1
- bcdata-0.10.2/bcdata.egg-info/requires.txt +0 -13
- bcdata-0.10.2/pyproject.toml +0 -16
- bcdata-0.10.2/requirements.txt +0 -8
- bcdata-0.10.2/setup.py +0 -55
- {bcdata-0.10.2 → bcdata-0.14.0}/LICENSE.txt +0 -0
- {bcdata-0.10.2 → bcdata-0.14.0}/setup.cfg +0 -0
- {bcdata-0.10.2 → bcdata-0.14.0/src}/bcdata.egg-info/dependency_links.txt +0 -0
- {bcdata-0.10.2 → bcdata-0.14.0/src}/bcdata.egg-info/entry_points.txt +0 -0
- {bcdata-0.10.2 → bcdata-0.14.0/src}/bcdata.egg-info/top_level.txt +0 -0
|
@@ -1,36 +1,34 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: bcdata
|
|
3
|
-
Version: 0.
|
|
4
|
-
Summary:
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
Keywords: gis geospatial data BC DataBC download "Britsh Columbia"
|
|
10
|
-
Classifier: Development Status :: 3 - Alpha
|
|
11
|
-
Classifier: Intended Audience :: Developers
|
|
12
|
-
Classifier: Intended Audience :: Science/Research
|
|
13
|
-
Classifier: Operating System :: OS Independent
|
|
3
|
+
Version: 0.14.0
|
|
4
|
+
Summary: Download open data, monitor and report on changes
|
|
5
|
+
Author-email: Simon Norris <snorris@hillcrestgeo.ca>
|
|
6
|
+
Project-URL: Homepage, https://github.com/smnorris/bcdata
|
|
7
|
+
Project-URL: Issues, https://github.com/smnorris/bcdata
|
|
8
|
+
Classifier: Development Status :: 1 - Planning
|
|
14
9
|
Classifier: Programming Language :: Python :: 3.9
|
|
15
10
|
Classifier: Programming Language :: Python :: 3.10
|
|
16
11
|
Classifier: Programming Language :: Python :: 3.11
|
|
17
12
|
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
-
Classifier:
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
19
15
|
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Requires-Python: >=3.9
|
|
20
18
|
Description-Content-Type: text/markdown
|
|
21
19
|
License-File: LICENSE.txt
|
|
22
|
-
Requires-Dist: geoalchemy2
|
|
23
|
-
Requires-Dist: geopandas
|
|
24
|
-
Requires-Dist: owslib
|
|
25
|
-
Requires-Dist: psycopg2-binary
|
|
26
|
-
Requires-Dist: rasterio
|
|
27
|
-
Requires-Dist: requests
|
|
28
|
-
Requires-Dist: sqlalchemy
|
|
29
|
-
Requires-Dist: stamina
|
|
20
|
+
Requires-Dist: geoalchemy2
|
|
21
|
+
Requires-Dist: geopandas
|
|
22
|
+
Requires-Dist: owslib
|
|
23
|
+
Requires-Dist: psycopg2-binary
|
|
24
|
+
Requires-Dist: rasterio
|
|
25
|
+
Requires-Dist: requests
|
|
26
|
+
Requires-Dist: sqlalchemy
|
|
27
|
+
Requires-Dist: stamina
|
|
30
28
|
Provides-Extra: test
|
|
31
|
-
Requires-Dist: pytest
|
|
29
|
+
Requires-Dist: pytest; extra == "test"
|
|
32
30
|
Requires-Dist: pre-commit; extra == "test"
|
|
33
|
-
Requires-Dist:
|
|
31
|
+
Requires-Dist: requests-mock; extra == "test"
|
|
34
32
|
|
|
35
33
|
# bcdata
|
|
36
34
|
|
|
@@ -43,7 +41,9 @@ There is a [wealth of British Columbia geographic information available as open
|
|
|
43
41
|
data](https://catalogue.data.gov.bc.ca/dataset?download_audience=Public),
|
|
44
42
|
but direct file download urls are not available and the syntax to accesss WFS via `ogr2ogr` and/or `curl/wget` can be awkward.
|
|
45
43
|
|
|
46
|
-
This
|
|
44
|
+
This tool attempts to simplify downloads of BC geographic data and smoothly integrate with PostGIS and Python GIS tools like `geopandas`, `fiona` and `rasterio`. As noted above, the module is an interface only to geographic data [avialable via WFS/WCS](https://bcgov.github.io/data-publication/pages/tips_tricks_webservices.html) - to work with other BC open data, use the [bcdata R package](https://github.com/bcgov/bcdata).
|
|
45
|
+
|
|
46
|
+
Note that this tool depends on BC Open Geospatial Web Services API at [openmaps.gov.bc.ca](https://openmaps.gov.bc.ca/geo/pub/ows?service=WFS&request=Getcapabilities) and the [BC Data Catalogue API](https://catalogue.data.gov.bc.ca/dataset/bc-data-catalogue-api). If downloads are failing, consult the [BCGov Data Systems and Services uptime monitor](https://uptime.com/statuspage/bcgov-dss).
|
|
47
47
|
|
|
48
48
|
**Disclaimer**
|
|
49
49
|
*It is the user's responsibility to check the licensing for any downloads, data are generally licensed as [OGL-BC](http://www2.gov.bc.ca/gov/content/governments/about-the-bc-government/databc/open-data/open-government-license-bc)*
|
|
@@ -123,7 +123,8 @@ AERODROME_STATUS AIRCRAFT_ACCESS_IND AIRPORT_NAME
|
|
|
123
123
|
```
|
|
124
124
|
|
|
125
125
|
### CLI
|
|
126
|
-
|
|
126
|
+
|
|
127
|
+
Commands available via the bcdata command line interface are documented with the --help option
|
|
127
128
|
|
|
128
129
|
```
|
|
129
130
|
|
|
@@ -136,7 +137,7 @@ Options:
|
|
|
136
137
|
--help Show this message and exit.
|
|
137
138
|
|
|
138
139
|
Commands:
|
|
139
|
-
bc2pg
|
|
140
|
+
bc2pg Load a DataBC WFS layer to a postgres db
|
|
140
141
|
cat Write DataBC features to stdout as GeoJSON feature objects.
|
|
141
142
|
dem Dump BC DEM to TIFF
|
|
142
143
|
dump Write DataBC features to stdout as GeoJSON feature collection.
|
|
@@ -151,27 +152,32 @@ $ bcdata bc2pg --help
|
|
|
151
152
|
|
|
152
153
|
Usage: bcdata bc2pg [OPTIONS] DATASET
|
|
153
154
|
|
|
154
|
-
|
|
155
|
+
Load a DataBC WFS layer to a postgres db
|
|
155
156
|
|
|
156
|
-
$ bcdata bc2pg
|
|
157
|
+
$ bcdata bc2pg whse_imagery_and_base_maps.gsr_airports_svw
|
|
157
158
|
|
|
158
159
|
Options:
|
|
159
|
-
-db, --db_url TEXT
|
|
160
|
-
|
|
161
|
-
--table TEXT
|
|
162
|
-
--schema TEXT
|
|
163
|
-
--geometry_type TEXT
|
|
164
|
-
--query TEXT
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
-
|
|
169
|
-
|
|
170
|
-
-
|
|
171
|
-
-
|
|
172
|
-
-
|
|
173
|
-
-
|
|
174
|
-
--
|
|
160
|
+
-db, --db_url TEXT Target database url, defaults to
|
|
161
|
+
$DATABASE_URL environment variable if set
|
|
162
|
+
--table TEXT Destination table name
|
|
163
|
+
--schema TEXT Destination schema name
|
|
164
|
+
--geometry_type TEXT Spatial type of geometry column
|
|
165
|
+
--query TEXT A valid CQL or ECQL query
|
|
166
|
+
--bounds TEXT Bounds: "left bottom right top" or "[left,
|
|
167
|
+
bottom, right, top]". Coordinates are BC
|
|
168
|
+
Albers (default) or --bounds_crs
|
|
169
|
+
--bounds-crs, --bounds_crs TEXT
|
|
170
|
+
CRS of provided bounds
|
|
171
|
+
-c, --count INTEGER Total number of features to load
|
|
172
|
+
-k, --primary_key TEXT Primary key of dataset
|
|
173
|
+
-s, --sortby TEXT Name of sort field
|
|
174
|
+
-e, --schema_only Create empty table from catalogue schema
|
|
175
|
+
-a, --append Append to existing table
|
|
176
|
+
-r, --refresh Truncate from existing table before load
|
|
177
|
+
-t, --no_timestamp Do not log download to bcdata.log
|
|
178
|
+
-v, --verbose Increase verbosity.
|
|
179
|
+
-q, --quiet Decrease verbosity.
|
|
180
|
+
--help Show this message and exit.
|
|
175
181
|
```
|
|
176
182
|
|
|
177
183
|
#### cat
|
|
@@ -185,16 +191,18 @@ Usage: bcdata cat [OPTIONS] DATASET
|
|
|
185
191
|
|
|
186
192
|
Options:
|
|
187
193
|
--query TEXT A valid CQL or ECQL query
|
|
194
|
+
-c, --count INTEGER Number of features to request and dump
|
|
188
195
|
--bounds TEXT Bounds: "left bottom right top" or "[left,
|
|
189
196
|
bottom, right, top]". Coordinates are BC
|
|
190
197
|
Albers (default) or --bounds_crs
|
|
198
|
+
--bounds-crs, --bounds_crs TEXT
|
|
199
|
+
CRS of provided bounds
|
|
191
200
|
--indent INTEGER Indentation level for JSON output
|
|
192
201
|
--compact / --not-compact Use compact separators (',', ':').
|
|
193
202
|
--dst-crs, --dst_crs TEXT Destination CRS
|
|
194
203
|
-s, --sortby TEXT Name of sort field
|
|
195
|
-
--bounds-crs, --bounds_crs TEXT
|
|
196
|
-
CRS of provided bounds
|
|
197
204
|
-l, --lowercase Write column/properties names as lowercase
|
|
205
|
+
-m, --promote-to-multi Promote features to multipart
|
|
198
206
|
-v, --verbose Increase verbosity.
|
|
199
207
|
-q, --quiet Decrease verbosity.
|
|
200
208
|
--help Show this message and exit.
|
|
@@ -214,9 +222,8 @@ Options:
|
|
|
214
222
|
--bounds TEXT Bounds: "left bottom right top" or "[left,
|
|
215
223
|
bottom, right, top]". Coordinates are BC
|
|
216
224
|
Albers (default) or --bounds_crs [required]
|
|
217
|
-
--dst-crs
|
|
218
|
-
--bounds-crs
|
|
219
|
-
CRS of provided bounds
|
|
225
|
+
--dst-crs TEXT CRS of output file
|
|
226
|
+
--bounds-crs TEXT CRS of provided bounds
|
|
220
227
|
-r, --resolution INTEGER
|
|
221
228
|
-a, --align Align provided bounds to provincial standard
|
|
222
229
|
-i, --interpolation [nearest|bilinear|bicubic]
|
|
@@ -238,18 +245,20 @@ Usage: bcdata dump [OPTIONS] DATASET
|
|
|
238
245
|
$ bcdata dump bc-airports --query "AIRPORT_NAME='Victoria Harbour (Shoal Point) Heliport'"
|
|
239
246
|
$ bcdata dump bc-airports --bounds xmin ymin xmax ymax
|
|
240
247
|
|
|
241
|
-
It can also be combined to read bounds of a feature dataset using Fiona:
|
|
248
|
+
It can also be combined to read bounds of a feature dataset using Fiona:
|
|
242
249
|
$ bcdata dump bc-airports --bounds $(fio info aoi.shp --bounds)
|
|
243
250
|
|
|
244
251
|
Options:
|
|
245
252
|
--query TEXT A valid CQL or ECQL query
|
|
246
|
-
-
|
|
253
|
+
-c, --count INTEGER Number of features to request and dump
|
|
247
254
|
--bounds TEXT Bounds: "left bottom right top" or "[left,
|
|
248
255
|
bottom, right, top]". Coordinates are BC
|
|
249
256
|
Albers (default) or --bounds_crs
|
|
250
257
|
--bounds-crs, --bounds_crs TEXT
|
|
251
258
|
CRS of provided bounds
|
|
259
|
+
-s, --sortby TEXT Name of sort field
|
|
252
260
|
-l, --lowercase Write column/properties names as lowercase
|
|
261
|
+
-m, --promote-to-multi Promote features to multipart
|
|
253
262
|
-v, --verbose Increase verbosity.
|
|
254
263
|
-q, --quiet Decrease verbosity.
|
|
255
264
|
--help Show this message and exit.
|
|
@@ -286,6 +295,7 @@ Usage: bcdata list [OPTIONS]
|
|
|
286
295
|
List DataBC layers available via WFS
|
|
287
296
|
|
|
288
297
|
Options:
|
|
298
|
+
-r, --refresh Refresh the cached list
|
|
289
299
|
--help Show this message and exit.
|
|
290
300
|
```
|
|
291
301
|
|
|
@@ -332,7 +342,7 @@ Describe a dataset. Note that if we know the id of a dataset, we can use that ra
|
|
|
332
342
|
|
|
333
343
|
The JSON output can be manipulated with [jq](https://stedolan.github.io/jq/). For example, to show only the fields available in the dataset:
|
|
334
344
|
|
|
335
|
-
$ bcdata info bc-airports | jq '.schema
|
|
345
|
+
$ bcdata info bc-airports | jq '.schema'
|
|
336
346
|
{
|
|
337
347
|
"CUSTODIAN_ORG_DESCRIPTION": "string",
|
|
338
348
|
"BUSINESS_CATEGORY_CLASS": "string",
|
|
@@ -360,6 +370,11 @@ Save a layer to a geopackage in BC Albers:
|
|
|
360
370
|
|
|
361
371
|
Note that this will not work if the source data has mixed geometry types.
|
|
362
372
|
|
|
373
|
+
Load to parquet on s3:
|
|
374
|
+
|
|
375
|
+
$ bcdata dump bc-airports \
|
|
376
|
+
| ogr2ogr -f Parquet /vsis3/$BUCKET/airports.parquet -s_srs EPSG:4326 -t_srs EPSG:3005 /vsistdin/
|
|
377
|
+
|
|
363
378
|
Load data to postgres and run a spatial query:
|
|
364
379
|
|
|
365
380
|
$ bcdata bc2pg bc-airports \
|
|
@@ -406,13 +421,12 @@ Load data to postgres and run a spatial query:
|
|
|
406
421
|
|
|
407
422
|
Create virtualenv and install `bcdata` in development mode:
|
|
408
423
|
|
|
409
|
-
$
|
|
410
|
-
$
|
|
411
|
-
$
|
|
412
|
-
|
|
413
|
-
(
|
|
414
|
-
(
|
|
415
|
-
(bcdata_env)$ py.test
|
|
424
|
+
$ git clone git@github.com:smnorris/bcdata.git
|
|
425
|
+
$ cd bcdata
|
|
426
|
+
$ python -m venv .venv
|
|
427
|
+
$ source .venv/bin/activate
|
|
428
|
+
(.venv)$ pip install -e .[test]
|
|
429
|
+
(.venv)$ py.test
|
|
416
430
|
|
|
417
431
|
|
|
418
432
|
## Other implementations
|
|
@@ -9,7 +9,9 @@ There is a [wealth of British Columbia geographic information available as open
|
|
|
9
9
|
data](https://catalogue.data.gov.bc.ca/dataset?download_audience=Public),
|
|
10
10
|
but direct file download urls are not available and the syntax to accesss WFS via `ogr2ogr` and/or `curl/wget` can be awkward.
|
|
11
11
|
|
|
12
|
-
This
|
|
12
|
+
This tool attempts to simplify downloads of BC geographic data and smoothly integrate with PostGIS and Python GIS tools like `geopandas`, `fiona` and `rasterio`. As noted above, the module is an interface only to geographic data [avialable via WFS/WCS](https://bcgov.github.io/data-publication/pages/tips_tricks_webservices.html) - to work with other BC open data, use the [bcdata R package](https://github.com/bcgov/bcdata).
|
|
13
|
+
|
|
14
|
+
Note that this tool depends on BC Open Geospatial Web Services API at [openmaps.gov.bc.ca](https://openmaps.gov.bc.ca/geo/pub/ows?service=WFS&request=Getcapabilities) and the [BC Data Catalogue API](https://catalogue.data.gov.bc.ca/dataset/bc-data-catalogue-api). If downloads are failing, consult the [BCGov Data Systems and Services uptime monitor](https://uptime.com/statuspage/bcgov-dss).
|
|
13
15
|
|
|
14
16
|
**Disclaimer**
|
|
15
17
|
*It is the user's responsibility to check the licensing for any downloads, data are generally licensed as [OGL-BC](http://www2.gov.bc.ca/gov/content/governments/about-the-bc-government/databc/open-data/open-government-license-bc)*
|
|
@@ -89,7 +91,8 @@ AERODROME_STATUS AIRCRAFT_ACCESS_IND AIRPORT_NAME
|
|
|
89
91
|
```
|
|
90
92
|
|
|
91
93
|
### CLI
|
|
92
|
-
|
|
94
|
+
|
|
95
|
+
Commands available via the bcdata command line interface are documented with the --help option
|
|
93
96
|
|
|
94
97
|
```
|
|
95
98
|
|
|
@@ -102,7 +105,7 @@ Options:
|
|
|
102
105
|
--help Show this message and exit.
|
|
103
106
|
|
|
104
107
|
Commands:
|
|
105
|
-
bc2pg
|
|
108
|
+
bc2pg Load a DataBC WFS layer to a postgres db
|
|
106
109
|
cat Write DataBC features to stdout as GeoJSON feature objects.
|
|
107
110
|
dem Dump BC DEM to TIFF
|
|
108
111
|
dump Write DataBC features to stdout as GeoJSON feature collection.
|
|
@@ -117,27 +120,32 @@ $ bcdata bc2pg --help
|
|
|
117
120
|
|
|
118
121
|
Usage: bcdata bc2pg [OPTIONS] DATASET
|
|
119
122
|
|
|
120
|
-
|
|
123
|
+
Load a DataBC WFS layer to a postgres db
|
|
121
124
|
|
|
122
|
-
$ bcdata bc2pg
|
|
125
|
+
$ bcdata bc2pg whse_imagery_and_base_maps.gsr_airports_svw
|
|
123
126
|
|
|
124
127
|
Options:
|
|
125
|
-
-db, --db_url TEXT
|
|
126
|
-
|
|
127
|
-
--table TEXT
|
|
128
|
-
--schema TEXT
|
|
129
|
-
--geometry_type TEXT
|
|
130
|
-
--query TEXT
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
-
|
|
135
|
-
|
|
136
|
-
-
|
|
137
|
-
-
|
|
138
|
-
-
|
|
139
|
-
-
|
|
140
|
-
--
|
|
128
|
+
-db, --db_url TEXT Target database url, defaults to
|
|
129
|
+
$DATABASE_URL environment variable if set
|
|
130
|
+
--table TEXT Destination table name
|
|
131
|
+
--schema TEXT Destination schema name
|
|
132
|
+
--geometry_type TEXT Spatial type of geometry column
|
|
133
|
+
--query TEXT A valid CQL or ECQL query
|
|
134
|
+
--bounds TEXT Bounds: "left bottom right top" or "[left,
|
|
135
|
+
bottom, right, top]". Coordinates are BC
|
|
136
|
+
Albers (default) or --bounds_crs
|
|
137
|
+
--bounds-crs, --bounds_crs TEXT
|
|
138
|
+
CRS of provided bounds
|
|
139
|
+
-c, --count INTEGER Total number of features to load
|
|
140
|
+
-k, --primary_key TEXT Primary key of dataset
|
|
141
|
+
-s, --sortby TEXT Name of sort field
|
|
142
|
+
-e, --schema_only Create empty table from catalogue schema
|
|
143
|
+
-a, --append Append to existing table
|
|
144
|
+
-r, --refresh Truncate from existing table before load
|
|
145
|
+
-t, --no_timestamp Do not log download to bcdata.log
|
|
146
|
+
-v, --verbose Increase verbosity.
|
|
147
|
+
-q, --quiet Decrease verbosity.
|
|
148
|
+
--help Show this message and exit.
|
|
141
149
|
```
|
|
142
150
|
|
|
143
151
|
#### cat
|
|
@@ -151,16 +159,18 @@ Usage: bcdata cat [OPTIONS] DATASET
|
|
|
151
159
|
|
|
152
160
|
Options:
|
|
153
161
|
--query TEXT A valid CQL or ECQL query
|
|
162
|
+
-c, --count INTEGER Number of features to request and dump
|
|
154
163
|
--bounds TEXT Bounds: "left bottom right top" or "[left,
|
|
155
164
|
bottom, right, top]". Coordinates are BC
|
|
156
165
|
Albers (default) or --bounds_crs
|
|
166
|
+
--bounds-crs, --bounds_crs TEXT
|
|
167
|
+
CRS of provided bounds
|
|
157
168
|
--indent INTEGER Indentation level for JSON output
|
|
158
169
|
--compact / --not-compact Use compact separators (',', ':').
|
|
159
170
|
--dst-crs, --dst_crs TEXT Destination CRS
|
|
160
171
|
-s, --sortby TEXT Name of sort field
|
|
161
|
-
--bounds-crs, --bounds_crs TEXT
|
|
162
|
-
CRS of provided bounds
|
|
163
172
|
-l, --lowercase Write column/properties names as lowercase
|
|
173
|
+
-m, --promote-to-multi Promote features to multipart
|
|
164
174
|
-v, --verbose Increase verbosity.
|
|
165
175
|
-q, --quiet Decrease verbosity.
|
|
166
176
|
--help Show this message and exit.
|
|
@@ -180,9 +190,8 @@ Options:
|
|
|
180
190
|
--bounds TEXT Bounds: "left bottom right top" or "[left,
|
|
181
191
|
bottom, right, top]". Coordinates are BC
|
|
182
192
|
Albers (default) or --bounds_crs [required]
|
|
183
|
-
--dst-crs
|
|
184
|
-
--bounds-crs
|
|
185
|
-
CRS of provided bounds
|
|
193
|
+
--dst-crs TEXT CRS of output file
|
|
194
|
+
--bounds-crs TEXT CRS of provided bounds
|
|
186
195
|
-r, --resolution INTEGER
|
|
187
196
|
-a, --align Align provided bounds to provincial standard
|
|
188
197
|
-i, --interpolation [nearest|bilinear|bicubic]
|
|
@@ -204,18 +213,20 @@ Usage: bcdata dump [OPTIONS] DATASET
|
|
|
204
213
|
$ bcdata dump bc-airports --query "AIRPORT_NAME='Victoria Harbour (Shoal Point) Heliport'"
|
|
205
214
|
$ bcdata dump bc-airports --bounds xmin ymin xmax ymax
|
|
206
215
|
|
|
207
|
-
It can also be combined to read bounds of a feature dataset using Fiona:
|
|
216
|
+
It can also be combined to read bounds of a feature dataset using Fiona:
|
|
208
217
|
$ bcdata dump bc-airports --bounds $(fio info aoi.shp --bounds)
|
|
209
218
|
|
|
210
219
|
Options:
|
|
211
220
|
--query TEXT A valid CQL or ECQL query
|
|
212
|
-
-
|
|
221
|
+
-c, --count INTEGER Number of features to request and dump
|
|
213
222
|
--bounds TEXT Bounds: "left bottom right top" or "[left,
|
|
214
223
|
bottom, right, top]". Coordinates are BC
|
|
215
224
|
Albers (default) or --bounds_crs
|
|
216
225
|
--bounds-crs, --bounds_crs TEXT
|
|
217
226
|
CRS of provided bounds
|
|
227
|
+
-s, --sortby TEXT Name of sort field
|
|
218
228
|
-l, --lowercase Write column/properties names as lowercase
|
|
229
|
+
-m, --promote-to-multi Promote features to multipart
|
|
219
230
|
-v, --verbose Increase verbosity.
|
|
220
231
|
-q, --quiet Decrease verbosity.
|
|
221
232
|
--help Show this message and exit.
|
|
@@ -252,6 +263,7 @@ Usage: bcdata list [OPTIONS]
|
|
|
252
263
|
List DataBC layers available via WFS
|
|
253
264
|
|
|
254
265
|
Options:
|
|
266
|
+
-r, --refresh Refresh the cached list
|
|
255
267
|
--help Show this message and exit.
|
|
256
268
|
```
|
|
257
269
|
|
|
@@ -298,7 +310,7 @@ Describe a dataset. Note that if we know the id of a dataset, we can use that ra
|
|
|
298
310
|
|
|
299
311
|
The JSON output can be manipulated with [jq](https://stedolan.github.io/jq/). For example, to show only the fields available in the dataset:
|
|
300
312
|
|
|
301
|
-
$ bcdata info bc-airports | jq '.schema
|
|
313
|
+
$ bcdata info bc-airports | jq '.schema'
|
|
302
314
|
{
|
|
303
315
|
"CUSTODIAN_ORG_DESCRIPTION": "string",
|
|
304
316
|
"BUSINESS_CATEGORY_CLASS": "string",
|
|
@@ -326,6 +338,11 @@ Save a layer to a geopackage in BC Albers:
|
|
|
326
338
|
|
|
327
339
|
Note that this will not work if the source data has mixed geometry types.
|
|
328
340
|
|
|
341
|
+
Load to parquet on s3:
|
|
342
|
+
|
|
343
|
+
$ bcdata dump bc-airports \
|
|
344
|
+
| ogr2ogr -f Parquet /vsis3/$BUCKET/airports.parquet -s_srs EPSG:4326 -t_srs EPSG:3005 /vsistdin/
|
|
345
|
+
|
|
329
346
|
Load data to postgres and run a spatial query:
|
|
330
347
|
|
|
331
348
|
$ bcdata bc2pg bc-airports \
|
|
@@ -372,13 +389,12 @@ Load data to postgres and run a spatial query:
|
|
|
372
389
|
|
|
373
390
|
Create virtualenv and install `bcdata` in development mode:
|
|
374
391
|
|
|
375
|
-
$
|
|
376
|
-
$
|
|
377
|
-
$
|
|
378
|
-
|
|
379
|
-
(
|
|
380
|
-
(
|
|
381
|
-
(bcdata_env)$ py.test
|
|
392
|
+
$ git clone git@github.com:smnorris/bcdata.git
|
|
393
|
+
$ cd bcdata
|
|
394
|
+
$ python -m venv .venv
|
|
395
|
+
$ source .venv/bin/activate
|
|
396
|
+
(.venv)$ pip install -e .[test]
|
|
397
|
+
(.venv)$ py.test
|
|
382
398
|
|
|
383
399
|
|
|
384
400
|
## Other implementations
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "bcdata"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
license = {file = "LICENSE"}
|
|
10
|
+
authors = [
|
|
11
|
+
{name="Simon Norris", email="snorris@hillcrestgeo.ca"},
|
|
12
|
+
]
|
|
13
|
+
description = "Download open data, monitor and report on changes"
|
|
14
|
+
requires-python = ">=3.9"
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 1 - Planning",
|
|
17
|
+
"Programming Language :: Python :: 3.9",
|
|
18
|
+
"Programming Language :: Python :: 3.10",
|
|
19
|
+
"Programming Language :: Python :: 3.11",
|
|
20
|
+
"Programming Language :: Python :: 3.12",
|
|
21
|
+
"Programming Language :: Python :: 3.13",
|
|
22
|
+
"License :: OSI Approved :: MIT License",
|
|
23
|
+
"Topic :: Scientific/Engineering :: GIS",
|
|
24
|
+
"Operating System :: OS Independent"
|
|
25
|
+
]
|
|
26
|
+
dependencies = [
|
|
27
|
+
"geoalchemy2",
|
|
28
|
+
"geopandas",
|
|
29
|
+
"owslib",
|
|
30
|
+
"psycopg2-binary",
|
|
31
|
+
"rasterio",
|
|
32
|
+
"requests",
|
|
33
|
+
"sqlalchemy",
|
|
34
|
+
"stamina"
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[project.optional-dependencies]
|
|
38
|
+
test = [
|
|
39
|
+
"pytest",
|
|
40
|
+
"pre-commit",
|
|
41
|
+
"requests-mock"
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[project.scripts]
|
|
45
|
+
bcdata = "bcdata.cli:cli"
|
|
46
|
+
|
|
47
|
+
[project.urls]
|
|
48
|
+
Homepage = "https://github.com/smnorris/bcdata"
|
|
49
|
+
Issues = "https://github.com/smnorris/bcdata"
|
|
50
|
+
|
|
51
|
+
[tool.setuptools.dynamic]
|
|
52
|
+
version = {attr = "bcdata.__version__"}
|
|
53
|
+
|
|
54
|
+
[tool.ruff]
|
|
55
|
+
line-length = 100
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
|
|
3
|
+
from .bc2pg import bc2pg as bc2pg
|
|
4
|
+
from .bcdc import get_table_definition as get_table_definition
|
|
5
|
+
from .bcdc import get_table_name as get_table_name
|
|
6
|
+
from .wcs import get_dem as get_dem
|
|
7
|
+
from .wfs import get_count as get_count
|
|
8
|
+
from .wfs import get_data as get_data
|
|
9
|
+
from .wfs import get_sortkey as get_sortkey
|
|
10
|
+
from .wfs import list_tables as list_tables
|
|
11
|
+
from .wfs import validate_name as validate_name
|
|
12
|
+
|
|
13
|
+
PRIMARY_KEY_DB_URL = "https://raw.githubusercontent.com/smnorris/bcdata/main/data/primary_keys.json"
|
|
14
|
+
|
|
15
|
+
# BCDC does not indicate which column in the schema is the primary key.
|
|
16
|
+
# In this absence, bcdata maintains its own dictionary of {table: primary_key},
|
|
17
|
+
# served via github. Retrieve the dict with this function"""
|
|
18
|
+
response = requests.get(PRIMARY_KEY_DB_URL)
|
|
19
|
+
if response.status_code == 200:
|
|
20
|
+
primary_keys = response.json()
|
|
21
|
+
else:
|
|
22
|
+
raise Exception(f"Failed to download primary key database at {PRIMARY_KEY_DB_URL}")
|
|
23
|
+
primary_keys = {}
|
|
24
|
+
|
|
25
|
+
__version__ = "0.14.0"
|