bcdata 0.13.0__tar.gz → 0.14.0__tar.gz

{bcdata-0.13.0/src/bcdata.egg-info → bcdata-0.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bcdata
-Version: 0.13.0
+Version: 0.14.0
 Summary: Download open data, monitor and report on changes
 Author-email: Simon Norris <snorris@hillcrestgeo.ca>
 Project-URL: Homepage, https://github.com/smnorris/bcdata
@@ -123,7 +123,8 @@ AERODROME_STATUS AIRCRAFT_ACCESS_IND                          AIRPORT_NAME
 ```
 
 ### CLI
-Commands available via the bcdata command line interface are documented with the `--help` option:
+
+Commands available via the bcdata command line interface are documented with the --help option
 
 ```
 
@@ -136,7 +137,7 @@ Options:
   --help     Show this message and exit.
 
 Commands:
-  bc2pg  Download a DataBC WFS layer to postgres
+  bc2pg  Load a DataBC WFS layer to a postgres db
   cat    Write DataBC features to stdout as GeoJSON feature objects.
   dem    Dump BC DEM to TIFF
   dump   Write DataBC features to stdout as GeoJSON feature collection.
@@ -151,7 +152,7 @@ $ bcdata bc2pg --help
 
 Usage: bcdata bc2pg [OPTIONS] DATASET
 
-  Download a DataBC WFS layer to postgres
+  Load a DataBC WFS layer to a postgres db
 
    $ bcdata bc2pg whse_imagery_and_base_maps.gsr_airports_svw
 
@@ -190,16 +191,18 @@ Usage: bcdata cat [OPTIONS] DATASET
 
 Options:
   --query TEXT                    A valid CQL or ECQL query
+  -c, --count INTEGER             Number of features to request and dump
   --bounds TEXT                   Bounds: "left bottom right top" or "[left,
                                   bottom, right, top]". Coordinates are BC
                                   Albers (default) or --bounds_crs
+  --bounds-crs, --bounds_crs TEXT
+                                  CRS of provided bounds
   --indent INTEGER                Indentation level for JSON output
   --compact / --not-compact       Use compact separators (',', ':').
   --dst-crs, --dst_crs TEXT       Destination CRS
   -s, --sortby TEXT               Name of sort field
-  --bounds-crs, --bounds_crs TEXT
-                                  CRS of provided bounds
   -l, --lowercase                 Write column/properties names as lowercase
+  -m, --promote-to-multi          Promote features to multipart
   -v, --verbose                   Increase verbosity.
   -q, --quiet                     Decrease verbosity.
   --help                          Show this message and exit.
@@ -219,9 +222,8 @@ Options:
   --bounds TEXT                   Bounds: "left bottom right top" or "[left,
                                   bottom, right, top]". Coordinates are BC
                                   Albers (default) or --bounds_crs  [required]
-  --dst-crs, --dst_crs TEXT       Destination CRS
-  --bounds-crs, --bounds_crs TEXT
-                                  CRS of provided bounds
+  --dst-crs TEXT                  CRS of output file
+  --bounds-crs TEXT               CRS of provided bounds
   -r, --resolution INTEGER
   -a, --align                     Align provided bounds to provincial standard
   -i, --interpolation [nearest|bilinear|bicubic]
@@ -243,18 +245,20 @@ Usage: bcdata dump [OPTIONS] DATASET
     $ bcdata dump bc-airports --query "AIRPORT_NAME='Victoria Harbour (Shoal Point) Heliport'"
     $ bcdata dump bc-airports --bounds xmin ymin xmax ymax
 
-   It can also be combined to read bounds of a feature dataset using Fiona:
+   It can also be combined to read bounds of a feature dataset using Fiona: 
    $ bcdata dump bc-airports --bounds $(fio info aoi.shp --bounds)
 
 Options:
   --query TEXT                    A valid CQL or ECQL query
-  -o, --out_file TEXT             Output file
+  -c, --count INTEGER             Number of features to request and dump
   --bounds TEXT                   Bounds: "left bottom right top" or "[left,
                                   bottom, right, top]". Coordinates are BC
                                   Albers (default) or --bounds_crs
   --bounds-crs, --bounds_crs TEXT
                                   CRS of provided bounds
+  -s, --sortby TEXT               Name of sort field
   -l, --lowercase                 Write column/properties names as lowercase
+  -m, --promote-to-multi          Promote features to multipart
   -v, --verbose                   Increase verbosity.
   -q, --quiet                     Decrease verbosity.
   --help                          Show this message and exit.
@@ -291,6 +295,7 @@ Usage: bcdata list [OPTIONS]
   List DataBC layers available via WFS
 
 Options:
+  -r, --refresh  Refresh the cached list
   --help         Show this message and exit.
 ```
 

{bcdata-0.13.0 → bcdata-0.14.0}/README.md

@@ -91,7 +91,8 @@ AERODROME_STATUS AIRCRAFT_ACCESS_IND                          AIRPORT_NAME
 ```
 
 ### CLI
-Commands available via the bcdata command line interface are documented with the `--help` option:
+
+Commands available via the bcdata command line interface are documented with the --help option
 
 ```
 
@@ -104,7 +105,7 @@ Options:
   --help     Show this message and exit.
 
 Commands:
-  bc2pg  Download a DataBC WFS layer to postgres
+  bc2pg  Load a DataBC WFS layer to a postgres db
   cat    Write DataBC features to stdout as GeoJSON feature objects.
   dem    Dump BC DEM to TIFF
   dump   Write DataBC features to stdout as GeoJSON feature collection.
@@ -119,7 +120,7 @@ $ bcdata bc2pg --help
 
 Usage: bcdata bc2pg [OPTIONS] DATASET
 
-  Download a DataBC WFS layer to postgres
+  Load a DataBC WFS layer to a postgres db
 
    $ bcdata bc2pg whse_imagery_and_base_maps.gsr_airports_svw
 
@@ -158,16 +159,18 @@ Usage: bcdata cat [OPTIONS] DATASET
 
 Options:
   --query TEXT                    A valid CQL or ECQL query
+  -c, --count INTEGER             Number of features to request and dump
   --bounds TEXT                   Bounds: "left bottom right top" or "[left,
                                   bottom, right, top]". Coordinates are BC
                                   Albers (default) or --bounds_crs
+  --bounds-crs, --bounds_crs TEXT
+                                  CRS of provided bounds
   --indent INTEGER                Indentation level for JSON output
   --compact / --not-compact       Use compact separators (',', ':').
   --dst-crs, --dst_crs TEXT       Destination CRS
   -s, --sortby TEXT               Name of sort field
-  --bounds-crs, --bounds_crs TEXT
-                                  CRS of provided bounds
   -l, --lowercase                 Write column/properties names as lowercase
+  -m, --promote-to-multi          Promote features to multipart
   -v, --verbose                   Increase verbosity.
   -q, --quiet                     Decrease verbosity.
   --help                          Show this message and exit.
@@ -187,9 +190,8 @@ Options:
   --bounds TEXT                   Bounds: "left bottom right top" or "[left,
                                   bottom, right, top]". Coordinates are BC
                                   Albers (default) or --bounds_crs  [required]
-  --dst-crs, --dst_crs TEXT       Destination CRS
-  --bounds-crs, --bounds_crs TEXT
-                                  CRS of provided bounds
+  --dst-crs TEXT                  CRS of output file
+  --bounds-crs TEXT               CRS of provided bounds
   -r, --resolution INTEGER
   -a, --align                     Align provided bounds to provincial standard
   -i, --interpolation [nearest|bilinear|bicubic]
@@ -211,18 +213,20 @@ Usage: bcdata dump [OPTIONS] DATASET
     $ bcdata dump bc-airports --query "AIRPORT_NAME='Victoria Harbour (Shoal Point) Heliport'"
     $ bcdata dump bc-airports --bounds xmin ymin xmax ymax
 
-   It can also be combined to read bounds of a feature dataset using Fiona:
+   It can also be combined to read bounds of a feature dataset using Fiona: 
    $ bcdata dump bc-airports --bounds $(fio info aoi.shp --bounds)
 
 Options:
   --query TEXT                    A valid CQL or ECQL query
-  -o, --out_file TEXT             Output file
+  -c, --count INTEGER             Number of features to request and dump
   --bounds TEXT                   Bounds: "left bottom right top" or "[left,
                                   bottom, right, top]". Coordinates are BC
                                   Albers (default) or --bounds_crs
   --bounds-crs, --bounds_crs TEXT
                                   CRS of provided bounds
+  -s, --sortby TEXT               Name of sort field
   -l, --lowercase                 Write column/properties names as lowercase
+  -m, --promote-to-multi          Promote features to multipart
   -v, --verbose                   Increase verbosity.
   -q, --quiet                     Decrease verbosity.
   --help                          Show this message and exit.
@@ -259,6 +263,7 @@ Usage: bcdata list [OPTIONS]
   List DataBC layers available via WFS
 
 Options:
+  -r, --refresh  Refresh the cached list
   --help         Show this message and exit.
 ```
 

{bcdata-0.13.0 → bcdata-0.14.0}/src/bcdata/__init__.py

@@ -4,10 +4,8 @@ from .bc2pg import bc2pg as bc2pg
 from .bcdc import get_table_definition as get_table_definition
 from .bcdc import get_table_name as get_table_name
 from .wcs import get_dem as get_dem
-from .wfs import define_requests as define_requests
 from .wfs import get_count as get_count
 from .wfs import get_data as get_data
-from .wfs import get_features as get_features
 from .wfs import get_sortkey as get_sortkey
 from .wfs import list_tables as list_tables
 from .wfs import validate_name as validate_name
@@ -24,4 +22,4 @@ else:
     raise Exception(f"Failed to download primary key database at {PRIMARY_KEY_DB_URL}")
     primary_keys = {}
 
-__version__ = "0.13.0"
+__version__ = "0.14.0"

{bcdata-0.13.0 → bcdata-0.14.0}/src/bcdata/bc2pg.py

@@ -73,14 +73,8 @@ def bc2pg(  # noqa: C901
     WFS = BCWFS()
 
     # define requests
-    urls = bcdata.define_requests(
-        dataset,
-        query=query,
-        bounds=bounds,
-        bounds_crs=bounds_crs,
-        count=count,
-        sortby=sortby,
-        crs="epsg:3005",
+    urls = WFS.define_requests(
+        dataset, query=query, bounds=bounds, bounds_crs=bounds_crs, count=count, sortby=sortby
     )
 
     df = None  # just for tracking if first download is done by geometry type check
@@ -101,7 +95,7 @@ def bc2pg(  # noqa: C901
 
         # if geometry type is not provided, determine type by making the first request
         if not geometry_type:
-            df = WFS.make_requests(dataset=dataset, urls=[urls[0]], as_gdf=True, crs="epsg:3005", lowercase=True)
+            df = WFS.request_features(url=urls[0], as_gdf=True, crs="epsg:3005", lowercase=True)
             geometry_type = df.geom_type.unique()[0]  # keep only the first type
             if numpy.any(df.has_z.unique()[0]):  # geopandas does not include Z in geom_type string
                 geometry_type = geometry_type + "Z"
@@ -110,9 +104,8 @@ def bc2pg(  # noqa: C901
         # (in case all entrys with geom are near the bottom)
         if not geometry_type:
             if not urls[-1] == urls[0]:
-                df_temp = WFS.make_requests(
-                    dataset=dataset,
-                    urls=[urls[-1]],
+                df_temp = WFS.request_features(
+                    url=urls[-1],
                     as_gdf=True,
                     crs="epsg:3005",
                     lowercase=True,
@@ -165,7 +158,7 @@ def bc2pg(  # noqa: C901
         for n, url in enumerate(urls):
             # if first url not downloaded above when checking geom type, do now
             if df is None:
-                df = WFS.make_requests(dataset=dataset, urls=[url], as_gdf=True, crs="epsg:3005", lowercase=True)
+                df = WFS.request_features(url=url, as_gdf=True, crs="epsg:3005", lowercase=True)
             # tidy the resulting dataframe
             df = df.rename_geometry("geom")
             # lowercasify

{bcdata-0.13.0 → bcdata-0.14.0}/src/bcdata/cli.py

@@ -6,12 +6,6 @@ import sys
 
 import click
 from cligj import compact_opt, indent_opt, quiet_opt, verbose_opt
-from shapely.geometry.linestring import LineString
-from shapely.geometry.multilinestring import MultiLineString
-from shapely.geometry.multipoint import MultiPoint
-from shapely.geometry.multipolygon import MultiPolygon
-from shapely.geometry.point import Point
-from shapely.geometry.polygon import Polygon
 
 import bcdata
 from bcdata.database import Database
@@ -27,6 +21,7 @@ def configure_logging(verbosity):
 def complete_dataset_names(ctx, param, incomplete):
     return [k for k in bcdata.list_tables() if k.startswith(incomplete)]
 
+
 # bounds handling direct from rasterio
 # https://github.com/mapbox/rasterio/blob/master/rasterio/rio/options.py
 # https://github.com/mapbox/rasterio/blob/master/rasterio/rio/clip.py
@@ -201,7 +196,13 @@ def dem(
     "--query",
     help="A valid CQL or ECQL query",
 )
-@click.option("--out_file", "-o", help="Output file")
+@click.option(
+    "--count",
+    "-c",
+    default=None,
+    type=int,
+    help="Number of features to request and dump",
+)
 @bounds_opt
 @click.option(
     "--bounds-crs",
@@ -209,17 +210,20 @@ def dem(
     help="CRS of provided bounds",
     default="EPSG:3005",
 )
+@click.option("--sortby", "-s", help="Name of sort field")
+@lowercase_opt
 @click.option(
-    "--no-clean",
-    "-nc",
-    help="Do not do any data standardization",
+    "--promote-to-multi",
+    "-m",
+    help="Promote features to multipart",
     is_flag=True,
-    default=True,
+    default=False,
 )
-@lowercase_opt
 @verbose_opt
 @quiet_opt
-def dump(dataset, query, out_file, bounds, bounds_crs, no_clean, lowercase, verbose, quiet):
+def dump(
+    dataset, query, count, bounds, bounds_crs, sortby, lowercase, promote_to_multi, verbose, quiet
+):
     """Write DataBC features to stdout as GeoJSON feature collection.
 
     \b
@@ -235,19 +239,19 @@ def dump(dataset, query, out_file, bounds, bounds_crs, no_clean, lowercase, verb
     verbosity = verbose - quiet
     configure_logging(verbosity)
     table = bcdata.validate_name(dataset)
-    if no_clean:
-        clean = False
-    else:
-        clean = True
     data = bcdata.get_data(
-        table, query=query, bounds=bounds, bounds_crs=bounds_crs, lowercase=lowercase, clean=clean
+        table,
+        query=query,
+        count=count,
+        bounds=bounds,
+        bounds_crs=bounds_crs,
+        sortby=sortby,
+        lowercase=lowercase,
+        promote_to_multi=promote_to_multi,
+        as_gdf=False,
     )
-    if out_file:
-        with open(out_file, "w") as sink:
-            sink.write(json.dumps(data))
-    else:
-        sink = click.get_text_stream("stdout")
-        sink.write(json.dumps(data))
+    sink = click.get_text_stream("stdout")
+    sink.write(json.dumps(data))
 
 
 @cli.command()
@@ -256,23 +260,38 @@ def dump(dataset, query, out_file, bounds, bounds_crs, no_clean, lowercase, verb
     "--query",
     help="A valid CQL or ECQL query",
 )
+@click.option(
+    "--count",
+    "-c",
+    default=None,
+    type=int,
+    help="Number of features to request and dump",
+)
 @bounds_opt
-@indent_opt
-@compact_opt
-@dst_crs_opt
-@click.option("--sortby", "-s", help="Name of sort field")
 @click.option(
     "--bounds-crs",
     "--bounds_crs",
     help="CRS of provided bounds",
     default="EPSG:3005",
 )
+@indent_opt
+@compact_opt
+@dst_crs_opt
+@click.option("--sortby", "-s", help="Name of sort field")
 @lowercase_opt
+@click.option(
+    "--promote-to-multi",
+    "-m",
+    help="Promote features to multipart",
+    is_flag=True,
+    default=False,
+)
 @verbose_opt
 @quiet_opt
 def cat(
     dataset,
     query,
+    count,
     bounds,
     bounds_crs,
     indent,
@@ -280,6 +299,7 @@ def cat(
     dst_crs,
     sortby,
     lowercase,
+    promote_to_multi,
     verbose,
     quiet,
 ):
@@ -295,23 +315,23 @@ def cat(
     if compact:
         dump_kwds["separators"] = (",", ":")
     table = bcdata.validate_name(dataset)
-    for feat in bcdata.get_features(
+    WFS = bcdata.wfs.BCWFS()
+    for url in WFS.define_requests(
         table,
         query=query,
+        count=count,
         bounds=bounds,
         bounds_crs=bounds_crs,
-        sortby=sortby,
-        crs=dst_crs,
-        lowercase=lowercase,
     ):
-        click.echo(json.dumps(feat, **dump_kwds))
-
-
-@cli.command()
-@verbose_opt
-@quiet_opt
-def clear_cache(verbose, quiet):
-    bcdata.clear_cache()
+        featurecollection = WFS.request_features(
+            url=url,
+            as_gdf=False,
+            lowercase=lowercase,
+            crs=dst_crs,
+            promote_to_multi=promote_to_multi,
+        )
+        for feat in featurecollection["features"]:
+            click.echo(json.dumps(feat, **dump_kwds))
 
 
 @cli.command()

{bcdata-0.13.0 → bcdata-0.14.0}/src/bcdata/wfs.py

@@ -10,6 +10,7 @@ from pathlib import Path
 from urllib.parse import urlencode
 
 import geopandas as gpd
+import pandas as pd
 import requests
 import stamina
 from owslib.feature import schema as wfs_schema
@@ -30,26 +31,22 @@ if not sys.warnoptions:
 log = logging.getLogger(__name__)
 
 
-def ensure_single_geometry_type(df):
-    """If mix of single/multi part geometries are present, promote all geometries to multipart"""
-    geomtypes = sorted(
-        [t.upper() for t in df.geometry.geom_type.dropna(axis=0, how="all").unique()], key=len
-    )
-    if len(geomtypes) > 1 and geomtypes[1] == "MULTI" + geomtypes[0]:
-        df.geometry = [
-            MultiPoint([feature]) if isinstance(feature, Point) else feature
-            for feature in df.geometry
-        ]
-        df.geometry = [
-            MultiLineString([feature]) if isinstance(feature, LineString) else feature
-            for feature in df.geometry
-        ]
-        df.geometry = [
-            MultiPolygon([feature]) if isinstance(feature, Polygon) else feature
-            for feature in df.geometry
-        ]
+def promote_gdf_to_multi(df):
+    """Promote all features to multipart"""
+    df.geometry = [
+        MultiPoint([feature]) if isinstance(feature, Point) else feature for feature in df.geometry
+    ]
+    df.geometry = [
+        MultiLineString([feature]) if isinstance(feature, LineString) else feature
+        for feature in df.geometry
+    ]
+    df.geometry = [
+        MultiPolygon([feature]) if isinstance(feature, Polygon) else feature
+        for feature in df.geometry
+    ]
     return df
 
+
 class ServiceException(Exception):
     pass
 
@@ -158,7 +155,7 @@ class BCWFS(object):
 
     @stamina.retry(on=requests.HTTPError, timeout=60)
     def _request_features(self, url, silent=False):
-        """Submit a getfeature request to DataBC WFS and return features"""
+        """Submit a getfeature request to DataBC WFS and return feature collection"""
         r = requests.get(url, headers=self.request_headers)
         if not silent:
             log.info(r.url)
@@ -176,6 +173,26 @@ class BCWFS(object):
             r.raise_for_status()
         return r.json()["features"]
 
+    @stamina.retry(on=requests.HTTPError, timeout=60)
+    def _request_featurecollection(self, url, silent=False):
+        """Submit a getfeature request to DataBC WFS and return feature collection"""
+        r = requests.get(url, headers=self.request_headers)
+        if not silent:
+            log.info(r.url)
+        else:
+            log.debug(r.url)
+        if r.status_code in [400, 401, 404]:
+            log.error(f"HTTP error {r.status_code}")
+            log.error(f"Response headers: {r.headers}")
+            log.error(f"Response text: {r.text}")
+            raise ServiceException(r.text)  # presumed request error
+        elif r.status_code in [500, 502, 503, 504]:  # presumed serivce error, retry
+            log.warning(f"HTTP error: {r.status_code}")
+            log.warning(f"Response headers: {r.headers}")
+            log.warning(f"Response text: {r.text}")
+            r.raise_for_status()
+        return r.json()
+
     def build_bounds_filter(self, query, bounds, bounds_crs, geom_column):
         """The bbox param shortcut is mutually exclusive with CQL_FILTER,
         combine query and bounds into a single CQL_FILTER expression
@@ -274,7 +291,6 @@ class BCWFS(object):
         self,
         dataset,
         query=None,
-        crs="epsg:4326",
         bounds=None,
         bounds_crs="EPSG:3005",
         count=None,
@@ -342,7 +358,7 @@ class BCWFS(object):
                 "request": "GetFeature",
                 "typeName": table,
                 "outputFormat": "json",
-                "SRSNAME": crs,
+                "SRSNAME": "EPSG:4326",
             }
             if sortby:
                 request["sortby"] = sortby.upper()
@@ -364,133 +380,79 @@ class BCWFS(object):
             urls.append(self.wfs_url + "?" + urlencode(request, doseq=True))
         return urls
 
-    def make_requests(
-        self, dataset, urls, as_gdf=False, crs="epsg4326", lowercase=False, silent=False, clean=True
+    def request_features(
+        self,
+        url,
+        as_gdf=False,
+        crs="EPSG:4326",
+        lowercase=False,
+        promote_to_multi=False,
     ):
-        """turn urls into data"""
-        # loop through urls
-        results = []
-        for url in urls:
-            results.append(self._request_features(url, silent))
-        outjson = dict(type="FeatureCollection", features=[])
-        for result in results:
-            outjson["features"] += result
-
-        # if specified, lowercasify all properties
-        if lowercase:
-            for feature in outjson["features"]:
-                feature["properties"] = {
-                    k.lower(): v for k, v in feature["properties"].items()
-                }
-
-        # load to geodataframe, standardize data slightly
-        if len(outjson["features"]) > 0:
-            gdf = gpd.GeoDataFrame.from_features(outjson)
-            gdf.crs = crs
-            # minor data cleaning as default
-            if clean:
-                if gdf.geometry.name != "geometry":
-                    gdf = gdf.rename_geometry("geometry")
-                gdf = ensure_single_geometry_type(gdf)
-                table_definition = bcdata.get_table_definition(dataset)
-                column_names = [
-                    c["column_name"]
-                    for c in table_definition["schema"]
-                    if c["column_name"] not in ["FEATURE_AREA_SQM", "FEATURE_LENGTH_M"]
-                    and c["data_type"] in ["NUMBER", "VARCHAR2", "DATE"]
-                ]
-                if lowercase:
-                    column_names = [c.lower() for c in column_names]
-                gdf = gdf[column_names + ["geometry"]]
+        """Make a request to WFS and return data as a FeatureCollection or a GeoDataFrame"""
+        # get the data
+        featurecollection = self._request_featurecollection(url)
+
+        # load to gdf for reprojection/minor data cleaning
+        if len(featurecollection["features"]) > 0:
+            gdf = gpd.GeoDataFrame.from_features(featurecollection)
+            gdf = gdf.set_crs("EPSG:4326")
+            if crs != "EPSG:4326":
+                gdf = gdf.to_crs(crs)
+            if gdf.geometry.name != "geometry":
+                gdf = gdf.rename_geometry("geometry")
+            if lowercase:
+                gdf.columns = [c.lower() for c in gdf.columns]
+            if promote_to_multi:
+                gdf = promote_gdf_to_multi(gdf)
         else:
             gdf = gpd.GeoDataFrame()
 
         if as_gdf:
             return gdf
-
         else:
             return json.loads(gdf.to_json())
 
-    def get_data(
-        self,
-        dataset,
-        query=None,
-        crs="epsg:4326",
-        bounds=None,
-        bounds_crs="epsg:3005",
-        count=None,
-        sortby=None,
-        as_gdf=False,
-        lowercase=False,
-        clean=True
-    ):
-        """Request features from DataBC WFS and return GeoJSON featurecollection or geodataframe"""
-        dataset = self.validate_name(dataset)
-        urls = self.define_requests(
-            dataset,
-            query=query,
-            crs=crs,
-            bounds=bounds,
-            bounds_crs=bounds_crs,
-            count=count,
-            sortby=sortby,
-        )
-        return self.make_requests(dataset, urls, as_gdf=as_gdf, crs=crs, lowercase=lowercase, clean=clean)
-
-    def get_features(
-        self,
-        dataset,
-        query=None,
-        crs="epsg:4326",
-        bounds=None,
-        bounds_crs="epsg:3005",
-        count=None,
-        sortby=None,
-        lowercase=False,
-        check_count=True,
-    ):
-        """Yield features from DataBC WFS"""
-        urls = self.define_requests(
-            dataset,
-            query=query,
-            crs=crs,
-            bounds=bounds,
-            bounds_crs=bounds_crs,
-            count=count,
-            sortby=sortby,
-            check_count=check_count,
-        )
-        for url in urls:
-            for feature in self._request_features(url):
-                if lowercase:
-                    feature["properties"] = {k.lower(): v for k, v in feature["properties"].items()}
-                yield feature
-
-
-# abstract away the WFS object
-
 
-def define_requests(
+def get_data(
     dataset,
     query=None,
     crs="epsg:4326",
     bounds=None,
-    bounds_crs="EPSG:3005",
+    bounds_crs="epsg:3005",
     count=None,
     sortby=None,
-    check_count=True,
+    as_gdf=False,
+    lowercase=False,
+    promote_to_multi=False,
 ):
+    """Request features from DataBC WFS, returning GeoJSON featurecollection or geodataframe"""
     WFS = BCWFS()
-    return WFS.define_requests(
-        dataset,
+    table = WFS.validate_name(dataset)
+    urls = WFS.define_requests(
+        table,
         query=query,
-        crs=crs,
         bounds=bounds,
         bounds_crs=bounds_crs,
         count=count,
         sortby=sortby,
-        check_count=check_count,
     )
+    results = []
+    for url in urls:
+        results.append(
+            WFS.request_features(
+                url, crs=crs, as_gdf=True, lowercase=lowercase, promote_to_multi=promote_to_multi
+            )
+        )
+    if len(results) > 1:
+        gdf = pd.concat(results)
+    elif len(results) == 1:
+        gdf = results[0]
+    else:
+        gdf = gpd.GeoDataFrame()
+    if as_gdf:
+        return gdf
+    else:
+        return json.loads(gdf.to_json())
 
 
 def get_count(dataset, query=None, bounds=None, bounds_crs="EPSG:3005"):
@@ -506,58 +468,6 @@ def get_count(dataset, query=None, bounds=None, bounds_crs="EPSG:3005"):
     )
 
 
-def get_data(
-    dataset,
-    query=None,
-    crs="epsg:4326",
-    bounds=None,
-    bounds_crs="epsg:3005",
-    count=None,
-    sortby=None,
-    as_gdf=False,
-    lowercase=False,
-    clean=True
-):
-    WFS = BCWFS()
-    return WFS.get_data(
-        dataset,
-        query=query,
-        crs=crs,
-        bounds=bounds,
-        bounds_crs=bounds_crs,
-        count=count,
-        sortby=sortby,
-        as_gdf=as_gdf,
-        lowercase=lowercase,
-        clean=clean
-    )
-
-
-def get_features(
-    dataset,
-    query=None,
-    crs="epsg:4326",
-    bounds=None,
-    bounds_crs="epsg:3005",
-    count=None,
-    sortby=None,
-    lowercase=False,
-    check_count=True,
-):
-    WFS = BCWFS()
-    return WFS.get_features(
-        dataset,
-        query=query,
-        crs=crs,
-        bounds=bounds,
-        bounds_crs=bounds_crs,
-        count=count,
-        sortby=sortby,
-        lowercase=lowercase,
-        check_count=check_count,
-    )
-
-
 def get_sortkey(dataset):
     WFS = BCWFS()
     table = WFS.validate_name(dataset)

{bcdata-0.13.0 → bcdata-0.14.0/src/bcdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bcdata
-Version: 0.13.0
+Version: 0.14.0
 Summary: Download open data, monitor and report on changes
 Author-email: Simon Norris <snorris@hillcrestgeo.ca>
 Project-URL: Homepage, https://github.com/smnorris/bcdata
@@ -123,7 +123,8 @@ AERODROME_STATUS AIRCRAFT_ACCESS_IND                          AIRPORT_NAME
 ```
 
 ### CLI
-Commands available via the bcdata command line interface are documented with the `--help` option:
+
+Commands available via the bcdata command line interface are documented with the --help option
 
 ```
 
@@ -136,7 +137,7 @@ Options:
   --help     Show this message and exit.
 
 Commands:
-  bc2pg  Download a DataBC WFS layer to postgres
+  bc2pg  Load a DataBC WFS layer to a postgres db
   cat    Write DataBC features to stdout as GeoJSON feature objects.
   dem    Dump BC DEM to TIFF
   dump   Write DataBC features to stdout as GeoJSON feature collection.
@@ -151,7 +152,7 @@ $ bcdata bc2pg --help
 
 Usage: bcdata bc2pg [OPTIONS] DATASET
 
-  Download a DataBC WFS layer to postgres
+  Load a DataBC WFS layer to a postgres db
 
    $ bcdata bc2pg whse_imagery_and_base_maps.gsr_airports_svw
 
@@ -190,16 +191,18 @@ Usage: bcdata cat [OPTIONS] DATASET
 
 Options:
   --query TEXT                    A valid CQL or ECQL query
+  -c, --count INTEGER             Number of features to request and dump
   --bounds TEXT                   Bounds: "left bottom right top" or "[left,
                                   bottom, right, top]". Coordinates are BC
                                   Albers (default) or --bounds_crs
+  --bounds-crs, --bounds_crs TEXT
+                                  CRS of provided bounds
   --indent INTEGER                Indentation level for JSON output
   --compact / --not-compact       Use compact separators (',', ':').
   --dst-crs, --dst_crs TEXT       Destination CRS
   -s, --sortby TEXT               Name of sort field
-  --bounds-crs, --bounds_crs TEXT
-                                  CRS of provided bounds
   -l, --lowercase                 Write column/properties names as lowercase
+  -m, --promote-to-multi          Promote features to multipart
   -v, --verbose                   Increase verbosity.
   -q, --quiet                     Decrease verbosity.
   --help                          Show this message and exit.
@@ -219,9 +222,8 @@ Options:
   --bounds TEXT                   Bounds: "left bottom right top" or "[left,
                                   bottom, right, top]". Coordinates are BC
                                   Albers (default) or --bounds_crs  [required]
-  --dst-crs, --dst_crs TEXT       Destination CRS
-  --bounds-crs, --bounds_crs TEXT
-                                  CRS of provided bounds
+  --dst-crs TEXT                  CRS of output file
+  --bounds-crs TEXT               CRS of provided bounds
   -r, --resolution INTEGER
   -a, --align                     Align provided bounds to provincial standard
   -i, --interpolation [nearest|bilinear|bicubic]
@@ -243,18 +245,20 @@ Usage: bcdata dump [OPTIONS] DATASET
     $ bcdata dump bc-airports --query "AIRPORT_NAME='Victoria Harbour (Shoal Point) Heliport'"
     $ bcdata dump bc-airports --bounds xmin ymin xmax ymax
 
-   It can also be combined to read bounds of a feature dataset using Fiona:
+   It can also be combined to read bounds of a feature dataset using Fiona: 
    $ bcdata dump bc-airports --bounds $(fio info aoi.shp --bounds)
 
 Options:
   --query TEXT                    A valid CQL or ECQL query
-  -o, --out_file TEXT             Output file
+  -c, --count INTEGER             Number of features to request and dump
   --bounds TEXT                   Bounds: "left bottom right top" or "[left,
                                   bottom, right, top]". Coordinates are BC
                                   Albers (default) or --bounds_crs
   --bounds-crs, --bounds_crs TEXT
                                   CRS of provided bounds
+  -s, --sortby TEXT               Name of sort field
   -l, --lowercase                 Write column/properties names as lowercase
+  -m, --promote-to-multi          Promote features to multipart
   -v, --verbose                   Increase verbosity.
   -q, --quiet                     Decrease verbosity.
   --help                          Show this message and exit.
@@ -291,6 +295,7 @@ Usage: bcdata list [OPTIONS]
   List DataBC layers available via WFS
 
 Options:
+  -r, --refresh  Refresh the cached list
   --help         Show this message and exit.
 ```
 

{bcdata-0.13.0 → bcdata-0.14.0}/tests/test_cli.py

@@ -1,3 +1,4 @@
+import json
 import os
 
 from click.testing import CliRunner
@@ -73,6 +74,14 @@ def test_cat_bounds_ll():
     assert len(result.output.split("\n")) == 4
 
 
+def test_dump():
+    runner = CliRunner()
+    result = runner.invoke(cli, ["dump", AIRPORTS_TABLE, "--count", 1])
+    assert result.exit_code == 0
+    assert json.loads(result.output)["type"] == "FeatureCollection"
+    assert len(json.loads(result.output)["features"]) == 1
+
+
 def test_bc2pg():
     runner = CliRunner()
     result = runner.invoke(cli, ["bc2pg", AIRPORTS_TABLE, "--db_url", DB_URL])

{bcdata-0.13.0 → bcdata-0.14.0}/tests/test_wfs.py

@@ -1,5 +1,4 @@
 import pytest
-import json
 import requests
 import requests_mock
 import stamina
@@ -94,14 +93,7 @@ def test_get_data_lowercase():
 
 def test_get_data_crs():
     data = bcdata.get_data(AIRPORTS_TABLE, crs="EPSG:3005")
-    assert (
-        data["crs"]["properties"]["name"] == 'urn:ogc:def:crs:EPSG::3005'
-    )
-
-
-def test_get_features():
-    data = [f for f in bcdata.get_features(AIRPORTS_TABLE)]
-    assert len(data) == 455
+    assert data["crs"]["properties"]["name"] == "urn:ogc:def:crs:EPSG::3005"
 
 
 def test_get_data_count():
@@ -147,20 +139,3 @@ def test_cql_bounds_filter():
     )
     assert len(data["features"]) == 1
     assert data["features"][0]["properties"]["AIRPORT_NAME"] == "Victoria International Airport"
-
-
-def test_clean():
-    data = bcdata.get_data(
-        AIRPORTS_TABLE,
-        query="AIRPORT_NAME='Terrace (Northwest Regional) Airport'",
-    )
-    assert "SE_ANNO_CAD_DATA" not in data["features"][0]["properties"].keys()
-
-
-def test_no_clean():
-    data = bcdata.get_data(
-        AIRPORTS_TABLE,
-        query="AIRPORT_NAME='Terrace (Northwest Regional) Airport'",
-        clean=False
-    )
-    assert "SE_ANNO_CAD_DATA" in data["features"][0]["properties"].keys()