bcdata 0.11.0__tar.gz → 0.12.2__tar.gz

{bcdata-0.11.0 → bcdata-0.12.2}/CHANGES.txt

@@ -1,6 +1,18 @@
 Changes
 =======
 
+0.12.2 (2024-08-15)
+------------------
+- fix error where incorrect schema returned for datasets with multiple resources (#196)
+
+0.12.1 (2024-08-15)
+------------------
+- fix syntax error
+
+0.12.0 (2024-08-15)
+------------------
+- expose primary key database as bcdata.primary_keys and via bcdata info (#193)
+
 0.11.0 (2024-07-29)
 ------------------
 - upgrade dependencies

{bcdata-0.11.0/bcdata.egg-info → bcdata-0.12.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bcdata
-Version: 0.11.0
+Version: 0.12.2
 Summary: Python tools for quick access to DataBC geo-data available via WFS
 Home-page: https://github.com/smnorris/bcdata
 Author: Simon Norris
@@ -339,7 +339,7 @@ Describe a dataset. Note that if we know the id of a dataset, we can use that ra
 
 The JSON output can be manipulated with [jq](https://stedolan.github.io/jq/). For example, to show only the fields available in the dataset:
 
-    $ bcdata info bc-airports | jq '.schema.properties'
+    $ bcdata info bc-airports | jq '.schema'
     {
       "CUSTODIAN_ORG_DESCRIPTION": "string",
       "BUSINESS_CATEGORY_CLASS": "string",

{bcdata-0.11.0 → bcdata-0.12.2}/README.md

@@ -305,7 +305,7 @@ Describe a dataset. Note that if we know the id of a dataset, we can use that ra
 
 The JSON output can be manipulated with [jq](https://stedolan.github.io/jq/). For example, to show only the fields available in the dataset:
 
-    $ bcdata info bc-airports | jq '.schema.properties'
+    $ bcdata info bc-airports | jq '.schema'
     {
       "CUSTODIAN_ORG_DESCRIPTION": "string",
       "BUSINESS_CATEGORY_CLASS": "string",

bcdata-0.12.2/bcdata/__init__.py

@@ -0,0 +1,30 @@
+import requests
+
+from .bc2pg import bc2pg
+from .bcdc import get_table_definition, get_table_name
+from .wcs import get_dem
+from .wfs import (
+    define_requests,
+    get_count,
+    get_data,
+    get_features,
+    get_sortkey,
+    list_tables,
+    validate_name,
+)
+
+PRIMARY_KEY_DB_URL = (
+    "https://raw.githubusercontent.com/smnorris/bcdata/main/data/primary_keys.json"
+)
+
+# BCDC does not indicate which column in the schema is the primary key.
+# In this absence, bcdata maintains its own dictionary of {table: primary_key},
+# served via github. Retrieve the dict with this function"""
+response = requests.get(PRIMARY_KEY_DB_URL)
+if response.status_code == 200:
+    primary_keys = response.json()
+else:
+    raise Exception(f"Failed to download primary key database at {PRIMARY_KEY_DB_URL}")
+    primary_keys = {}
+
+__version__ = "0.12.2"

{bcdata-0.11.0 → bcdata-0.12.2}/bcdata/bc2pg.py

@@ -34,19 +34,6 @@ SUPPORTED_TYPES = [
 ]
 
 
-def get_primary_keys():
-    """download primary key data file"""
-    response = requests.get(bcdata.PRIMARY_KEY_DB_URL)
-    if response.status_code == 200:
-        primary_keys = response.json()
-    else:
-        log.warning(
-            f"Failed to download primary key database at {bcdata.PRIMARY_KEY_DB_URL}"
-        )
-        primary_keys = {}
-    return primary_keys
-
-
 def bc2pg(  # noqa: C901
     dataset,
     db_url,
@@ -148,9 +135,8 @@ def bc2pg(  # noqa: C901
             raise ValueError("Geometry type {geometry_type} is not supported")
 
         # if primary key is not supplied, use default (if present in list)
-        primary_keys = get_primary_keys()
-        if not primary_key and dataset.lower() in primary_keys:
-            primary_key = primary_keys[dataset.lower()]
+        if not primary_key and dataset.lower() in bcdata.primary_keys:
+            primary_key = bcdata.primary_keys[dataset.lower()]
 
         # fail if specified primary key is not in the table
         if primary_key and primary_key.upper() not in [

bcdata-0.12.2/bcdata/bcdc.py

@@ -0,0 +1,134 @@
+import json
+import logging
+from urllib.parse import urlparse
+
+import requests
+import stamina
+
+import bcdata
+
+log = logging.getLogger(__name__)
+
+BCDC_API_URL = "https://catalogue.data.gov.bc.ca/api/3/action/"
+
+
+class ServiceException(Exception):
+    pass
+
+
+@stamina.retry(on=requests.HTTPError, timeout=60)
+def _package_show(package):
+    r = requests.get(BCDC_API_URL + "package_show", params={"id": package})
+    if r.status_code in [400, 404]:
+        log.error(f"HTTP error {r.status_code}")
+        log.error(f"Response headers: {r.headers}")
+        log.error(f"Response text: {r.text}")
+        raise ValueError(f"Dataset {package} not found in DataBC API list")
+    if r.status_code in [500, 502, 503, 504]:  # presumed serivce error, retry
+        log.warning(f"HTTP error: {r.status_code}")
+        log.warning(f"Response headers: {r.headers}")
+        log.warning(f"Response text: {r.text}")
+        r.raise_for_status()
+    else:
+        log.debug(r.text)
+    return r
+
+
+@stamina.retry(on=requests.HTTPError, timeout=60)
+def _table_definition(table_name):
+    r = requests.get(
+        BCDC_API_URL + "package_search",
+        params={"q": "res_extras_object_name:" + table_name},
+    )
+    if r.status_code != 200:
+        log.warning(r.headers)
+    if r.status_code in [400, 401, 404]:
+        raise ServiceException(r.text)  # presumed request error
+    if r.status_code in [500, 502, 503, 504]:  # presumed serivce error, retry
+        r.raise_for_status()
+    return r
+
+
+def get_table_name(package):
+    """Query DataBC API to find WFS table/layer name for given package"""
+    package = package.lower()  # package names are lowercase
+    r = _package_show(package)
+    result = r.json()["result"]
+    # Because the object_name in the result json is not a 100% reliable key
+    # for WFS requests, parse URL in WMS resource(s).
+    # Also, some packages may have >1 WFS layer - if this is the case, bail
+    # and provide user with a list of layers
+    layer_urls = [r["url"] for r in result["resources"] if r["format"] == "wms"]
+    layer_names = [urlparse(url).path.split("/")[3] for url in layer_urls]
+    if len(layer_names) > 1:
+        raise ValueError(
+            "Package {} includes more than one WFS resource, specify one of the following: \n{}".format(
+                package, "\n".join(layer_names)
+            )
+        )
+    return layer_names[0]
+
+
+def get_table_definition(table_name):
+    """
+    Given a table/object name, search BCDC for the first package/resource with a matching "object_name",
+    returns dict: {"comments": <>, "notes": <>, "schema": {<schema dict>} }
+    """
+    # only allow searching for tables present in WFS list
+    table_name = table_name.upper()
+    if table_name not in bcdata.list_tables():
+        raise ValueError(
+            f"Only tables available via WFS are supported, {table_name} not found"
+        )
+
+    # search the api for the provided table
+    r = _table_definition(table_name)
+
+    # start with an empty table definition dict
+    table_definition = {
+        "description": None,
+        "comments": None,
+        "schema": [],
+        "primary_key": None,
+    }
+
+    # if there are no matching results, let the user know
+    if r.json()["result"]["count"] == 0:
+        log.warning(
+            f"BC Data Catalouge API search provides no results for: {table_name}"
+        )
+    else:
+        # iterate through results of search (packages)
+        for result in r.json()["result"]["results"]:
+            # description is at top level, same for all resources
+            table_definition["description"] = result["notes"]
+            # iterate through resources associated with each package
+            for resource in result["resources"]:
+                # only examine geographic resources with object name key
+                if (
+                    "object_name" in resource.keys()
+                    and resource["bcdc_type"] == "geographic"
+                ):
+                    # confirm that object name matches table name and schema is present
+                    if (
+                        resource["object_name"] == table_name
+                        and "details" in resource.keys()
+                        and resource["details"] != ""
+                    ):
+                        table_definition["schema"] = json.loads(resource["details"])
+                        # look for comments only if details/schema was found
+                        if "object_table_comments" in resource.keys():
+                            table_definition["comments"] = resource[
+                                "object_table_comments"
+                            ]
+
+    if not table_definition["schema"]:
+        log.warning(
+            f"BC Data Catalouge API search provides no schema for: {table_name}"
+        )
+
+    # add primary key if present in bcdata.primary_keys
+    if table_name.lower() in bcdata.primary_keys:
+        table_definition["primary_key"] = bcdata.primary_keys[table_name.lower()]
+
+    return table_definition

{bcdata-0.11.0 → bcdata-0.12.2}/bcdata/cli.py

@@ -131,14 +131,9 @@ def info(dataset, indent, meta_member, verbose, quiet):
     verbosity = verbose - quiet
     configure_logging(verbosity)
     dataset = bcdata.validate_name(dataset)
-    info = {}
+    info = bcdata.get_table_definition(dataset)
     info["name"] = dataset
     info["count"] = bcdata.get_count(dataset)
-    table_definition = bcdata.get_table_definition(dataset)
-    info["description"] = table_definition["description"]
-    info["table_comments"] = table_definition["comments"]
-    info["schema"] = table_definition["schema"]
-
     if meta_member:
         click.echo(info[meta_member])
     else:

{bcdata-0.11.0 → bcdata-0.12.2}/bcdata/wfs.py

@@ -223,9 +223,8 @@ class BCWFS(object):
         """Check data for unique columns available for sorting paged requests"""
         columns = list(self.get_schema(table)["properties"].keys())
         # use known primary key if it is present in the bcdata repository
-        known_primary_keys = bcdata.get_primary_keys()
-        if table.lower() in known_primary_keys:
-            return known_primary_keys[table.lower()].upper()
+        if table.lower() in bcdata.primary_keys:
+            return bcdata.primary_keys[table.lower()].upper()
         # if pk not known, use OBJECTID as default sort key when present
         elif "OBJECTID" in columns:
             return "OBJECTID"

{bcdata-0.11.0 → bcdata-0.12.2/bcdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bcdata
-Version: 0.11.0
+Version: 0.12.2
 Summary: Python tools for quick access to DataBC geo-data available via WFS
 Home-page: https://github.com/smnorris/bcdata
 Author: Simon Norris
@@ -339,7 +339,7 @@ Describe a dataset. Note that if we know the id of a dataset, we can use that ra
 
 The JSON output can be manipulated with [jq](https://stedolan.github.io/jq/). For example, to show only the fields available in the dataset:
 
-    $ bcdata info bc-airports | jq '.schema.properties'
+    $ bcdata info bc-airports | jq '.schema'
     {
       "CUSTODIAN_ORG_DESCRIPTION": "string",
       "BUSINESS_CATEGORY_CLASS": "string",

{bcdata-0.11.0 → bcdata-0.12.2}/tests/test_bc2pg.py

@@ -126,8 +126,7 @@ def test_bc2pg_primary_key():
 
 
 def test_bc2pg_get_primary_keys():
-    primary_keys = bcdata.get_primary_keys()
-    assert primary_keys[ASSESSMENTS_TABLE] == "stream_crossing_id"
+    assert bcdata.primary_keys[ASSESSMENTS_TABLE] == "stream_crossing_id"
 
 
 def test_bc2pg_primary_key_default():

{bcdata-0.11.0 → bcdata-0.12.2}/tests/test_bcdc.py

@@ -2,6 +2,7 @@ import json
 
 import pytest
 
+import bcdata
 from bcdata import bcdc
 
 AIRPORTS_PACKAGE = "bc-airports"
@@ -40,6 +41,13 @@ def test_get_table_definition_format_multi():
     assert table_definition["description"]
     assert table_definition["comments"]
     assert table_definition["schema"]
+    columns = [c["column_name"] for c in table_definition["schema"]]
+    assert (
+        bcdata.primary_keys[
+            "whse_forest_vegetation.ogsr_priority_def_area_cur_sp"
+        ].upper()
+        in columns
+    )
 
 
 def test_get_table_definition_format_multi_nopreview():

bcdata-0.11.0/bcdata/__init__.py

@@ -1,18 +0,0 @@
-from .bc2pg import bc2pg, get_primary_keys
-from .bcdc import get_table_definition, get_table_name
-from .wcs import get_dem
-from .wfs import (
-    define_requests,
-    get_count,
-    get_data,
-    get_features,
-    get_sortkey,
-    list_tables,
-    validate_name,
-)
-
-PRIMARY_KEY_DB_URL = (
-    "https://raw.githubusercontent.com/smnorris/bcdata/main/data/primary_keys.json"
-)
-
-__version__ = "0.11.0"

bcdata-0.11.0/bcdata/bcdc.py

@@ -1,157 +0,0 @@
-import json
-import logging
-from urllib.parse import urlparse
-
-import requests
-import stamina
-
-import bcdata
-
-log = logging.getLogger(__name__)
-
-BCDC_API_URL = "https://catalogue.data.gov.bc.ca/api/3/action/"
-
-
-class ServiceException(Exception):
-    pass
-
-
-@stamina.retry(on=requests.HTTPError, timeout=60)
-def _package_show(package):
-    r = requests.get(BCDC_API_URL + "package_show", params={"id": package})
-    if r.status_code in [400, 404]:
-        log.error(f"HTTP error {r.status_code}")
-        log.error(f"Response headers: {r.headers}")
-        log.error(f"Response text: {r.text}")
-        raise ValueError(f"Dataset {package} not found in DataBC API list")
-    if r.status_code in [500, 502, 503, 504]:  # presumed serivce error, retry
-        log.warning(f"HTTP error: {r.status_code}")
-        log.warning(f"Response headers: {r.headers}")
-        log.warning(f"Response text: {r.text}")
-        r.raise_for_status()
-    else:
-        log.debug(r.text)
-    return r
-
-
-@stamina.retry(on=requests.HTTPError, timeout=60)
-def _table_definition(table_name):
-    r = requests.get(BCDC_API_URL + "package_search", params={"q": table_name})
-    if r.status_code != 200:
-        log.warning(r.headers)
-    if r.status_code in [400, 401, 404]:
-        raise ServiceException(r.text)  # presumed request error
-    if r.status_code in [500, 502, 503, 504]:  # presumed serivce error, retry
-        r.raise_for_status()
-    return r
-
-
-def get_table_name(package):
-    """Query DataBC API to find WFS table/layer name for given package"""
-    package = package.lower()  # package names are lowercase
-    r = _package_show(package)
-    result = r.json()["result"]
-    # Because the object_name in the result json is not a 100% reliable key
-    # for WFS requests, parse URL in WMS resource(s).
-    # Also, some packages may have >1 WFS layer - if this is the case, bail
-    # and provide user with a list of layers
-    layer_urls = [r["url"] for r in result["resources"] if r["format"] == "wms"]
-    layer_names = [urlparse(url).path.split("/")[3] for url in layer_urls]
-    if len(layer_names) > 1:
-        raise ValueError(
-            "Package {} includes more than one WFS resource, specify one of the following: \n{}".format(
-                package, "\n".join(layer_names)
-            )
-        )
-    return layer_names[0]
-
-
-def get_table_definition(table_name):  # noqa: C901
-    """
-    Given a table/object name, search BCDC for the first package/resource with a matching "object_name",
-    returns dict: {"comments": <>, "notes": <>, "schema": {<schema dict>} }
-    """
-    # only allow searching for tables present in WFS list
-    table_name = table_name.upper()
-    if table_name not in bcdata.list_tables():
-        raise ValueError(
-            f"Only tables available via WFS are supported, {table_name} not found"
-        )
-    # search the api for the provided table
-    r = _table_definition(table_name)
-    # if there are no matching results, let the user know
-    if r.json()["result"]["count"] == 0:
-        log.warning(
-            f"BC Data Catalouge API search provides no results for: {table_name}"
-        )
-        return []
-    else:
-        matches = []
-        # iterate through results of search (packages)
-        for result in r.json()["result"]["results"]:
-            notes = result["notes"]
-            # iterate through resources associated with each package
-            for resource in result["resources"]:
-                # where to find schema details depends on format type
-                if resource["format"] == "wms":
-                    if urlparse(resource["url"]).path.split("/")[3] == table_name:
-                        if "object_table_comments" in resource.keys():
-                            table_comments = resource["object_table_comments"]
-                        else:
-                            table_comments = None
-                        # only add to matches if schema details found
-                        if "details" in resource.keys() and resource["details"] != "":
-                            table_details = resource["details"]
-                            matches.append((notes, table_comments, table_details))
-                            log.debug(resource)
-                # oracle sde format type
-                if resource["format"] == "oracle_sde":
-                    if resource["object_name"] == table_name:
-                        if "object_table_comments" in resource.keys():
-                            table_comments = resource["object_table_comments"]
-                        else:
-                            table_comments = None
-                        # only add to matches if schema details found
-                        if "details" in resource.keys() and resource["details"] != "":
-                            table_details = resource["details"]
-                            matches.append((notes, table_comments, table_details))
-                            log.debug(resource)
-
-                # multiple format resource
-                elif resource["format"] == "multiple":
-                    # if multiple format, check for table name match in this location
-                    if resource["preview_info"]:
-                        # check that layer_name key is present
-                        if "layer_name" in json.loads(resource["preview_info"]):
-                            # then check if it matches the table name
-                            if (
-                                json.loads(resource["preview_info"])["layer_name"]
-                                == table_name
-                            ):
-                                if "object_table_comments" in resource.keys():
-                                    table_comments = resource["object_table_comments"]
-                                else:
-                                    table_comments = None
-                                # only add to matches if schema details found
-                                if (
-                                    "details" in resource.keys()
-                                    and resource["details"] != ""
-                                ):
-                                    table_details = resource["details"]
-                                    matches.append(
-                                        (notes, table_comments, table_details)
-                                    )
-                                    log.debug(resource)
-
-        # uniquify the result
-        if len(matches) > 0:
-            matched = list(set(matches))[0]
-            return {
-                "description": matched[0],  # notes=description
-                "comments": matched[1],
-                "schema": json.loads(matched[2]),
-            }
-        else:
-            raise ValueError(
-                f"BCDC search for {table_name} does not return a table schema"
-            )