bcdata 0.10.2__tar.gz → 0.10.4__tar.gz

{bcdata-0.10.2 → bcdata-0.10.4}/CHANGES.txt

@@ -1,6 +1,16 @@
 Changes
 =======
 
+0.10.4 (2024-06-28)
+------------------
+- bc2pg - enable --no_timestamp on --refresh
+
+0.10.3 (2024-06-28)
+------------------
+- bc2pg - fix logging error on data refresh (#184)
+- fix failures on specific tables due to API quirks (#179/#181)
+- bc2pg - add missing --bounds option (#173)
+
 0.10.2 (2024-03-01)
 ------------------
 - improve connection management for better stability (#171)

{bcdata-0.10.2/bcdata.egg-info → bcdata-0.10.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bcdata
-Version: 0.10.2
+Version: 0.10.4
 Summary: Python tools for quick access to DataBC geo-data available via WFS
 Home-page: https://github.com/smnorris/bcdata
 Author: Simon Norris
@@ -153,25 +153,30 @@ Usage: bcdata bc2pg [OPTIONS] DATASET
 
   Download a DataBC WFS layer to postgres
 
-   $ bcdata bc2pg bc-airports --db_url postgresql://postgres:postgres@localhost:5432/postgis
+   $ bcdata bc2pg whse_imagery_and_base_maps.gsr_airports_svw
 
 Options:
-  -db, --db_url TEXT      Target database url, defaults to $DATABASE_URL
-                          environment variable if set
-  --table TEXT            Destination table name
-  --schema TEXT           Destination schema name
-  --geometry_type TEXT    Spatial type of geometry column
-  --query TEXT            A valid CQL or ECQL query
-  -c, --count INTEGER     Total number of features to load
-  -k, --primary_key TEXT  Primary key of dataset
-  -s, --sortby TEXT       Name of sort field
-  -e, --schema_only       Create empty table from catalogue schema
-  -a, --append            Append to existing table
-  -r, --refresh           Truncate from existing table before load
-  -t, --no_timestamp      Do not log download to bcdata.log
-  -v, --verbose           Increase verbosity.
-  -q, --quiet             Decrease verbosity.
-  --help                  Show this message and exit.
+  -db, --db_url TEXT              Target database url, defaults to
+                                  $DATABASE_URL environment variable if set
+  --table TEXT                    Destination table name
+  --schema TEXT                   Destination schema name
+  --geometry_type TEXT            Spatial type of geometry column
+  --query TEXT                    A valid CQL or ECQL query
+  --bounds TEXT                   Bounds: "left bottom right top" or "[left,
+                                  bottom, right, top]". Coordinates are BC
+                                  Albers (default) or --bounds_crs
+  --bounds-crs, --bounds_crs TEXT
+                                  CRS of provided bounds
+  -c, --count INTEGER             Total number of features to load
+  -k, --primary_key TEXT          Primary key of dataset
+  -s, --sortby TEXT               Name of sort field
+  -e, --schema_only               Create empty table from catalogue schema
+  -a, --append                    Append to existing table
+  -r, --refresh                   Truncate from existing table before load
+  -t, --no_timestamp              Do not log download to bcdata.log
+  -v, --verbose                   Increase verbosity.
+  -q, --quiet                     Decrease verbosity.
+  --help                          Show this message and exit.
 ```
 
 #### cat

{bcdata-0.10.2 → bcdata-0.10.4}/README.md

@@ -119,25 +119,30 @@ Usage: bcdata bc2pg [OPTIONS] DATASET
 
   Download a DataBC WFS layer to postgres
 
-   $ bcdata bc2pg bc-airports --db_url postgresql://postgres:postgres@localhost:5432/postgis
+   $ bcdata bc2pg whse_imagery_and_base_maps.gsr_airports_svw
 
 Options:
-  -db, --db_url TEXT      Target database url, defaults to $DATABASE_URL
-                          environment variable if set
-  --table TEXT            Destination table name
-  --schema TEXT           Destination schema name
-  --geometry_type TEXT    Spatial type of geometry column
-  --query TEXT            A valid CQL or ECQL query
-  -c, --count INTEGER     Total number of features to load
-  -k, --primary_key TEXT  Primary key of dataset
-  -s, --sortby TEXT       Name of sort field
-  -e, --schema_only       Create empty table from catalogue schema
-  -a, --append            Append to existing table
-  -r, --refresh           Truncate from existing table before load
-  -t, --no_timestamp      Do not log download to bcdata.log
-  -v, --verbose           Increase verbosity.
-  -q, --quiet             Decrease verbosity.
-  --help                  Show this message and exit.
+  -db, --db_url TEXT              Target database url, defaults to
+                                  $DATABASE_URL environment variable if set
+  --table TEXT                    Destination table name
+  --schema TEXT                   Destination schema name
+  --geometry_type TEXT            Spatial type of geometry column
+  --query TEXT                    A valid CQL or ECQL query
+  --bounds TEXT                   Bounds: "left bottom right top" or "[left,
+                                  bottom, right, top]". Coordinates are BC
+                                  Albers (default) or --bounds_crs
+  --bounds-crs, --bounds_crs TEXT
+                                  CRS of provided bounds
+  -c, --count INTEGER             Total number of features to load
+  -k, --primary_key TEXT          Primary key of dataset
+  -s, --sortby TEXT               Name of sort field
+  -e, --schema_only               Create empty table from catalogue schema
+  -a, --append                    Append to existing table
+  -r, --refresh                   Truncate from existing table before load
+  -t, --no_timestamp              Do not log download to bcdata.log
+  -v, --verbose                   Increase verbosity.
+  -q, --quiet                     Decrease verbosity.
+  --help                          Show this message and exit.
 ```
 
 #### cat

{bcdata-0.10.2 → bcdata-0.10.4}/bcdata/__init__.py

@@ -15,4 +15,4 @@ PRIMARY_KEY_DB_URL = (
     "https://raw.githubusercontent.com/smnorris/bcdata/main/data/primary_keys.json"
 )
 
-__version__ = "0.10.2"
+__version__ = "0.10.4"

{bcdata-0.10.2 → bcdata-0.10.4}/bcdata/bc2pg.py

@@ -54,12 +54,15 @@ def bc2pg(  # noqa: C901
     schema=None,
     geometry_type=None,
     query=None,
+    bounds=None,
+    bounds_crs=None,
     count=None,
     sortby=None,
     primary_key=None,
     timestamp=True,
     schema_only=False,
     append=False,
+    refresh=False,
 ):
     """Request table definition from bcdc and replicate in postgres"""
     if schema_only and append:
@@ -82,6 +85,8 @@ def bc2pg(  # noqa: C901
     urls = bcdata.define_requests(
         dataset,
         query=query,
+        bounds=bounds,
+        bounds_crs=bounds_crs,
         count=count,
         sortby=sortby,
         crs="epsg:3005",
@@ -224,22 +229,8 @@ def bc2pg(  # noqa: C901
             df = None
 
         # once load complete, note date/time of load completion in bcdata.log
-        if timestamp:
-            log.info("Logging download date to bcdata.log")
-            db.execute(
-                """CREATE SCHEMA IF NOT EXISTS bcdata;
-                   CREATE TABLE IF NOT EXISTS bcdata.log (
-                     table_name text PRIMARY KEY,
-                     latest_download timestamp WITH TIME ZONE
-                   );
-                """
-            )
-            db.execute(
-                """INSERT INTO bcdata.log (table_name, latest_download)
-                   SELECT %s as table_name, NOW() as latest_download
-                   ON CONFLICT (table_name) DO UPDATE SET latest_download = NOW();
-                """,
-                (schema_name + "." + table_name,),
-            )
+        # do not log refreshes here, they called by cli once loaded to target table
+        if timestamp and not refresh:
+            db.log(schema_name, table_name)
 
     return schema_name + "." + table_name

{bcdata-0.10.2 → bcdata-0.10.4}/bcdata/bcdc.py

@@ -66,7 +66,7 @@ def get_table_name(package):
     return layer_names[0]
 
 
-def get_table_definition(table_name):
+def get_table_definition(table_name):  # noqa: C901
     """
     Given a table/object name, search BCDC for the first package/resource with a matching "object_name",
     returns dict: {"comments": <>, "notes": <>, "schema": {<schema dict>} }
@@ -92,9 +92,8 @@ def get_table_definition(table_name):
             notes = result["notes"]
             # iterate through resources associated with each package
             for resource in result["resources"]:
-                # wms format resource
+                # where to find schema details depends on format type
                 if resource["format"] == "wms":
-                    # if wms, check for table name match in this location
                     if urlparse(resource["url"]).path.split("/")[3] == table_name:
                         if "object_table_comments" in resource.keys():
                             table_comments = resource["object_table_comments"]
@@ -105,29 +104,44 @@ def get_table_definition(table_name):
                             table_details = resource["details"]
                             matches.append((notes, table_comments, table_details))
                             log.debug(resource)
+                # oracle sde format type
+                if resource["format"] == "oracle_sde":
+                    if resource["object_name"] == table_name:
+                        if "object_table_comments" in resource.keys():
+                            table_comments = resource["object_table_comments"]
+                        else:
+                            table_comments = None
+                        # only add to matches if schema details found
+                        if "details" in resource.keys() and resource["details"] != "":
+                            table_details = resource["details"]
+                            matches.append((notes, table_comments, table_details))
+                            log.debug(resource)
 
                 # multiple format resource
                 elif resource["format"] == "multiple":
                     # if multiple format, check for table name match in this location
-                    if resource[
-                        "preview_info"
-                    ]:  # first check that the key is not empty
-                        if (
-                            json.loads(resource["preview_info"])["layer_name"]
-                            == table_name
-                        ):
-                            if "object_table_comments" in resource.keys():
-                                table_comments = resource["object_table_comments"]
-                            else:
-                                table_comments = None
-                            # only add to matches if schema details found
+                    if resource["preview_info"]:
+                        # check that layer_name key is present
+                        if "layer_name" in json.loads(resource["preview_info"]):
+                            # then check if it matches the table name
                             if (
-                                "details" in resource.keys()
-                                and resource["details"] != ""
+                                json.loads(resource["preview_info"])["layer_name"]
+                                == table_name
                             ):
-                                table_details = resource["details"]
-                                matches.append((notes, table_comments, table_details))
-                                log.debug(resource)
+                                if "object_table_comments" in resource.keys():
+                                    table_comments = resource["object_table_comments"]
+                                else:
+                                    table_comments = None
+                                # only add to matches if schema details found
+                                if (
+                                    "details" in resource.keys()
+                                    and resource["details"] != ""
+                                ):
+                                    table_details = resource["details"]
+                                    matches.append(
+                                        (notes, table_comments, table_details)
+                                    )
+                                    log.debug(resource)
 
         # uniquify the result
         if len(matches) > 0:

{bcdata-0.10.2 → bcdata-0.10.4}/bcdata/cli.py

@@ -320,6 +320,13 @@ def clear_cache(verbose, quiet):
     "--query",
     help="A valid CQL or ECQL query",
 )
+@bounds_opt
+@click.option(
+    "--bounds-crs",
+    "--bounds_crs",
+    help="CRS of provided bounds",
+    default="EPSG:3005",
+)
 @click.option(
     "--count",
     "-c",
@@ -362,6 +369,8 @@ def bc2pg(
     schema,
     geometry_type,
     query,
+    bounds,
+    bounds_crs,
     count,
     primary_key,
     sortby,
@@ -372,10 +381,10 @@ def bc2pg(
     verbose,
     quiet,
 ):
-    """Download a DataBC WFS layer to postgres
+    """Load a DataBC WFS layer to a postgres db
 
     \b
-     $ bcdata bc2pg bc-airports --db_url postgresql://postgres:postgres@localhost:5432/postgis
+     $ bcdata bc2pg whse_imagery_and_base_maps.gsr_airports_svw
     """
     # for this command, default to INFO level logging
     verbosity = verbose - quiet
@@ -411,6 +420,8 @@ def bc2pg(
         table=table,
         schema=schema,
         query=query,
+        bounds=bounds,
+        bounds_crs=bounds_crs,
         geometry_type=geometry_type,
         count=count,
         primary_key=primary_key,
@@ -418,6 +429,7 @@ def bc2pg(
         timestamp=timestamp,
         schema_only=schema_only,
         append=append,
+        refresh=refresh,
     )
 
     # if refreshing, flush from temp bcdata schema to target schema
@@ -426,6 +438,8 @@ def bc2pg(
         s, table = out_table.split(".")
         db.refresh(schema_target, table)
         out_table = schema_target + "." + table
+        if timestamp:
+            db.log(schema_target, table)
 
     # do not notify of data load completion when no data load has occured
     if not schema_only:

{bcdata-0.10.2 → bcdata-0.10.4}/bcdata/database.py

@@ -212,3 +212,21 @@ class Database(object):
         metadata_obj = MetaData(schema=schema)
         table = Table(table, metadata_obj, schema=schema, autoload_with=self.engine)
         return list(table.columns.keys())
+
+    def log(self, schema_name, table_name):
+        log.info("Logging download date to bcdata.log")
+        self.execute(
+            """CREATE SCHEMA IF NOT EXISTS bcdata;
+               CREATE TABLE IF NOT EXISTS bcdata.log (
+                 table_name text PRIMARY KEY,
+                 latest_download timestamp WITH TIME ZONE
+               );
+            """
+        )
+        self.execute(
+            """INSERT INTO bcdata.log (table_name, latest_download)
+               SELECT %s as table_name, NOW() as latest_download
+               ON CONFLICT (table_name) DO UPDATE SET latest_download = NOW();
+            """,
+            (schema_name + "." + table_name,),
+        )

{bcdata-0.10.2 → bcdata-0.10.4/bcdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bcdata
-Version: 0.10.2
+Version: 0.10.4
 Summary: Python tools for quick access to DataBC geo-data available via WFS
 Home-page: https://github.com/smnorris/bcdata
 Author: Simon Norris
@@ -153,25 +153,30 @@ Usage: bcdata bc2pg [OPTIONS] DATASET
 
   Download a DataBC WFS layer to postgres
 
-   $ bcdata bc2pg bc-airports --db_url postgresql://postgres:postgres@localhost:5432/postgis
+   $ bcdata bc2pg whse_imagery_and_base_maps.gsr_airports_svw
 
 Options:
-  -db, --db_url TEXT      Target database url, defaults to $DATABASE_URL
-                          environment variable if set
-  --table TEXT            Destination table name
-  --schema TEXT           Destination schema name
-  --geometry_type TEXT    Spatial type of geometry column
-  --query TEXT            A valid CQL or ECQL query
-  -c, --count INTEGER     Total number of features to load
-  -k, --primary_key TEXT  Primary key of dataset
-  -s, --sortby TEXT       Name of sort field
-  -e, --schema_only       Create empty table from catalogue schema
-  -a, --append            Append to existing table
-  -r, --refresh           Truncate from existing table before load
-  -t, --no_timestamp      Do not log download to bcdata.log
-  -v, --verbose           Increase verbosity.
-  -q, --quiet             Decrease verbosity.
-  --help                  Show this message and exit.
+  -db, --db_url TEXT              Target database url, defaults to
+                                  $DATABASE_URL environment variable if set
+  --table TEXT                    Destination table name
+  --schema TEXT                   Destination schema name
+  --geometry_type TEXT            Spatial type of geometry column
+  --query TEXT                    A valid CQL or ECQL query
+  --bounds TEXT                   Bounds: "left bottom right top" or "[left,
+                                  bottom, right, top]". Coordinates are BC
+                                  Albers (default) or --bounds_crs
+  --bounds-crs, --bounds_crs TEXT
+                                  CRS of provided bounds
+  -c, --count INTEGER             Total number of features to load
+  -k, --primary_key TEXT          Primary key of dataset
+  -s, --sortby TEXT               Name of sort field
+  -e, --schema_only               Create empty table from catalogue schema
+  -a, --append                    Append to existing table
+  -r, --refresh                   Truncate from existing table before load
+  -t, --no_timestamp              Do not log download to bcdata.log
+  -v, --verbose                   Increase verbosity.
+  -q, --quiet                     Decrease verbosity.
+  --help                          Show this message and exit.
 ```
 
 #### cat

{bcdata-0.10.2 → bcdata-0.10.4}/tests/test_bc2pg.py

@@ -38,6 +38,28 @@ def test_bc2pg_count():
     DB_CONNECTION.execute("drop table " + AIRPORTS_TABLE)
 
 
+def test_bc2pg_bounds():
+    bcdata.bc2pg(AIRPORTS_TABLE, DB_URL, bounds=[1188000, 377051, 1207437, 390361])
+    assert AIRPORTS_TABLE in DB_CONNECTION.tables
+    r = DB_CONNECTION.query(
+        "select airport_name from whse_imagery_and_base_maps.gsr_airports_svw"
+    )
+    assert len(r) == 8
+    DB_CONNECTION.execute("drop table " + AIRPORTS_TABLE)
+
+
+def test_bc2pg_bounds_count():
+    bcdata.bc2pg(
+        AIRPORTS_TABLE, DB_URL, bounds=[1188000, 377051, 1207437, 390361], count=6
+    )
+    assert AIRPORTS_TABLE in DB_CONNECTION.tables
+    r = DB_CONNECTION.query(
+        "select airport_name from whse_imagery_and_base_maps.gsr_airports_svw"
+    )
+    assert len(r) == 6
+    DB_CONNECTION.execute("drop table " + AIRPORTS_TABLE)
+
+
 def test_bc2pg_table():
     bcdata.bc2pg(AIRPORTS_TABLE, DB_URL, table="testtable")
     assert "whse_imagery_and_base_maps.testtable" in DB_CONNECTION.tables

{bcdata-0.10.2 → bcdata-0.10.4}/tests/test_bcdc.py

@@ -49,3 +49,21 @@ def test_get_table_definition_format_multi_nopreview():
     assert table_definition["description"]
     assert table_definition["comments"]
     assert table_definition["schema"]
+
+
+def test_get_table_definition_format_multi_nolayer():
+    table_definition = bcdc.get_table_definition(
+        "WHSE_HUMAN_CULTURAL_ECONOMIC.HIST_HISTORIC_ENVIRONMENTS_SP"
+    )
+    assert table_definition["description"]
+    assert table_definition["comments"]
+    assert table_definition["schema"]
+
+
+def test_get_table_definition_format_oracle_sde():
+    table_definition = bcdc.get_table_definition(
+        "WHSE_LAND_USE_PLANNING.RMP_LANDSCAPE_RSRV_DESIGN_SP"
+    )
+    assert table_definition["description"]
+    assert table_definition["comments"]
+    assert table_definition["schema"]