bcdata 0.10.2__tar.gz → 0.13.0__tar.gz

{bcdata-0.10.2/bcdata.egg-info → bcdata-0.13.0}/PKG-INFO

@@ -1,36 +1,34 @@
 Metadata-Version: 2.1
 Name: bcdata
-Version: 0.10.2
-Summary: Python tools for quick access to DataBC geo-data available via WFS
-Home-page: https://github.com/smnorris/bcdata
-Author: Simon Norris
-Author-email: snorris@hillcrestgeo.ca
-License: MIT
-Keywords: gis geospatial data BC DataBC download "Britsh Columbia"
-Classifier: Development Status :: 3 - Alpha
-Classifier: Intended Audience :: Developers
-Classifier: Intended Audience :: Science/Research
-Classifier: Operating System :: OS Independent
+Version: 0.13.0
+Summary: Download open data, monitor and report on changes
+Author-email: Simon Norris <snorris@hillcrestgeo.ca>
+Project-URL: Homepage, https://github.com/smnorris/bcdata
+Project-URL: Issues, https://github.com/smnorris/bcdata
+Classifier: Development Status :: 1 - Planning
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Classifier: Topic :: Utilities
+Classifier: Programming Language :: Python :: 3.13
+Classifier: License :: OSI Approved :: MIT License
 Classifier: Topic :: Scientific/Engineering :: GIS
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
-Requires-Dist: geoalchemy2==0.14.*
-Requires-Dist: geopandas==0.14.*
-Requires-Dist: owslib==0.29.*
-Requires-Dist: psycopg2-binary==2.9.*
-Requires-Dist: rasterio==1.3.*
-Requires-Dist: requests==2.31.*
-Requires-Dist: sqlalchemy==2.0.*
-Requires-Dist: stamina==23.2.*
+Requires-Dist: geoalchemy2
+Requires-Dist: geopandas
+Requires-Dist: owslib
+Requires-Dist: psycopg2-binary
+Requires-Dist: rasterio
+Requires-Dist: requests
+Requires-Dist: sqlalchemy
+Requires-Dist: stamina
 Provides-Extra: test
-Requires-Dist: pytest>=3; extra == "test"
+Requires-Dist: pytest; extra == "test"
 Requires-Dist: pre-commit; extra == "test"
-Requires-Dist: requests_mock; extra == "test"
+Requires-Dist: requests-mock; extra == "test"
 
 # bcdata
 
@@ -43,7 +41,9 @@ There is a [wealth of British Columbia geographic information available as open
 data](https://catalogue.data.gov.bc.ca/dataset?download_audience=Public),
 but direct file download urls are not available and the syntax to accesss WFS via `ogr2ogr` and/or `curl/wget` can be awkward.
 
-This Python module and CLI attempts to simplify downloads of BC geographic data and smoothly integrate with PostGIS and Python GIS tools like `geopandas`, `fiona` and `rasterio`. As noted above, the module is an interface only to geographic data [avialable via WFS/WCS](https://bcgov.github.io/data-publication/pages/tips_tricks_webservices.html)  - to work with other BC open data, use the [bcdata R package](https://github.com/bcgov/bcdata).
+This tool attempts to simplify downloads of BC geographic data and smoothly integrate with PostGIS and Python GIS tools like `geopandas`, `fiona` and `rasterio`. As noted above, the module is an interface only to geographic data [avialable via WFS/WCS](https://bcgov.github.io/data-publication/pages/tips_tricks_webservices.html)  - to work with other BC open data, use the [bcdata R package](https://github.com/bcgov/bcdata).
+
+Note that this tool depends on BC Open Geospatial Web Services API at [openmaps.gov.bc.ca](https://openmaps.gov.bc.ca/geo/pub/ows?service=WFS&request=Getcapabilities) and the [BC Data Catalogue API](https://catalogue.data.gov.bc.ca/dataset/bc-data-catalogue-api).  If downloads are failing, consult the [BCGov Data Systems and Services uptime monitor](https://uptime.com/statuspage/bcgov-dss).
 
 **Disclaimer**  
 *It is the user's responsibility to check the licensing for any downloads, data are generally licensed as [OGL-BC](http://www2.gov.bc.ca/gov/content/governments/about-the-bc-government/databc/open-data/open-government-license-bc)*
@@ -153,25 +153,30 @@ Usage: bcdata bc2pg [OPTIONS] DATASET
 
   Download a DataBC WFS layer to postgres
 
-   $ bcdata bc2pg bc-airports --db_url postgresql://postgres:postgres@localhost:5432/postgis
+   $ bcdata bc2pg whse_imagery_and_base_maps.gsr_airports_svw
 
 Options:
-  -db, --db_url TEXT      Target database url, defaults to $DATABASE_URL
-                          environment variable if set
-  --table TEXT            Destination table name
-  --schema TEXT           Destination schema name
-  --geometry_type TEXT    Spatial type of geometry column
-  --query TEXT            A valid CQL or ECQL query
-  -c, --count INTEGER     Total number of features to load
-  -k, --primary_key TEXT  Primary key of dataset
-  -s, --sortby TEXT       Name of sort field
-  -e, --schema_only       Create empty table from catalogue schema
-  -a, --append            Append to existing table
-  -r, --refresh           Truncate from existing table before load
-  -t, --no_timestamp      Do not log download to bcdata.log
-  -v, --verbose           Increase verbosity.
-  -q, --quiet             Decrease verbosity.
-  --help                  Show this message and exit.
+  -db, --db_url TEXT              Target database url, defaults to
+                                  $DATABASE_URL environment variable if set
+  --table TEXT                    Destination table name
+  --schema TEXT                   Destination schema name
+  --geometry_type TEXT            Spatial type of geometry column
+  --query TEXT                    A valid CQL or ECQL query
+  --bounds TEXT                   Bounds: "left bottom right top" or "[left,
+                                  bottom, right, top]". Coordinates are BC
+                                  Albers (default) or --bounds_crs
+  --bounds-crs, --bounds_crs TEXT
+                                  CRS of provided bounds
+  -c, --count INTEGER             Total number of features to load
+  -k, --primary_key TEXT          Primary key of dataset
+  -s, --sortby TEXT               Name of sort field
+  -e, --schema_only               Create empty table from catalogue schema
+  -a, --append                    Append to existing table
+  -r, --refresh                   Truncate from existing table before load
+  -t, --no_timestamp              Do not log download to bcdata.log
+  -v, --verbose                   Increase verbosity.
+  -q, --quiet                     Decrease verbosity.
+  --help                          Show this message and exit.
 ```
 
 #### cat
@@ -332,7 +337,7 @@ Describe a dataset. Note that if we know the id of a dataset, we can use that ra
 
 The JSON output can be manipulated with [jq](https://stedolan.github.io/jq/). For example, to show only the fields available in the dataset:
 
-    $ bcdata info bc-airports | jq '.schema.properties'
+    $ bcdata info bc-airports | jq '.schema'
     {
       "CUSTODIAN_ORG_DESCRIPTION": "string",
       "BUSINESS_CATEGORY_CLASS": "string",
@@ -360,6 +365,11 @@ Save a layer to a geopackage in BC Albers:
 
 Note that this will not work if the source data has mixed geometry types.
 
+Load to parquet on s3:
+
+    $ bcdata dump bc-airports \
+      | ogr2ogr -f Parquet /vsis3/$BUCKET/airports.parquet -s_srs EPSG:4326 -t_srs EPSG:3005 /vsistdin/
+
 Load data to postgres and run a spatial query:
 
     $ bcdata bc2pg bc-airports \
@@ -406,13 +416,12 @@ Load data to postgres and run a spatial query:
 
 Create virtualenv and install `bcdata` in development mode:
 
-    $ mkdir bcdata_env
-    $ virtualenv bcdata_env
-    $ source bcdata_env/bin/activate
-    (bcdata_env)$ git clone git@github.com:smnorris/bcdata.git
-    (bcdata_env)$ cd bcdata
-    (bcdata_env)$ pip install -e .[test]
-    (bcdata_env)$ py.test
+    $ git clone git@github.com:smnorris/bcdata.git
+    $ cd bcdata
+    $ python -m venv .venv
+    $ source .venv/bin/activate
+    (.venv)$ pip install -e .[test]
+    (.venv)$ py.test
 
 
 ## Other implementations

{bcdata-0.10.2 → bcdata-0.13.0}/README.md

@@ -9,7 +9,9 @@ There is a [wealth of British Columbia geographic information available as open
 data](https://catalogue.data.gov.bc.ca/dataset?download_audience=Public),
 but direct file download urls are not available and the syntax to accesss WFS via `ogr2ogr` and/or `curl/wget` can be awkward.
 
-This Python module and CLI attempts to simplify downloads of BC geographic data and smoothly integrate with PostGIS and Python GIS tools like `geopandas`, `fiona` and `rasterio`. As noted above, the module is an interface only to geographic data [avialable via WFS/WCS](https://bcgov.github.io/data-publication/pages/tips_tricks_webservices.html)  - to work with other BC open data, use the [bcdata R package](https://github.com/bcgov/bcdata).
+This tool attempts to simplify downloads of BC geographic data and smoothly integrate with PostGIS and Python GIS tools like `geopandas`, `fiona` and `rasterio`. As noted above, the module is an interface only to geographic data [avialable via WFS/WCS](https://bcgov.github.io/data-publication/pages/tips_tricks_webservices.html)  - to work with other BC open data, use the [bcdata R package](https://github.com/bcgov/bcdata).
+
+Note that this tool depends on BC Open Geospatial Web Services API at [openmaps.gov.bc.ca](https://openmaps.gov.bc.ca/geo/pub/ows?service=WFS&request=Getcapabilities) and the [BC Data Catalogue API](https://catalogue.data.gov.bc.ca/dataset/bc-data-catalogue-api).  If downloads are failing, consult the [BCGov Data Systems and Services uptime monitor](https://uptime.com/statuspage/bcgov-dss).
 
 **Disclaimer**  
 *It is the user's responsibility to check the licensing for any downloads, data are generally licensed as [OGL-BC](http://www2.gov.bc.ca/gov/content/governments/about-the-bc-government/databc/open-data/open-government-license-bc)*
@@ -119,25 +121,30 @@ Usage: bcdata bc2pg [OPTIONS] DATASET
 
   Download a DataBC WFS layer to postgres
 
-   $ bcdata bc2pg bc-airports --db_url postgresql://postgres:postgres@localhost:5432/postgis
+   $ bcdata bc2pg whse_imagery_and_base_maps.gsr_airports_svw
 
 Options:
-  -db, --db_url TEXT      Target database url, defaults to $DATABASE_URL
-                          environment variable if set
-  --table TEXT            Destination table name
-  --schema TEXT           Destination schema name
-  --geometry_type TEXT    Spatial type of geometry column
-  --query TEXT            A valid CQL or ECQL query
-  -c, --count INTEGER     Total number of features to load
-  -k, --primary_key TEXT  Primary key of dataset
-  -s, --sortby TEXT       Name of sort field
-  -e, --schema_only       Create empty table from catalogue schema
-  -a, --append            Append to existing table
-  -r, --refresh           Truncate from existing table before load
-  -t, --no_timestamp      Do not log download to bcdata.log
-  -v, --verbose           Increase verbosity.
-  -q, --quiet             Decrease verbosity.
-  --help                  Show this message and exit.
+  -db, --db_url TEXT              Target database url, defaults to
+                                  $DATABASE_URL environment variable if set
+  --table TEXT                    Destination table name
+  --schema TEXT                   Destination schema name
+  --geometry_type TEXT            Spatial type of geometry column
+  --query TEXT                    A valid CQL or ECQL query
+  --bounds TEXT                   Bounds: "left bottom right top" or "[left,
+                                  bottom, right, top]". Coordinates are BC
+                                  Albers (default) or --bounds_crs
+  --bounds-crs, --bounds_crs TEXT
+                                  CRS of provided bounds
+  -c, --count INTEGER             Total number of features to load
+  -k, --primary_key TEXT          Primary key of dataset
+  -s, --sortby TEXT               Name of sort field
+  -e, --schema_only               Create empty table from catalogue schema
+  -a, --append                    Append to existing table
+  -r, --refresh                   Truncate from existing table before load
+  -t, --no_timestamp              Do not log download to bcdata.log
+  -v, --verbose                   Increase verbosity.
+  -q, --quiet                     Decrease verbosity.
+  --help                          Show this message and exit.
 ```
 
 #### cat
@@ -298,7 +305,7 @@ Describe a dataset. Note that if we know the id of a dataset, we can use that ra
 
 The JSON output can be manipulated with [jq](https://stedolan.github.io/jq/). For example, to show only the fields available in the dataset:
 
-    $ bcdata info bc-airports | jq '.schema.properties'
+    $ bcdata info bc-airports | jq '.schema'
     {
       "CUSTODIAN_ORG_DESCRIPTION": "string",
       "BUSINESS_CATEGORY_CLASS": "string",
@@ -326,6 +333,11 @@ Save a layer to a geopackage in BC Albers:
 
 Note that this will not work if the source data has mixed geometry types.
 
+Load to parquet on s3:
+
+    $ bcdata dump bc-airports \
+      | ogr2ogr -f Parquet /vsis3/$BUCKET/airports.parquet -s_srs EPSG:4326 -t_srs EPSG:3005 /vsistdin/
+
 Load data to postgres and run a spatial query:
 
     $ bcdata bc2pg bc-airports \
@@ -372,13 +384,12 @@ Load data to postgres and run a spatial query:
 
 Create virtualenv and install `bcdata` in development mode:
 
-    $ mkdir bcdata_env
-    $ virtualenv bcdata_env
-    $ source bcdata_env/bin/activate
-    (bcdata_env)$ git clone git@github.com:smnorris/bcdata.git
-    (bcdata_env)$ cd bcdata
-    (bcdata_env)$ pip install -e .[test]
-    (bcdata_env)$ py.test
+    $ git clone git@github.com:smnorris/bcdata.git
+    $ cd bcdata
+    $ python -m venv .venv
+    $ source .venv/bin/activate
+    (.venv)$ pip install -e .[test]
+    (.venv)$ py.test
 
 
 ## Other implementations

bcdata-0.13.0/pyproject.toml

@@ -0,0 +1,55 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "bcdata"
+dynamic = ["version"]
+readme = "README.md"
+license = {file = "LICENSE"}
+authors = [
+  {name="Simon Norris", email="snorris@hillcrestgeo.ca"},
+]
+description = "Download open data, monitor and report on changes"
+requires-python = ">=3.9"
+classifiers = [
+    "Development Status :: 1 - Planning",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "License :: OSI Approved :: MIT License",
+    "Topic :: Scientific/Engineering :: GIS",
+    "Operating System :: OS Independent"
+]
+dependencies = [
+    "geoalchemy2",
+    "geopandas",
+    "owslib",
+    "psycopg2-binary",
+    "rasterio",
+    "requests",
+    "sqlalchemy",
+    "stamina"
+]
+
+[project.optional-dependencies]
+test = [
+  "pytest",
+  "pre-commit",
+  "requests-mock"
+]
+
+[project.scripts]
+bcdata = "bcdata.cli:cli"
+
+[project.urls]
+Homepage = "https://github.com/smnorris/bcdata"
+Issues = "https://github.com/smnorris/bcdata"
+
+[tool.setuptools.dynamic]
+version = {attr = "bcdata.__version__"}
+
+[tool.ruff]
+line-length = 100

bcdata-0.13.0/src/bcdata/__init__.py

@@ -0,0 +1,27 @@
+import requests
+
+from .bc2pg import bc2pg as bc2pg
+from .bcdc import get_table_definition as get_table_definition
+from .bcdc import get_table_name as get_table_name
+from .wcs import get_dem as get_dem
+from .wfs import define_requests as define_requests
+from .wfs import get_count as get_count
+from .wfs import get_data as get_data
+from .wfs import get_features as get_features
+from .wfs import get_sortkey as get_sortkey
+from .wfs import list_tables as list_tables
+from .wfs import validate_name as validate_name
+
+PRIMARY_KEY_DB_URL = "https://raw.githubusercontent.com/smnorris/bcdata/main/data/primary_keys.json"
+
+# BCDC does not indicate which column in the schema is the primary key.
+# In this absence, bcdata maintains its own dictionary of {table: primary_key},
+# served via github. Retrieve the dict with this function"""
+response = requests.get(PRIMARY_KEY_DB_URL)
+if response.status_code == 200:
+    primary_keys = response.json()
+else:
+    raise Exception(f"Failed to download primary key database at {PRIMARY_KEY_DB_URL}")
+    primary_keys = {}
+
+__version__ = "0.13.0"

{bcdata-0.10.2 → bcdata-0.13.0/src}/bcdata/bc2pg.py

@@ -1,12 +1,6 @@
-import json
 import logging
-import os
 
-import geopandas as gpd
 import numpy
-import stamina
-from geoalchemy2 import Geometry
-import requests
 from shapely.geometry.linestring import LineString
 from shapely.geometry.multilinestring import MultiLineString
 from shapely.geometry.multipoint import MultiPoint
@@ -34,19 +28,6 @@ SUPPORTED_TYPES = [
 ]
 
 
-def get_primary_keys():
-    """download primary key data file"""
-    response = requests.get(bcdata.PRIMARY_KEY_DB_URL)
-    if response.status_code == 200:
-        primary_keys = response.json()
-    else:
-        log.warning(
-            f"Failed to download primary key database at {bcdata.PRIMARY_KEY_DB_URL}"
-        )
-        primary_keys = {}
-    return primary_keys
-
-
 def bc2pg(  # noqa: C901
     dataset,
     db_url,
@@ -54,12 +35,15 @@ def bc2pg(  # noqa: C901
     schema=None,
     geometry_type=None,
     query=None,
+    bounds=None,
+    bounds_crs="EPSG:3005",
     count=None,
     sortby=None,
     primary_key=None,
     timestamp=True,
     schema_only=False,
     append=False,
+    refresh=False,
 ):
     """Request table definition from bcdc and replicate in postgres"""
     if schema_only and append:
@@ -72,6 +56,16 @@ def bc2pg(  # noqa: C901
     if table:
         table_name = table.lower()
 
+    # take note if geometry type is specified as option
+    geometry_type_opt = geometry_type
+
+    # do not promote to multi if singlepart specified,
+    # otherwise default to always promoting to multipart in case mixed types are returned
+    if geometry_type_opt and geometry_type_opt.upper() in ["POINT", "LINE", "POLYGON"]:
+        promote_to_multi = False
+    else:
+        promote_to_multi = True
+
     # connect to target db
     db = Database(db_url)
 
@@ -82,6 +76,8 @@ def bc2pg(  # noqa: C901
     urls = bcdata.define_requests(
         dataset,
         query=query,
+        bounds=bounds,
+        bounds_crs=bounds_crs,
         count=count,
         sortby=sortby,
         crs="epsg:3005",
@@ -101,19 +97,13 @@ def bc2pg(  # noqa: C901
         table_definition = bcdata.get_table_definition(dataset)
 
         if not table_definition["schema"]:
-            raise ValueError(
-                "Cannot create table, schema details not found via bcdc api"
-            )
+            raise ValueError("Cannot create table, schema details not found via bcdc api")
 
         # if geometry type is not provided, determine type by making the first request
         if not geometry_type:
-            df = WFS.make_requests(
-                [urls[0]], as_gdf=True, crs="epsg:3005", lowercase=True
-            )
+            df = WFS.make_requests(dataset=dataset, urls=[urls[0]], as_gdf=True, crs="epsg:3005", lowercase=True)
             geometry_type = df.geom_type.unique()[0]  # keep only the first type
-            if numpy.any(
-                df.has_z.unique()[0]
-            ):  # geopandas does not include Z in geom_type string
+            if numpy.any(df.has_z.unique()[0]):  # geopandas does not include Z in geom_type string
                 geometry_type = geometry_type + "Z"
 
         # if geometry type is still not populated try the last request
@@ -121,15 +111,14 @@ def bc2pg(  # noqa: C901
         if not geometry_type:
             if not urls[-1] == urls[0]:
                 df_temp = WFS.make_requests(
-                    [urls[-1]],
+                    dataset=dataset,
+                    urls=[urls[-1]],
                     as_gdf=True,
                     crs="epsg:3005",
                     lowercase=True,
                     silent=True,
                 )
-                geometry_type = df_temp.geom_type.unique()[
-                    0
-                ]  # keep only the first type
+                geometry_type = df_temp.geom_type.unique()[0]  # keep only the first type
                 if numpy.any(
                     df_temp.has_z.unique()[0]
                 ):  # geopandas does not include Z in geom_type string
@@ -143,9 +132,8 @@ def bc2pg(  # noqa: C901
             raise ValueError("Geometry type {geometry_type} is not supported")
 
         # if primary key is not supplied, use default (if present in list)
-        primary_keys = get_primary_keys()
-        if not primary_key and dataset.lower() in primary_keys:
-            primary_key = primary_keys[dataset.lower()]
+        if not primary_key and dataset.lower() in bcdata.primary_keys:
+            primary_key = bcdata.primary_keys[dataset.lower()]
 
         # fail if specified primary key is not in the table
         if primary_key and primary_key.upper() not in [
@@ -161,6 +149,7 @@ def bc2pg(  # noqa: C901
             table_name,
             table_definition["schema"],
             geometry_type,
+            promote_to_multi,
             table_definition["comments"],
             primary_key,
         )
@@ -168,9 +157,7 @@ def bc2pg(  # noqa: C901
 
     # check if column provided in sortby option is present in dataset
     if sortby and sortby.lower() not in column_names:
-        raise ValueError(
-            f"Specified sortby column {sortby} is not present in {dataset}"
-        )
+        raise ValueError(f"Specified sortby column {sortby} is not present in {dataset}")
 
     # load the data
     if not schema_only:
@@ -178,9 +165,7 @@ def bc2pg(  # noqa: C901
         for n, url in enumerate(urls):
             # if first url not downloaded above when checking geom type, do now
             if df is None:
-                df = WFS.make_requests(
-                    [url], as_gdf=True, crs="epsg:3005", lowercase=True
-                )
+                df = WFS.make_requests(dataset=dataset, urls=[url], as_gdf=True, crs="epsg:3005", lowercase=True)
             # tidy the resulting dataframe
             df = df.rename_geometry("geom")
             # lowercasify
@@ -193,23 +178,20 @@ def bc2pg(  # noqa: C901
             df_nulls = df_nulls.drop(columns=["geom"])
             # remove rows with null geometry from geodataframe
             df = df[df["geom"].notna()]
-            # cast to everything multipart because responses can have mixed types
-            # geopandas does not have a built in function:
-            # https://gis.stackexchange.com/questions/311320/casting-geometry-to-multi-using-geopandas
-            df["geom"] = [
-                MultiPoint([feature]) if isinstance(feature, Point) else feature
-                for feature in df["geom"]
-            ]
-            df["geom"] = [
-                MultiLineString([feature])
-                if isinstance(feature, LineString)
-                else feature
-                for feature in df["geom"]
-            ]
-            df["geom"] = [
-                MultiPolygon([feature]) if isinstance(feature, Polygon) else feature
-                for feature in df["geom"]
-            ]
+            # promote to multipart
+            if promote_to_multi:
+                df["geom"] = [
+                    MultiPoint([feature]) if isinstance(feature, Point) else feature
+                    for feature in df["geom"]
+                ]
+                df["geom"] = [
+                    MultiLineString([feature]) if isinstance(feature, LineString) else feature
+                    for feature in df["geom"]
+                ]
+                df["geom"] = [
+                    MultiPolygon([feature]) if isinstance(feature, Polygon) else feature
+                    for feature in df["geom"]
+                ]
 
             # run the load in two parts, one with geoms, one with no geoms
             log.info(f"Writing {dataset} to database as {schema_name}.{table_name}")
@@ -224,22 +206,8 @@ def bc2pg(  # noqa: C901
             df = None
 
         # once load complete, note date/time of load completion in bcdata.log
-        if timestamp:
-            log.info("Logging download date to bcdata.log")
-            db.execute(
-                """CREATE SCHEMA IF NOT EXISTS bcdata;
-                   CREATE TABLE IF NOT EXISTS bcdata.log (
-                     table_name text PRIMARY KEY,
-                     latest_download timestamp WITH TIME ZONE
-                   );
-                """
-            )
-            db.execute(
-                """INSERT INTO bcdata.log (table_name, latest_download)
-                   SELECT %s as table_name, NOW() as latest_download
-                   ON CONFLICT (table_name) DO UPDATE SET latest_download = NOW();
-                """,
-                (schema_name + "." + table_name,),
-            )
+        # do not log refreshes here, they called by cli once loaded to target table
+        if timestamp and not refresh:
+            db.log(schema_name, table_name)
 
     return schema_name + "." + table_name