bcdata 0.10.0__tar.gz → 0.10.2__tar.gz

{bcdata-0.10.0 → bcdata-0.10.2}/CHANGES.txt

@@ -1,6 +1,14 @@
 Changes
 =======
 
+0.10.2 (2024-03-01)
+------------------
+- improve connection management for better stability (#171)
+
+0.10.1 (2024-02-19)
+------------------
+- check that table to be refreshed exists when using bc2pg --refresh (#167)
+
 0.10.0 (2024-02-19)
 ------------------
 - store known primary keys in repository (data/primary_keys.json)

{bcdata-0.10.0/bcdata.egg-info → bcdata-0.10.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bcdata
-Version: 0.10.0
+Version: 0.10.2
 Summary: Python tools for quick access to DataBC geo-data available via WFS
 Home-page: https://github.com/smnorris/bcdata
 Author: Simon Norris

{bcdata-0.10.0 → bcdata-0.10.2}/bcdata/__init__.py

@@ -15,4 +15,4 @@ PRIMARY_KEY_DB_URL = (
     "https://raw.githubusercontent.com/smnorris/bcdata/main/data/primary_keys.json"
 )
 
-__version__ = "0.10.0"
+__version__ = "0.10.2"

{bcdata-0.10.0 → bcdata-0.10.2}/bcdata/cli.py

@@ -395,9 +395,16 @@ def bc2pg(
     elif refresh and schema:
         schema_target = schema
     elif refresh and not schema:
-        schema_target, t = bcdata.validate_name(dataset).lower().split(".")
+        schema_target, table = bcdata.validate_name(dataset).lower().split(".")
     if refresh:
+        db = Database(db_url)
         schema = "bcdata"
+        if not table:
+            table = bcdata.validate_name(dataset).lower().split(".")
+        if schema_target + "." + table not in db.tables:
+            raise ValueError(
+                f"Cannot refresh, {schema_target}.{table} not found in database"
+            )
     out_table = bcdata.bc2pg(
         dataset,
         db_url,
@@ -420,4 +427,6 @@ def bc2pg(
         db.refresh(schema_target, table)
         out_table = schema_target + "." + table
 
-    log.info("Load of {} to {} in {} complete".format(dataset, out_table, db_url))
+    # do not notify of data load completion when no data load has occured
+    if not schema_only:
+        log.info("Load of {} to {} in {} complete".format(dataset, out_table, db_url))

{bcdata-0.10.0 → bcdata-0.10.2}/bcdata/database.py

@@ -1,9 +1,8 @@
 import logging
 import os
 
-import psycopg2
 from geoalchemy2 import Geometry
-from psycopg2 import sql
+from psycopg2 import errors, sql
 from sqlalchemy import Column, MetaData, Table, create_engine
 from sqlalchemy.dialects.postgresql import DATE, NUMERIC, VARCHAR
 
@@ -11,18 +10,20 @@ log = logging.getLogger(__name__)
 
 
 class Database(object):
-    """Wrapper around psycopg2 and sqlachemy"""
+    """Wrapper around sqlalchemy"""
 
     def __init__(self, url=os.environ.get("DATABASE_URL")):
         self.url = url
         self.engine = create_engine(url)
-        self.conn = psycopg2.connect(url)
         # make sure postgis is available
         try:
             self.query("SELECT postgis_full_version()")
-        except psycopg2.errors.UndefinedFunction:
-            log.error("Cannot find PostGIS, is extension added to database %s ?", url)
-            raise psycopg2.errors.UndefinedFunction
+        except errors.UndefinedFunction:
+            log.error(
+                "Cannot find PostGIS, has extension been installed on database %s ?",
+                url,
+            )
+            raise errors.UndefinedFunction
 
         # supported oracle/wfs to postgres types
         self.supported_types = {
@@ -56,24 +57,26 @@ class Database(object):
 
     def query(self, sql, params=None):
         """Execute sql and return all results"""
-        with self.conn:
-            with self.conn.cursor() as curs:
-                curs.execute(sql, params)
-                result = curs.fetchall()
+        conn = self.engine.raw_connection()
+        with conn.cursor() as curs:
+            curs.execute(sql, params)
+            result = curs.fetchall()
         return result
 
     def execute(self, sql, params=None):
         """Execute sql and return only whether the query was successful"""
-        with self.conn:
-            with self.conn.cursor() as curs:
-                result = curs.execute(sql, params)
+        conn = self.engine.raw_connection()
+        with conn.cursor() as curs:
+            result = curs.execute(sql, params)
+            conn.commit()
         return result
 
     def execute_many(self, sql, params):
         """Execute many sql"""
-        with self.conn:
-            with self.conn.cursor() as curs:
-                curs.executemany(sql, params)
+        conn = self.engine.raw_connection()
+        with conn.cursor() as curs:
+            curs.executemany(sql, params)
+            conn.commit()
 
     def create_schema(self, schema):
         if schema not in self.schemas:
@@ -120,6 +123,10 @@ class Database(object):
             )
             self.execute(dbq)
             self.drop_table("bcdata", table)
+        else:
+            raise ValueError(
+                f"Target table {schema}.{table} does not exist in database"
+            )
 
     def define_table(
         self,

{bcdata-0.10.0 → bcdata-0.10.2/bcdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bcdata
-Version: 0.10.0
+Version: 0.10.2
 Summary: Python tools for quick access to DataBC geo-data available via WFS
 Home-page: https://github.com/smnorris/bcdata
 Author: Simon Norris

{bcdata-0.10.0 → bcdata-0.10.2}/tests/test_cli.py

@@ -112,3 +112,20 @@ def test_bc2pg_refresh():
     assert AIRPORTS_TABLE.lower() in DB_CONNECTION.tables
     assert len(q) == 1
     DB_CONNECTION.execute("drop table " + AIRPORTS_TABLE.lower())
+
+
+def test_bc2pg_refresh_fails_table_does_not_exist():
+    runner = CliRunner()
+    result = runner.invoke(
+        cli,
+        [
+            "bc2pg",
+            AIRPORTS_TABLE,
+            "--db_url",
+            DB_URL,
+            "--count",
+            "1",
+            "--refresh",
+        ],
+    )
+    assert result.exit_code == 1