datamarket 0.9.8__tar.gz → 0.9.10__tar.gz

{datamarket-0.9.8 → datamarket-0.9.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.8
+Version: 0.9.10
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.8 → datamarket-0.9.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.8"
+version = "0.9.10"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"

{datamarket-0.9.8 → datamarket-0.9.10}/src/datamarket/interfaces/alchemy.py

@@ -5,6 +5,7 @@ import logging
 from urllib.parse import quote_plus
 
 from sqlalchemy import DDL, create_engine, text
+from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import sessionmaker
 
@@ -117,6 +118,34 @@ class AlchemyInterface:
 
             self.session.rollback()
 
+    def upsert_alchemy_obj(self, alchemy_obj, index_elements, silent=False):
+        if not silent:
+            logger.info(f"upserting {alchemy_obj}")
+
+        primary_keys = list(
+            column.name for column in alchemy_obj.__table__.primary_key.columns.values()
+        )
+        obj_dict = {
+            column.name: getattr(alchemy_obj, column.name)
+            for column in alchemy_obj.__table__.columns
+            if column.name not in primary_keys
+        }
+
+        statement = (
+            insert(alchemy_obj.__table__)
+            .values(obj_dict)
+            .on_conflict_do_update(index_elements=index_elements, set_=obj_dict)
+        )
+
+        try:
+            self.session.execute(statement)
+            self.session.commit()
+        except IntegrityError:
+            if not silent:
+                logger.info(f"could not upsert {alchemy_obj}")
+
+            self.session.rollback()
+
     def reset_column(self, query_results, column_name):
         if not query_results:
             logger.warning("No objects to reset column for.")

{datamarket-0.9.8 → datamarket-0.9.10}/src/datamarket/interfaces/peerdb.py

@@ -112,6 +112,8 @@ class ClickhousePeer:
     def __init__(self, config):
         if "clickhouse" in config:
             self.config = config["clickhouse"]
+            self.credentials = {key: self.config[key] for key in ["user", "password", "host", "port"]}
+            
         else:
             logger.warning("no clickhouse section in config")
 
@@ -121,7 +123,7 @@ class ClickhousePeer:
 
         self.ensure_database_exists(database)
         self.config["database"] = database
-        self.client = clickhouse_driver.Client(**self.config)
+        self.client = clickhouse_driver.Client(**self.credentials)
 
     def _check_connection(self):
         if self.client is None:
@@ -129,7 +131,7 @@ class ClickhousePeer:
 
     def ensure_database_exists(self, database):
         logger.info(f"Checking if database '{database}' exists in Clickhouse")
-        temp_client = clickhouse_driver.Client(**self.config)
+        temp_client = clickhouse_driver.Client(**self.credentials)
         databases = temp_client.execute("SHOW DATABASES")
         if database not in [db[0] for db in databases]:
             logger.info(f"Creating database '{database}'")