datamarket 0.9.9__tar.gz → 0.9.11__tar.gz

{datamarket-0.9.9 → datamarket-0.9.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.9
+Version: 0.9.11
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.9 → datamarket-0.9.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.9"
+version = "0.9.11"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"

{datamarket-0.9.9 → datamarket-0.9.11}/src/datamarket/interfaces/alchemy.py

@@ -5,6 +5,7 @@ import logging
 from urllib.parse import quote_plus
 
 from sqlalchemy import DDL, create_engine, text
+from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import sessionmaker
 
@@ -117,6 +118,34 @@ class AlchemyInterface:
 
             self.session.rollback()
 
+    def upsert_alchemy_obj(self, alchemy_obj, index_elements, silent=False):
+        if not silent:
+            logger.info(f"upserting {alchemy_obj}")
+
+        primary_keys = list(
+            column.name for column in alchemy_obj.__table__.primary_key.columns.values()
+        )
+        obj_dict = {
+            column.name: getattr(alchemy_obj, column.name)
+            for column in alchemy_obj.__table__.columns
+            if column.name not in primary_keys
+        }
+
+        statement = (
+            insert(alchemy_obj.__table__)
+            .values(obj_dict)
+            .on_conflict_do_update(index_elements=index_elements, set_=obj_dict)
+        )
+
+        try:
+            self.session.execute(statement)
+            self.session.commit()
+        except IntegrityError:
+            if not silent:
+                logger.info(f"could not upsert {alchemy_obj}")
+
+            self.session.rollback()
+
     def reset_column(self, query_results, column_name):
         if not query_results:
             logger.warning("No objects to reset column for.")

{datamarket-0.9.9 → datamarket-0.9.11}/src/datamarket/interfaces/peerdb.py

@@ -113,7 +113,7 @@ class ClickhousePeer:
         if "clickhouse" in config:
             self.config = config["clickhouse"]
             self.credentials = {key: self.config[key] for key in ["user", "password", "host", "port"]}
-            
+
         else:
             logger.warning("no clickhouse section in config")
 
@@ -122,7 +122,7 @@ class ClickhousePeer:
             return
 
         self.ensure_database_exists(database)
-        self.config["database"] = database
+        self.config["database"] = self.credentials["database"] = database
         self.client = clickhouse_driver.Client(**self.credentials)
 
     def _check_connection(self):