datamarket 0.9.9__py3-none-any.whl → 0.9.11__py3-none-any.whl

datamarket/interfaces/alchemy.py

@@ -5,6 +5,7 @@ import logging
 from urllib.parse import quote_plus
 
 from sqlalchemy import DDL, create_engine, text
+from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import sessionmaker
 
@@ -117,6 +118,34 @@ class AlchemyInterface:
 
             self.session.rollback()
 
+    def upsert_alchemy_obj(self, alchemy_obj, index_elements, silent=False):
+        if not silent:
+            logger.info(f"upserting {alchemy_obj}")
+
+        primary_keys = list(
+            column.name for column in alchemy_obj.__table__.primary_key.columns.values()
+        )
+        obj_dict = {
+            column.name: getattr(alchemy_obj, column.name)
+            for column in alchemy_obj.__table__.columns
+            if column.name not in primary_keys
+        }
+
+        statement = (
+            insert(alchemy_obj.__table__)
+            .values(obj_dict)
+            .on_conflict_do_update(index_elements=index_elements, set_=obj_dict)
+        )
+
+        try:
+            self.session.execute(statement)
+            self.session.commit()
+        except IntegrityError:
+            if not silent:
+                logger.info(f"could not upsert {alchemy_obj}")
+
+            self.session.rollback()
+
     def reset_column(self, query_results, column_name):
         if not query_results:
             logger.warning("No objects to reset column for.")

datamarket/interfaces/peerdb.py

@@ -113,7 +113,7 @@ class ClickhousePeer:
         if "clickhouse" in config:
             self.config = config["clickhouse"]
             self.credentials = {key: self.config[key] for key in ["user", "password", "host", "port"]}
-            
+
         else:
             logger.warning("no clickhouse section in config")
 
@@ -122,7 +122,7 @@ class ClickhousePeer:
             return
 
         self.ensure_database_exists(database)
-        self.config["database"] = database
+        self.config["database"] = self.credentials["database"] = database
         self.client = clickhouse_driver.Client(**self.credentials)
 
     def _check_connection(self):

{datamarket-0.9.9.dist-info → datamarket-0.9.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.9
+Version: 0.9.11
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.9.dist-info → datamarket-0.9.11.dist-info}/RECORD

@@ -1,11 +1,11 @@
 datamarket/__init__.py,sha256=FHS77P9qNewKMoN-p0FLEUEC60oWIYup1QkbJZP4ays,12
 datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamarket/interfaces/alchemy.py,sha256=9Guzsdy8k8tBsODeozQhuuXshxf3dMgERPVhuumiP2M,5427
+datamarket/interfaces/alchemy.py,sha256=5ZER9xiGj-7TqWrLiLUMk-rpiJFrqLffvqZzsqJdG5o,6388
 datamarket/interfaces/aws.py,sha256=7KLUeBxmPN7avEMPsu5HC_KHB1N7W6Anp2X8fo43mlw,2383
 datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIgE,2915
 datamarket/interfaces/ftp.py,sha256=o0KlJxtksbop9OjCiQRzyAa2IeG_ExVXagS6apwrAQo,1881
 datamarket/interfaces/nominatim.py,sha256=_gFJ04D-ju5xn3wuaGT5Pj5jhf4F5eINpxOpuQL_dIQ,3664
-datamarket/interfaces/peerdb.py,sha256=WKMMHlTGB8piDah-mpepWhgKFvgPZmhaVU3Ueh04oY0,22213
+datamarket/interfaces/peerdb.py,sha256=2FrG7E5BAGxhwoxjjlPP7ceE7bmUpQi54jtTPtjQul0,22232
 datamarket/interfaces/proxy.py,sha256=updoOStKd8-nQBbxWbnD9eOt6HksnYi-5dQ0rEySf5M,3152
 datamarket/interfaces/tinybird.py,sha256=AYrcRGNOCoCt7ojilkWa27POROee9sTCwZ61GGHEPeM,2698
 datamarket/params/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -17,7 +17,7 @@ datamarket/utils/main.py,sha256=O6rX-65h4h0j2zs9dofdTPlly5reKDnvgLtTwbLmbWg,6529
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
-datamarket-0.9.9.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.9.dist-info/METADATA,sha256=0CjAFeYD6ybomculTyKLtQqMgD67YXeXsRQ3UtTz31A,6362
-datamarket-0.9.9.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-datamarket-0.9.9.dist-info/RECORD,,
+datamarket-0.9.11.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.11.dist-info/METADATA,sha256=u6jfz9BqUNb4JdeVunWGwSVdsL2bzipytRBcxT-HzeQ,6363
+datamarket-0.9.11.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+datamarket-0.9.11.dist-info/RECORD,,