datamarket 0.9.7__tar.gz → 0.9.9__tar.gz

{datamarket-0.9.7 → datamarket-0.9.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.7
+Version: 0.9.9
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.7 → datamarket-0.9.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.7"
+version = "0.9.9"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"

{datamarket-0.9.7 → datamarket-0.9.9}/src/datamarket/interfaces/alchemy.py

@@ -4,7 +4,7 @@
 import logging
 from urllib.parse import quote_plus
 
-from sqlalchemy import DDL, create_engine
+from sqlalchemy import DDL, create_engine, text
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import sessionmaker
 
@@ -14,6 +14,13 @@ from sqlalchemy.orm import sessionmaker
 logger = logging.getLogger(__name__)
 
 
+class MockContext:
+    def __init__(self, column):
+        self.current_parameters = {}
+        self.current_column = column
+        self.connection = None
+
+
 class AlchemyInterface:
     def __init__(self, config):
         if "db" in config:
@@ -28,10 +35,10 @@ class AlchemyInterface:
 
     def get_conn_str(self):
         return (
-            f'{self.config["engine"]}://'
-            f'{self.config["user"]}:{quote_plus(self.config["password"])}'
-            f'@{self.config["host"]}:{self.config["port"]}'
-            f'/{self.config["database"]}'
+            f"{self.config['engine']}://"
+            f"{self.config['user']}:{quote_plus(self.config['password'])}"
+            f"@{self.config['host']}:{self.config['port']}"
+            f"/{self.config['database']}"
         )
 
     @staticmethod
@@ -109,3 +116,30 @@ class AlchemyInterface:
                 logger.info(f"{alchemy_obj} already in db")
 
             self.session.rollback()
+
+    def reset_column(self, query_results, column_name):
+        if not query_results:
+            logger.warning("No objects to reset column for.")
+            return
+
+        first_obj = query_results[0]
+        model_class = first_obj.__class__
+        table = model_class.__table__
+
+        if column_name not in table.columns:
+            logger.warning(f"Column {column_name} does not exist in table {table.name}.")
+            return
+
+        column = table.columns[column_name]
+
+        if column.server_default is not None:
+            query_results.update({column_name: text("DEFAULT")}, synchronize_session=False)
+        elif column.default is not None:
+            default_value = column.default.arg
+            if callable(default_value):
+                default_value = default_value(MockContext(column))
+            query_results.update({column_name: default_value}, synchronize_session=False)
+        else:
+            raise ValueError(f"Column '{column_name}' doesn't have a default value defined.")
+
+        self.session.commit()

{datamarket-0.9.7 → datamarket-0.9.9}/src/datamarket/interfaces/peerdb.py

@@ -112,6 +112,8 @@ class ClickhousePeer:
     def __init__(self, config):
         if "clickhouse" in config:
             self.config = config["clickhouse"]
+            self.credentials = {key: self.config[key] for key in ["user", "password", "host", "port"]}
+            
         else:
             logger.warning("no clickhouse section in config")
 
@@ -121,7 +123,7 @@ class ClickhousePeer:
 
         self.ensure_database_exists(database)
         self.config["database"] = database
-        self.client = clickhouse_driver.Client(**self.config)
+        self.client = clickhouse_driver.Client(**self.credentials)
 
     def _check_connection(self):
         if self.client is None:
@@ -129,7 +131,7 @@ class ClickhousePeer:
 
     def ensure_database_exists(self, database):
         logger.info(f"Checking if database '{database}' exists in Clickhouse")
-        temp_client = clickhouse_driver.Client(**self.config)
+        temp_client = clickhouse_driver.Client(**self.credentials)
         databases = temp_client.execute("SHOW DATABASES")
         if database not in [db[0] for db in databases]:
             logger.info(f"Creating database '{database}'")