datamarket 0.9.5__tar.gz → 0.9.7__tar.gz

{datamarket-0.9.5 → datamarket-0.9.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.5
+Version: 0.9.7
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.5 → datamarket-0.9.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.5"
+version = "0.9.7"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"

{datamarket-0.9.5 → datamarket-0.9.7}/src/datamarket/interfaces/aws.py

@@ -4,8 +4,7 @@
 import io
 import logging
 import boto3
-
-from ..utils.main import Config
+from dynaconf import Dynaconf
 
 ########################################################################################################################
 # CLASSES
@@ -14,7 +13,7 @@ logger = logging.getLogger(__name__)
 
 
 class AWSInterface:
-    def __init__(self, config: Config) -> None:
+    def __init__(self, config: Dynaconf) -> None:
         self.profiles = []
         self.config = config
 

{datamarket-0.9.5 → datamarket-0.9.7}/src/datamarket/interfaces/ftp.py

@@ -5,7 +5,7 @@ import logging
 from ftplib import FTP, FTP_TLS
 from pathlib import Path
 
-from ..utils.main import Config
+from dynaconf import Dynaconf
 
 ########################################################################################################################
 # CLASSES
@@ -13,7 +13,7 @@ from ..utils.main import Config
 logger = logging.getLogger(__name__)
 
 class FTPInterface:
-    def __init__(self, config: Config):
+    def __init__(self, config: Dynaconf):
         if "ftp" in config:
             self.config = config["ftp"]
 

{datamarket-0.9.5 → datamarket-0.9.7}/src/datamarket/interfaces/peerdb.py

@@ -15,6 +15,13 @@ from tenacity import before_sleep_log, retry, stop_after_attempt, wait_exponenti
 
 from .alchemy import AlchemyInterface
 
+########################################################################################################################
+# EXCEPTIONS
+
+class DatabaseNotConnectedError(Exception):
+    """Custom error for when database is not connected."""
+    pass
+
 ########################################################################################################################
 # CLASSES
 
@@ -105,32 +112,32 @@ class ClickhousePeer:
     def __init__(self, config):
         if "clickhouse" in config:
             self.config = config["clickhouse"]
-            self.ensure_database_exists()
-            self.client = self._create_client(database=self.config["database"])
         else:
             logger.warning("no clickhouse section in config")
 
-    def _create_client(self, database=None):
-        client_config = {
-            "host": self.config["host"],
-            "port": self.config["port"],
-            "user": self.config["user"],
-            "password": self.config["password"],
-        }
-        if database:
-            client_config["database"] = database
-        return clickhouse_driver.Client(**client_config)
+    def connect(self, database):
+        if not database:
+            return
+
+        self.ensure_database_exists(database)
+        self.config["database"] = database
+        self.client = clickhouse_driver.Client(**self.config)
+
+    def _check_connection(self):
+        if self.client is None:
+            raise DatabaseNotConnectedError("Database not connected. Call connect() method first.")
 
-    def ensure_database_exists(self):
-        logger.info(f"Checking if database '{self.config['database']}' exists in Clickhouse")
-        temp_client = self._create_client()
+    def ensure_database_exists(self, database):
+        logger.info(f"Checking if database '{database}' exists in Clickhouse")
+        temp_client = clickhouse_driver.Client(**self.config)
         databases = temp_client.execute("SHOW DATABASES")
-        if self.config["database"] not in [db[0] for db in databases]:
-            logger.info(f"Creating database '{self.config['database']}'")
-            temp_client.execute(f"CREATE DATABASE IF NOT EXISTS {self.config['database']}")
+        if database not in [db[0] for db in databases]:
+            logger.info(f"Creating database '{database}'")
+            temp_client.execute(f"CREATE DATABASE IF NOT EXISTS {database}")
         temp_client.disconnect()
 
     def delete_existing_tables(self, table_names):
+        self._check_connection()
         logger.info(f"Deleting existing tables in Clickhouse for database: {self.config['database']}")
 
         all_tables = self.client.execute("SHOW TABLES")
@@ -156,6 +163,7 @@ class ClickhousePeer:
         logger.info("Finished deleting existing tables in Clickhouse")
 
     def create_row_policies(self, schema_name, table_names):
+        self._check_connection()
         logger.info(f"Creating row policies for schema: {schema_name}")
         for table_name in table_names:
             policy_name = "non_deleted"
@@ -167,6 +175,7 @@ class ClickhousePeer:
             logger.info(f"Created row policy '{policy_name}' for table '{table_name}'")
 
     def execute_sql_file(self, file_path):
+        self._check_connection()
         try:
             with file_path.open("r") as sql_file:
                 sql_content = sql_file.read()
@@ -501,6 +510,7 @@ class PeerDBInterface:
         self.source.create_user(peerdb_user, peerdb_pwd)
         self.source.grant_permissions(schema_name, peerdb_user)
         self.source.create_publication(schema_name, mirror_tablenames)
+        self.destination.connect(schema_name)
         self.create_postgres_peer()
         self.create_clickhouse_peer(schema_name)
         self.pre_init(schema_name, mirror_tablenames, clickhouse_sql_path, resync, hard_resync)

{datamarket-0.9.5 → datamarket-0.9.7}/src/datamarket/utils/main.py

@@ -2,6 +2,7 @@
 # IMPORTS
 
 import asyncio
+from dataclasses import dataclass
 import inspect
 import logging
 import random
@@ -11,23 +12,35 @@ import shutil
 import subprocess
 import time
 from pathlib import Path
-from typing import Literal, Union
+from typing import Literal, Optional, Union
 
 import pendulum
 from croniter import croniter
 from configparser import RawConfigParser
 from dynaconf import Dynaconf, add_converter
 
+logger = logging.getLogger(__name__)
+
 ########################################################################################################################
-# FUNCTIONS
+# CLASSES
+
+
+@dataclass
+class ProjectMetadata:
+    cmd_prefix: str
+    package_name: str
+    env_name: str
+    path: Path
+    config_path: Path
 
-logger = logging.getLogger(__name__)
 
-Config = Union[RawConfigParser, Dynaconf]
+########################################################################################################################
+# FUNCTIONS
 
 
 def get_granular_date(
-    granularity: Union[Literal["monthly", "biweekly", "weekly", "daily"], str], tz: str = "Europe/Madrid"
+    granularity: Union[Literal["monthly", "biweekly", "weekly", "daily"], str],
+    tz: str = "Europe/Madrid",
 ) -> pendulum.DateTime:
     """
     Returns the most recent date based on the given granularity or a custom cron expression.
@@ -67,8 +80,10 @@ def read_converter(path_str: str):
 
 
 def get_config(
-    config_file: Path, tz: str = "Europe/Madrid"
-) -> Union[RawConfigParser, Dynaconf]:
+    config_file: Optional[Path] = None, tz: str = "Europe/Madrid"
+) -> Dynaconf:
+    config_file = config_file or get_project_metadata().config_path
+
     if Path(config_file).suffix == ".ini":
         logger.warning("Using legacy INI config reader. Please migrate to TOML")
         cfg = RawConfigParser()
@@ -106,17 +121,20 @@ def get_config(
 
     return config
 
-
-def get_project_metadata():
+def get_project_metadata() -> ProjectMetadata:
     caller_frame = inspect.stack()[1]
     current_file_parts = Path(caller_frame.filename).resolve().parts
     src_index = current_file_parts.index("src")
+
     cmd_prefix = "dix vnc run --" if shutil.which("dix") else ""
-    pkg_name = current_file_parts[src_index + 1]
-    env_name = f"{pkg_name}_env"
+    package_name = current_file_parts[src_index + 1]
+    env_name = f"{package_name}_env"
     project_path = Path(*current_file_parts[:src_index])
+    config_path = project_path / "config.toml"
 
-    return {"cmd_prefix": cmd_prefix, "pkg_name": pkg_name, "env_name": env_name, "project_path": project_path}
+    return ProjectMetadata(
+        cmd_prefix, package_name, env_name, project_path, config_path
+    )
 
 
 def set_logger(level):