datamarket 0.9.19__py3-none-any.whl → 0.9.21__py3-none-any.whl

datamarket/interfaces/alchemy.py

@@ -3,14 +3,14 @@
 
 import logging
 from collections.abc import MutableMapping
-from typing import Any, Iterator, List, Type, TypeVar
+from typing import Any, Iterator, List, Optional, Type, TypeVar
 from urllib.parse import quote_plus
 
 from sqlalchemy import DDL, FrozenResult, Result, Select, SQLColumnExpression, create_engine, func, select, text
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.declarative import DeclarativeMeta
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import Session, sessionmaker
 
 ########################################################################################################################
 # CLASSES
@@ -29,16 +29,58 @@ class MockContext:
 
 class AlchemyInterface:
     def __init__(self, config: MutableMapping) -> None:
+        self.session: Optional[Session] = None
         if "db" in config:
             self.config = config["db"]
-
             self.engine = create_engine(self.get_conn_str())
-            self.session = sessionmaker(bind=self.engine)()
-            self.cursor = self.session.connection().connection.cursor()
-
+            self.Session = sessionmaker(bind=self.engine)
         else:
             logger.warning("no db section in config")
 
+    def __enter__(self) -> "AlchemyInterface":
+        """Enter the runtime context related to this object (starts session)."""
+        self.start()
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        """Exit the runtime context related to this object (stops session)."""
+        should_commit = exc_type is None
+        self.stop(commit=should_commit)
+
+    def start(self) -> None:
+        """Starts a new SQLAlchemy session manually."""
+        if not hasattr(self, "Session"):
+            raise AttributeError("Database configuration not initialized. Cannot create session.")
+        if self.session is not None:
+            raise RuntimeError("Session already active.")
+        self.session = self.Session()
+        logger.debug("SQLAlchemy session started manually.")
+
+    def stop(self, commit: bool = True) -> None:
+        """Stops the manually started SQLAlchemy session."""
+        if self.session is None:
+            logger.warning("No active session to stop.")
+            return
+
+        try:
+            if commit:
+                logger.debug("Committing SQLAlchemy session before stopping.")
+                self.session.commit()
+            else:
+                logger.debug("Rolling back SQLAlchemy session before stopping.")
+                self.session.rollback()
+        except Exception as e:
+            logger.error(f"Exception during session commit/rollback on stop: {e}", exc_info=True)
+            try:
+                self.session.rollback()
+            except Exception as rb_exc:
+                logger.error(f"Exception during secondary rollback attempt on stop: {rb_exc}", exc_info=True)
+            raise
+        finally:
+            logger.debug("Closing SQLAlchemy session.")
+            self.session.close()
+            self.session = None
+
     def get_conn_str(self):
         return (
             f"{self.config['engine']}://"
@@ -110,6 +152,16 @@ class AlchemyInterface:
         self.create_tables(tables)
 
     def reset_column(self, query_results: List[Result[Any]], column_name: str) -> None:
+        """
+        Reset a column to its default value for a list of query results.
+
+        Args:
+            query_results: List of query results to update
+            column_name: Name of the column to reset
+        """
+        if self.session is None:
+            raise RuntimeError("Session not active. Use 'with AlchemyInterface(...):' or call start()")
+
         if not query_results:
             logger.warning("No objects to reset column for.")
             return
@@ -124,33 +176,38 @@ class AlchemyInterface:
 
         column = table.columns[column_name]
 
+        # Determine the default value to use
         if column.server_default is not None:
-            query_results.update({column_name: text("DEFAULT")}, synchronize_session=False)
+            default_value = text("DEFAULT")
         elif column.default is not None:
             default_value = column.default.arg
             if callable(default_value):
                 default_value = default_value(MockContext(column))
-            query_results.update({column_name: default_value}, synchronize_session=False)
         else:
             raise ValueError(f"Column '{column_name}' doesn't have a default value defined.")
 
-        self.session.commit()
+        query_results.update({column_name: default_value}, synchronize_session=False)
 
     def insert_alchemy_obj(self, alchemy_obj: ModelType, silent: bool = False) -> None:
-        try:
-            if not silent:
-                logger.info(f"adding {alchemy_obj}...")
-
-            self.session.add(alchemy_obj)
-            self.session.commit()
+        if self.session is None:
+            raise RuntimeError("Session not active. Use 'with AlchemyInterface(...):' or call start()")
 
+        try:
+            # Use a savepoint (nested transaction)
+            with self.session.begin_nested():
+                if not silent:
+                    logger.info(f"adding {alchemy_obj}...")
+                self.session.add(alchemy_obj)
         except IntegrityError:
+            # Rollback is handled automatically by begin_nested() context manager on error
             if not silent:
-                logger.info(f"{alchemy_obj} already in db")
-
-            self.session.rollback()
+                logger.info(f"{alchemy_obj} already in db (savepoint rolled back)")
+            # Do not re-raise, allow outer transaction/loop to continue
 
     def upsert_alchemy_obj(self, alchemy_obj: ModelType, index_elements: List[str], silent: bool = False) -> None:
+        if self.session is None:
+            raise RuntimeError("Session not active. Use 'with AlchemyInterface(...):' or call start()")
+
         if not silent:
             logger.info(f"upserting {alchemy_obj}")
 
@@ -168,13 +225,14 @@ class AlchemyInterface:
         )
 
         try:
-            self.session.execute(statement)
-            self.session.commit()
+            # Use a savepoint (nested transaction)
+            with self.session.begin_nested():
+                self.session.execute(statement)
         except IntegrityError:
+            # Rollback is handled automatically by begin_nested() context manager on error
             if not silent:
-                logger.info(f"could not upsert {alchemy_obj}")
-
-            self.session.rollback()
+                logger.info(f"could not upsert {alchemy_obj} (savepoint rolled back)")
+            # Do not re-raise, allow outer transaction/loop to continue
 
     def windowed_query(
         self,
@@ -183,7 +241,7 @@ class AlchemyInterface:
         windowsize: int,
     ) -> Iterator[Result[Any]]:
         """
-        Executes a windowed query on the given statement.
+        Executes a windowed query, fetching each window in a separate, short-lived session.
 
         Args:
             stmt: The SQL select statement to execute.
@@ -192,14 +250,15 @@ class AlchemyInterface:
 
         Returns:
             An iterator of Result objects, each containing a window of data.
+            The session used to fetch the Result is closed immediately after yielding.
 
         More info: https://github.com/sqlalchemy/sqlalchemy/wiki/RangeQuery-and-WindowedRangeQuery
         """
         # Add row_number over the specified order
         row_number = func.row_number().over(order_by=order_by).label("row_number")
 
-        # Add the windowing column to the statement and sort by it for deterministic results
-        inner_stmt = stmt.add_columns(row_number).order_by(row_number)
+        # Add the windowing column to the statement
+        inner_stmt = stmt.with_session(None).add_columns(row_number).order_by(row_number)
         subq = inner_stmt.subquery()
 
         # Create an outer query that selects from the subquery
@@ -209,24 +268,37 @@ class AlchemyInterface:
         last_row_number = 0
 
         while True:
-            # Filter on row_number in the outer query
-            current_query = outer_query.where(subq.c.row_number > last_row_number).limit(windowsize)
-            result = self.session.execute(current_query)
+            session_active = False
+            commit_needed = False
+            try:
+                self.start()
+                session_active = True
+
+                # Filter on row_number in the outer query
+                current_query = outer_query.where(subq.c.row_number > last_row_number).limit(windowsize)
+                result = self.session.execute(current_query)
+
+                # Create a FrozenResult to allow peeking at the data without consuming
+                frozen_result: FrozenResult = result.freeze()
+                chunk = frozen_result().all()
 
-            # Create a FrozenResult to allow peeking at the data without consuming
-            frozen_result: FrozenResult = result.freeze()
-            chunk = frozen_result().all()
+                if not chunk:
+                    break
 
-            if not chunk:
-                break
+                # Update for next iteration
+                last_row_number = chunk[-1].row_number
 
-            # Update for next iteration
-            last_row_number = chunk[-1].row_number - 1
+                # Create a new Result object from the FrozenResult
+                yield_result = frozen_result()
 
-            # Create a new Result object from the FrozenResult
-            yield_result = frozen_result()
+                # Remove row_number from result before yielding
+                # Ensure we don't yield the row_number column itself
+                original_col_count = len(cols) - 1
+                yield_result = yield_result.columns(*range(original_col_count))
 
-            # Remove row_number from result
-            yield_result = yield_result.columns(*range(len(cols) - 1))
+                yield yield_result
+                commit_needed = True
 
-            yield yield_result
+            finally:
+                if session_active and self.session:
+                    self.stop(commit=commit_needed)

datamarket/utils/main.py

@@ -2,8 +2,6 @@
 # IMPORTS
 
 import asyncio
-from dataclasses import dataclass
-import inspect
 import logging
 import random
 import re
@@ -12,26 +10,25 @@ import shutil
 import subprocess
 import time
 from pathlib import Path
-from typing import Literal, Optional, Union
+from typing import Any, Literal, Self, Union
 
 import pendulum
 from croniter import croniter
-from configparser import RawConfigParser
 from dynaconf import Dynaconf, add_converter
 
 logger = logging.getLogger(__name__)
 
-########################################################################################################################
-# CLASSES
+
+class NoProjectFoundError(Exception):
+    def __init__(self):
+        super().__init__("Could not detect project. Did you add the config file?")
 
 
-@dataclass
-class ProjectMetadata:
-    cmd_prefix: str
-    package_name: str
-    env_name: str
-    path: Path
-    config_path: Path
+class NoPackageFoundError(Exception):
+    def __init__(self):
+        super().__init__(
+            "A project was detected but it has no packages inside the 'src' directory"
+        )
 
 
 ########################################################################################################################
@@ -79,62 +76,80 @@ def read_converter(path_str: str):
         return f.read()
 
 
-def get_config(
-    config_file: Optional[Path] = None, tz: str = "Europe/Madrid"
-) -> Dynaconf:
-    config_file = config_file or get_project_metadata().config_path
-
-    if Path(config_file).suffix == ".ini":
-        logger.warning("Using legacy INI config reader. Please migrate to TOML")
-        cfg = RawConfigParser()
-        cfg.read(config_file)
-        return cfg
-
-    add_converter("read", read_converter)
-
-    dt_now = get_granular_date("now", tz)
-    dt_weekly = get_granular_date("weekly", tz)
-    dt_biweekly = get_granular_date("biweekly", tz)
-
-    config = Dynaconf(
-        environments=True,
-        env_switcher="SYSTYPE",
-    )
-
-    config.load_file(path=config_file)
-    config.load_file(path=Path.home() / config_file.name)
-
-    config.vars = {
-        "year": dt_now.strftime("%Y"),
-        "month": dt_now.strftime("%m"),
-        "day": dt_now.strftime("%d"),
-        "now": dt_now.strftime("%Y-%m-%d %H:%M:%S"),
-        "now_stripped": dt_now.strftime("%Y%m%d%H%M%S"),
-        "today": dt_now.strftime("%Y-%m-%d"),
-        "today_stripped": dt_now.strftime("%Y%m%d"),
-        "weekly_date": dt_weekly.strftime("%Y-%m-%d"),
-        "weekly_date_stripped": dt_weekly.strftime("%Y%m%d"),
-        "biweekly_date": dt_biweekly.strftime("%Y-%m-%d"),
-        "biweekly_date_stripped": dt_biweekly.strftime("%Y%m%d"),
-        "dynaconf_merge": True,
-    }
-
-    return config
-
-def get_project_metadata() -> ProjectMetadata:
-    caller_frame = inspect.stack()[1]
-    current_file_parts = Path(caller_frame.filename).resolve().parts
-    src_index = current_file_parts.index("src")
-
-    cmd_prefix = "dix vnc run --" if shutil.which("dix") else ""
-    package_name = current_file_parts[src_index + 1]
-    env_name = f"{package_name}_env"
-    project_path = Path(*current_file_parts[:src_index])
-    config_path = project_path / "config.toml"
-
-    return ProjectMetadata(
-        cmd_prefix, package_name, env_name, project_path, config_path
-    )
+class Project:
+    CONFIG_FILE_NAME = "config.toml"
+
+    def __init__(self, path: Path):
+        self.path = path
+
+        try:
+            self.pkg_name = next((self.path / "src").glob("*")).name
+        except StopIteration:
+            raise NoPackageFoundError()
+
+        self.env_name = f"{self.pkg_name}_env"
+        self.config_path = self.path / self.CONFIG_FILE_NAME
+        self.tests_path = self.path / "tests"
+        self.cmd_prefix = "dix vnc run --" if shutil.which("dix") else ""
+
+    @classmethod
+    def from_file(cls, file: Path | str) -> Self:
+        file_path = Path(file).resolve()
+
+        while file_path != file_path.parent:
+            config_file = file_path / cls.CONFIG_FILE_NAME
+
+            if not config_file.is_file():
+                file_path = file_path.parent
+                continue
+
+            if file_path.parent == Path("/home"):
+                raise NoProjectFoundError()
+
+            return cls(file_path)
+
+        raise NoProjectFoundError()
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "project_path": self.path,
+            "cmd_prefix": self.cmd_prefix,
+            "env_name": self.env_name,
+            "pkg_name": self.pkg_name,
+            "config_path": self.config_path,
+        }
+
+    def get_config(self, tz: str = "Europe/Madrid") -> Dynaconf:
+        add_converter("read", read_converter)
+
+        dt_now = get_granular_date("now", tz)
+        dt_weekly = get_granular_date("weekly", tz)
+        dt_biweekly = get_granular_date("biweekly", tz)
+
+        config = Dynaconf(
+            environments=True,
+            env_switcher="SYSTYPE",
+        )
+
+        config.load_file(path=self.config_path)
+        config.load_file(path=Path.home() / self.config_path.name)
+
+        config.vars = {
+            "year": dt_now.strftime("%Y"),
+            "month": dt_now.strftime("%m"),
+            "day": dt_now.strftime("%d"),
+            "now": dt_now.strftime("%Y-%m-%d %H:%M:%S"),
+            "now_stripped": dt_now.strftime("%Y%m%d%H%M%S"),
+            "today": dt_now.strftime("%Y-%m-%d"),
+            "today_stripped": dt_now.strftime("%Y%m%d"),
+            "weekly_date": dt_weekly.strftime("%Y-%m-%d"),
+            "weekly_date_stripped": dt_weekly.strftime("%Y%m%d"),
+            "biweekly_date": dt_biweekly.strftime("%Y-%m-%d"),
+            "biweekly_date_stripped": dt_biweekly.strftime("%Y%m%d"),
+            "dynaconf_merge": True,
+        }
+
+        return config
 
 
 def set_logger(level):

{datamarket-0.9.19.dist-info → datamarket-0.9.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.19
+Version: 0.9.21
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.19.dist-info → datamarket-0.9.21.dist-info}/RECORD

@@ -1,6 +1,6 @@
 datamarket/__init__.py,sha256=FHS77P9qNewKMoN-p0FLEUEC60oWIYup1QkbJZP4ays,12
 datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamarket/interfaces/alchemy.py,sha256=z7VarlKZ-JfsXWtuDXCYnNp_pSzEYo5IvrD7wqoRpbI,8891
+datamarket/interfaces/alchemy.py,sha256=Qxgi_-TSzlDlUPy9SV1GoEfg1VzLx4KnPcSRVT-BFdc,12365
 datamarket/interfaces/aws.py,sha256=7KLUeBxmPN7avEMPsu5HC_KHB1N7W6Anp2X8fo43mlw,2383
 datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIgE,2915
 datamarket/interfaces/ftp.py,sha256=o0KlJxtksbop9OjCiQRzyAa2IeG_ExVXagS6apwrAQo,1881
@@ -13,11 +13,11 @@ datamarket/params/nominatim.py,sha256=pBYRfoBkkLBg2INbFymefmYSzaAVujQSpEro5c1hD_
 datamarket/utils/__init__.py,sha256=8D5a8oKgqd6WA1RUkiKCn4l_PVemtyuckxQut0vDHXM,20
 datamarket/utils/airflow.py,sha256=al0vc0YUikNu3Oy51VSn52I7pMU40akFBOl_UlHa2E4,795
 datamarket/utils/alchemy.py,sha256=SRq6kgh1aANXVShBPgAuglmNhZssPWwWEY503gKSia8,635
-datamarket/utils/main.py,sha256=O6rX-65h4h0j2zs9dofdTPlly5reKDnvgLtTwbLmbWg,6529
+datamarket/utils/main.py,sha256=j8wnAxeLvijdRU9M4V6HunWH7vgWWHP4u4xamzkWcUU,7009
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
-datamarket-0.9.19.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.19.dist-info/METADATA,sha256=yMqp6i6mEyKhwB6VOxXF5Gg1sldJSbkuB9HH2w3NdPQ,6459
-datamarket-0.9.19.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-datamarket-0.9.19.dist-info/RECORD,,
+datamarket-0.9.21.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.21.dist-info/METADATA,sha256=OQQsXVewgsgXDBi86Ld6OddXq7VHTSZXA-zbZf-yH-U,6459
+datamarket-0.9.21.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+datamarket-0.9.21.dist-info/RECORD,,