datamarket 0.9.22__tar.gz → 0.9.24__tar.gz

{datamarket-0.9.22 → datamarket-0.9.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.22
+Version: 0.9.24
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.22 → datamarket-0.9.24}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.22"
+version = "0.9.24"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"

{datamarket-0.9.22 → datamarket-0.9.24}/src/datamarket/interfaces/alchemy.py

@@ -6,11 +6,12 @@ from collections.abc import MutableMapping
 from typing import Any, Iterator, List, Optional, Type, TypeVar
 from urllib.parse import quote_plus
 
-from sqlalchemy import DDL, FrozenResult, Result, Select, SQLColumnExpression, create_engine, func, select, text
+from sqlalchemy import DDL, FrozenResult, Result, Select, SQLColumnExpression, create_engine, text
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.declarative import DeclarativeMeta
 from sqlalchemy.orm import Session, sessionmaker
+from enum import Enum
 
 ########################################################################################################################
 # CLASSES
@@ -20,6 +21,11 @@ logger = logging.getLogger(__name__)
 ModelType = TypeVar("ModelType", bound=DeclarativeMeta)
 
 
+class CommitStrategy(Enum):
+    COMMIT_ON_SUCCESS = "commit_on_success"
+    FORCE_COMMIT = "force_commit"
+
+
 class MockContext:
     def __init__(self, column: SQLColumnExpression) -> None:
         self.current_parameters = {}
@@ -188,7 +194,7 @@ class AlchemyInterface:
 
         query_results.update({column_name: default_value}, synchronize_session=False)
 
-    def insert_alchemy_obj(self, alchemy_obj: ModelType, silent: bool = False) -> None:
+    def insert_alchemy_obj(self, alchemy_obj: ModelType, silent: bool = False) -> bool:
         if self.session is None:
             raise RuntimeError("Session not active. Use 'with AlchemyInterface(...):' or call start()")
 
@@ -203,8 +209,11 @@ class AlchemyInterface:
             if not silent:
                 logger.info(f"{alchemy_obj} already in db (savepoint rolled back)")
             # Do not re-raise, allow outer transaction/loop to continue
+            return False
 
-    def upsert_alchemy_obj(self, alchemy_obj: ModelType, index_elements: List[str], silent: bool = False) -> None:
+        return True
+
+    def upsert_alchemy_obj(self, alchemy_obj: ModelType, index_elements: List[str], silent: bool = False) -> bool:
         if self.session is None:
             raise RuntimeError("Session not active. Use 'with AlchemyInterface(...):' or call start()")
 
@@ -233,12 +242,16 @@ class AlchemyInterface:
             if not silent:
                 logger.info(f"could not upsert {alchemy_obj} (savepoint rolled back)")
             # Do not re-raise, allow outer transaction/loop to continue
+            return False
+
+        return True
 
     def windowed_query(
         self,
         stmt: Select[Any],
         order_by: List[SQLColumnExpression[Any]],
         windowsize: int,
+        commit_strategy: CommitStrategy = CommitStrategy.COMMIT_ON_SUCCESS,
     ) -> Iterator[Result[Any]]:
         """
         Executes a windowed query, fetching each window in a separate, short-lived session.
@@ -247,6 +260,8 @@ class AlchemyInterface:
             stmt: The SQL select statement to execute.
             order_by: The columns to use for ordering.
             windowsize: The number of rows to fetch in each window.
+            commit_strategy: The strategy to use for committing the session after each window.
+                             Defaults to CommitStrategy.COMMIT_ON_SUCCESS.
 
         Returns:
             An iterator of Result objects, each containing a window of data.
@@ -254,19 +269,13 @@ class AlchemyInterface:
 
         More info: https://github.com/sqlalchemy/sqlalchemy/wiki/RangeQuery-and-WindowedRangeQuery
         """
-        # Add row_number over the specified order
-        row_number = func.row_number().over(order_by=order_by).label("row_number")
 
-        # Add the windowing column to the statement
-        inner_stmt = stmt.with_session(None).add_columns(row_number).order_by(row_number)
-        subq = inner_stmt.subquery()
-
-        # Create an outer query that selects from the subquery
-        cols = [c for c in subq.c]
-        outer_query = select(*cols)
-
-        last_row_number = 0
+        # Find id column in stmt
+        if not any(column.get("entity").id for column in stmt.column_descriptions):
+            raise Exception("Column 'id' not found in any entity of the query.")
+        id_column = stmt.column_descriptions[0]["entity"].id
 
+        last_id = 0
         while True:
             session_active = False
             commit_needed = False
@@ -275,7 +284,7 @@ class AlchemyInterface:
                 session_active = True
 
                 # Filter on row_number in the outer query
-                current_query = outer_query.where(subq.c.row_number > last_row_number).limit(windowsize)
+                current_query = stmt.where(id_column > last_id).order_by(order_by[0], *order_by[1:]).limit(windowsize)
                 result = self.session.execute(current_query)
 
                 # Create a FrozenResult to allow peeking at the data without consuming
@@ -286,19 +295,25 @@ class AlchemyInterface:
                     break
 
                 # Update for next iteration
-                last_row_number = chunk[-1].row_number
+                last_id = chunk[-1].id
 
                 # Create a new Result object from the FrozenResult
                 yield_result = frozen_result()
 
-                # Remove row_number from result before yielding
-                # Ensure we don't yield the row_number column itself
-                original_col_count = len(cols) - 1
-                yield_result = yield_result.columns(*range(original_col_count))
-
                 yield yield_result
                 commit_needed = True
 
             finally:
                 if session_active and self.session:
-                    self.stop(commit=commit_needed)
+                    if commit_strategy == CommitStrategy.FORCE_COMMIT:
+                        # For forced commit, always attempt to commit.
+                        # The self.stop() method already handles potential exceptions during commit/rollback.
+                        self.stop(commit=True)
+                    elif commit_strategy == CommitStrategy.COMMIT_ON_SUCCESS:
+                        # Commit only if no exception occurred before yielding the result.
+                        self.stop(commit=commit_needed)
+                    else:
+                        # Fallback or error for unknown strategy, though type hinting should prevent this.
+                        # For safety, default to rollback.
+                        logger.warning(f"Unknown commit strategy: {commit_strategy}. Defaulting to rollback.")
+                        self.stop(commit=False)