datamarket 0.9.22__py3-none-any.whl → 0.9.24__py3-none-any.whl

datamarket/interfaces/alchemy.py

@@ -6,11 +6,12 @@ from collections.abc import MutableMapping
 from typing import Any, Iterator, List, Optional, Type, TypeVar
 from urllib.parse import quote_plus
 
-from sqlalchemy import DDL, FrozenResult, Result, Select, SQLColumnExpression, create_engine, func, select, text
+from sqlalchemy import DDL, FrozenResult, Result, Select, SQLColumnExpression, create_engine, text
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.declarative import DeclarativeMeta
 from sqlalchemy.orm import Session, sessionmaker
+from enum import Enum
 
 ########################################################################################################################
 # CLASSES
@@ -20,6 +21,11 @@ logger = logging.getLogger(__name__)
 ModelType = TypeVar("ModelType", bound=DeclarativeMeta)
 
 
+class CommitStrategy(Enum):
+    COMMIT_ON_SUCCESS = "commit_on_success"
+    FORCE_COMMIT = "force_commit"
+
+
 class MockContext:
     def __init__(self, column: SQLColumnExpression) -> None:
         self.current_parameters = {}
@@ -188,7 +194,7 @@ class AlchemyInterface:
 
         query_results.update({column_name: default_value}, synchronize_session=False)
 
-    def insert_alchemy_obj(self, alchemy_obj: ModelType, silent: bool = False) -> None:
+    def insert_alchemy_obj(self, alchemy_obj: ModelType, silent: bool = False) -> bool:
         if self.session is None:
             raise RuntimeError("Session not active. Use 'with AlchemyInterface(...):' or call start()")
 
@@ -203,8 +209,11 @@ class AlchemyInterface:
             if not silent:
                 logger.info(f"{alchemy_obj} already in db (savepoint rolled back)")
             # Do not re-raise, allow outer transaction/loop to continue
+            return False
 
-    def upsert_alchemy_obj(self, alchemy_obj: ModelType, index_elements: List[str], silent: bool = False) -> None:
+        return True
+
+    def upsert_alchemy_obj(self, alchemy_obj: ModelType, index_elements: List[str], silent: bool = False) -> bool:
         if self.session is None:
             raise RuntimeError("Session not active. Use 'with AlchemyInterface(...):' or call start()")
 
@@ -233,12 +242,16 @@ class AlchemyInterface:
             if not silent:
                 logger.info(f"could not upsert {alchemy_obj} (savepoint rolled back)")
             # Do not re-raise, allow outer transaction/loop to continue
+            return False
+
+        return True
 
     def windowed_query(
         self,
         stmt: Select[Any],
         order_by: List[SQLColumnExpression[Any]],
         windowsize: int,
+        commit_strategy: CommitStrategy = CommitStrategy.COMMIT_ON_SUCCESS,
     ) -> Iterator[Result[Any]]:
         """
         Executes a windowed query, fetching each window in a separate, short-lived session.
@@ -247,6 +260,8 @@ class AlchemyInterface:
             stmt: The SQL select statement to execute.
             order_by: The columns to use for ordering.
             windowsize: The number of rows to fetch in each window.
+            commit_strategy: The strategy to use for committing the session after each window.
+                             Defaults to CommitStrategy.COMMIT_ON_SUCCESS.
 
         Returns:
             An iterator of Result objects, each containing a window of data.
@@ -254,19 +269,13 @@ class AlchemyInterface:
 
         More info: https://github.com/sqlalchemy/sqlalchemy/wiki/RangeQuery-and-WindowedRangeQuery
         """
-        # Add row_number over the specified order
-        row_number = func.row_number().over(order_by=order_by).label("row_number")
 
-        # Add the windowing column to the statement
-        inner_stmt = stmt.with_session(None).add_columns(row_number).order_by(row_number)
-        subq = inner_stmt.subquery()
-
-        # Create an outer query that selects from the subquery
-        cols = [c for c in subq.c]
-        outer_query = select(*cols)
-
-        last_row_number = 0
+        # Find id column in stmt
+        if not any(column.get("entity").id for column in stmt.column_descriptions):
+            raise Exception("Column 'id' not found in any entity of the query.")
+        id_column = stmt.column_descriptions[0]["entity"].id
 
+        last_id = 0
         while True:
             session_active = False
             commit_needed = False
@@ -275,7 +284,7 @@ class AlchemyInterface:
                 session_active = True
 
                 # Filter on row_number in the outer query
-                current_query = outer_query.where(subq.c.row_number > last_row_number).limit(windowsize)
+                current_query = stmt.where(id_column > last_id).order_by(order_by[0], *order_by[1:]).limit(windowsize)
                 result = self.session.execute(current_query)
 
                 # Create a FrozenResult to allow peeking at the data without consuming
@@ -286,19 +295,25 @@ class AlchemyInterface:
                     break
 
                 # Update for next iteration
-                last_row_number = chunk[-1].row_number
+                last_id = chunk[-1].id
 
                 # Create a new Result object from the FrozenResult
                 yield_result = frozen_result()
 
-                # Remove row_number from result before yielding
-                # Ensure we don't yield the row_number column itself
-                original_col_count = len(cols) - 1
-                yield_result = yield_result.columns(*range(original_col_count))
-
                 yield yield_result
                 commit_needed = True
 
             finally:
                 if session_active and self.session:
-                    self.stop(commit=commit_needed)
+                    if commit_strategy == CommitStrategy.FORCE_COMMIT:
+                        # For forced commit, always attempt to commit.
+                        # The self.stop() method already handles potential exceptions during commit/rollback.
+                        self.stop(commit=True)
+                    elif commit_strategy == CommitStrategy.COMMIT_ON_SUCCESS:
+                        # Commit only if no exception occurred before yielding the result.
+                        self.stop(commit=commit_needed)
+                    else:
+                        # Fallback or error for unknown strategy, though type hinting should prevent this.
+                        # For safety, default to rollback.
+                        logger.warning(f"Unknown commit strategy: {commit_strategy}. Defaulting to rollback.")
+                        self.stop(commit=False)

{datamarket-0.9.22.dist-info → datamarket-0.9.24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.22
+Version: 0.9.24
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.22.dist-info → datamarket-0.9.24.dist-info}/RECORD

@@ -1,6 +1,6 @@
 datamarket/__init__.py,sha256=FHS77P9qNewKMoN-p0FLEUEC60oWIYup1QkbJZP4ays,12
 datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamarket/interfaces/alchemy.py,sha256=Qxgi_-TSzlDlUPy9SV1GoEfg1VzLx4KnPcSRVT-BFdc,12365
+datamarket/interfaces/alchemy.py,sha256=pL71oG0ld2gx6uGSdgBzSv6RnQPJ3Ps2yTVPCCTXGLU,13178
 datamarket/interfaces/aws.py,sha256=7KLUeBxmPN7avEMPsu5HC_KHB1N7W6Anp2X8fo43mlw,2383
 datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIgE,2915
 datamarket/interfaces/ftp.py,sha256=o0KlJxtksbop9OjCiQRzyAa2IeG_ExVXagS6apwrAQo,1881
@@ -18,7 +18,7 @@ datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
 datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
-datamarket-0.9.22.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.22.dist-info/METADATA,sha256=P21dnfivUdzY529pRAh83IVfa48at80lm7NuomuMuF4,6459
-datamarket-0.9.22.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-datamarket-0.9.22.dist-info/RECORD,,
+datamarket-0.9.24.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.24.dist-info/METADATA,sha256=18l6aTHI7ejx_26UJ6QtDhPdMbIGMtiToq-qFrx2EVw,6459
+datamarket-0.9.24.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datamarket-0.9.24.dist-info/RECORD,,

{datamarket-0.9.22.dist-info → datamarket-0.9.24.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.2
+Generator: poetry-core 2.1.3
 Root-Is-Purelib: true
 Tag: py3-none-any