datamarket 0.9.22__tar.gz → 0.9.23__tar.gz

{datamarket-0.9.22 → datamarket-0.9.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.22
+Version: 0.9.23
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.22 → datamarket-0.9.23}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.22"
+version = "0.9.23"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"

{datamarket-0.9.22 → datamarket-0.9.23}/src/datamarket/interfaces/alchemy.py

@@ -188,7 +188,7 @@ class AlchemyInterface:
 
         query_results.update({column_name: default_value}, synchronize_session=False)
 
-    def insert_alchemy_obj(self, alchemy_obj: ModelType, silent: bool = False) -> None:
+    def insert_alchemy_obj(self, alchemy_obj: ModelType, silent: bool = False) -> bool:
         if self.session is None:
             raise RuntimeError("Session not active. Use 'with AlchemyInterface(...):' or call start()")
 
@@ -203,8 +203,11 @@ class AlchemyInterface:
             if not silent:
                 logger.info(f"{alchemy_obj} already in db (savepoint rolled back)")
             # Do not re-raise, allow outer transaction/loop to continue
+            return False
 
-    def upsert_alchemy_obj(self, alchemy_obj: ModelType, index_elements: List[str], silent: bool = False) -> None:
+        return True
+
+    def upsert_alchemy_obj(self, alchemy_obj: ModelType, index_elements: List[str], silent: bool = False) -> bool:
         if self.session is None:
             raise RuntimeError("Session not active. Use 'with AlchemyInterface(...):' or call start()")
 
@@ -233,6 +236,9 @@ class AlchemyInterface:
             if not silent:
                 logger.info(f"could not upsert {alchemy_obj} (savepoint rolled back)")
             # Do not re-raise, allow outer transaction/loop to continue
+            return False
+
+        return True
 
     def windowed_query(
         self,
@@ -254,19 +260,13 @@ class AlchemyInterface:
 
         More info: https://github.com/sqlalchemy/sqlalchemy/wiki/RangeQuery-and-WindowedRangeQuery
         """
-        # Add row_number over the specified order
-        row_number = func.row_number().over(order_by=order_by).label("row_number")
-
-        # Add the windowing column to the statement
-        inner_stmt = stmt.with_session(None).add_columns(row_number).order_by(row_number)
-        subq = inner_stmt.subquery()
 
-        # Create an outer query that selects from the subquery
-        cols = [c for c in subq.c]
-        outer_query = select(*cols)
-
-        last_row_number = 0
+        # Find id column in stmt
+        if not any(column.get("entity").id for column in stmt.column_descriptions):
+            raise Exception("Column 'id' not found in any entity of the query.")
+        id_column = stmt.column_descriptions[0]["entity"].id
 
+        last_id = 0
         while True:
             session_active = False
             commit_needed = False
@@ -275,7 +275,7 @@ class AlchemyInterface:
                 session_active = True
 
                 # Filter on row_number in the outer query
-                current_query = outer_query.where(subq.c.row_number > last_row_number).limit(windowsize)
+                current_query = stmt.where(id_column > last_id).order_by(order_by[0], *order_by[1:]).limit(windowsize)
                 result = self.session.execute(current_query)
 
                 # Create a FrozenResult to allow peeking at the data without consuming
@@ -286,16 +286,11 @@ class AlchemyInterface:
                     break
 
                 # Update for next iteration
-                last_row_number = chunk[-1].row_number
+                last_id = chunk[-1].id
 
                 # Create a new Result object from the FrozenResult
                 yield_result = frozen_result()
 
-                # Remove row_number from result before yielding
-                # Ensure we don't yield the row_number column itself
-                original_col_count = len(cols) - 1
-                yield_result = yield_result.columns(*range(original_col_count))
-
                 yield yield_result
                 commit_needed = True