datamarket 0.9.15__tar.gz → 0.9.16__tar.gz

{datamarket-0.9.15 → datamarket-0.9.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.15
+Version: 0.9.16
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.15 → datamarket-0.9.16}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.15"
+version = "0.9.16"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"

{datamarket-0.9.15 → datamarket-0.9.16}/src/datamarket/interfaces/alchemy.py

@@ -2,16 +2,15 @@
 # IMPORTS
 
 import logging
-from urllib.parse import quote_plus
-from typing import Any, Iterator, List, Type, TypeVar
 from collections.abc import MutableMapping
+from typing import Any, Iterator, List, Type, TypeVar
+from urllib.parse import quote_plus
 
-from sqlalchemy import DDL, Result, Select, SQLColumnExpression, create_engine, text
+from sqlalchemy import DDL, FrozenResult, Result, Select, SQLColumnExpression, create_engine, func, select, text
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.exc import IntegrityError
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy import FrozenResult
 from sqlalchemy.ext.declarative import DeclarativeMeta
+from sqlalchemy.orm import sessionmaker
 
 ########################################################################################################################
 # CLASSES
@@ -180,7 +179,7 @@ class AlchemyInterface:
     def windowed_query(
         self,
         stmt: Select[Any],
-        column: SQLColumnExpression[Any],
+        order_by: List[SQLColumnExpression[Any]],
         windowsize: int,
     ) -> Iterator[Result[Any]]:
         """
@@ -188,26 +187,31 @@ class AlchemyInterface:
 
         Args:
             stmt: The SQL select statement to execute.
-            column: The column to use for windowing and sorting.
+            order_by: The columns to use for ordering.
             windowsize: The number of rows to fetch in each window.
 
         Returns:
             An iterator of Result objects, each containing a window of data.
 
-        Source: https://github.com/sqlalchemy/sqlalchemy/wiki/RangeQuery-and-WindowedRangeQuery
+        More info: https://github.com/sqlalchemy/sqlalchemy/wiki/RangeQuery-and-WindowedRangeQuery
         """
+        # Add row_number over the specified order
+        row_number = func.row_number().over(order_by=order_by).label("row_number")
 
         # Add the windowing column to the statement and sort by it for deterministic results
-        stmt = stmt.add_columns(column).order_by(column)
-        last_id = None
+        inner_stmt = stmt.add_columns(row_number).order_by(row_number)
+        subq = inner_stmt.subquery()
 
-        while True:
-            subq = stmt
+        # Create an outer query that selects from the subquery
+        cols = [c for c in subq.c]
+        outer_query = select(*cols)
 
-            if last_id is not None:
-                subq = subq.filter(column > last_id)
+        last_row_number = 0
 
-            result: Result = self.session.execute(subq.limit(windowsize))
+        while True:
+            # Filter on row_number in the outer query
+            current_query = outer_query.where(subq.c.row_number > last_row_number).limit(windowsize)
+            result = self.session.execute(current_query)
 
             # Create a FrozenResult to allow peeking at the data without consuming
             frozen_result: FrozenResult = result.freeze()
@@ -216,13 +220,13 @@ class AlchemyInterface:
             if not chunk:
                 break
 
-            result_width = len(chunk)
-            last_id = chunk[-1][-1]
+            # Update for next iteration
+            last_row_number = chunk[-1].row_number - 1
 
             # Create a new Result object from the FrozenResult
-            yield_result: Result = frozen_result()
+            yield_result = frozen_result()
 
-            # Remove the windowing column from the result
-            yield_result = yield_result.columns(*list(range(0, result_width - 1)))
+            # Remove row_number from result
+            yield_result = yield_result.columns(*range(len(cols) - 1))
 
             yield yield_result