datamarket 0.9.15__py3-none-any.whl → 0.9.16__py3-none-any.whl

datamarket/interfaces/alchemy.py

@@ -2,16 +2,15 @@
 # IMPORTS
 
 import logging
-from urllib.parse import quote_plus
-from typing import Any, Iterator, List, Type, TypeVar
 from collections.abc import MutableMapping
+from typing import Any, Iterator, List, Type, TypeVar
+from urllib.parse import quote_plus
 
-from sqlalchemy import DDL, Result, Select, SQLColumnExpression, create_engine, text
+from sqlalchemy import DDL, FrozenResult, Result, Select, SQLColumnExpression, create_engine, func, select, text
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.exc import IntegrityError
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy import FrozenResult
 from sqlalchemy.ext.declarative import DeclarativeMeta
+from sqlalchemy.orm import sessionmaker
 
 ########################################################################################################################
 # CLASSES
@@ -180,7 +179,7 @@ class AlchemyInterface:
     def windowed_query(
         self,
         stmt: Select[Any],
-        column: SQLColumnExpression[Any],
+        order_by: List[SQLColumnExpression[Any]],
         windowsize: int,
     ) -> Iterator[Result[Any]]:
         """
@@ -188,26 +187,31 @@ class AlchemyInterface:
 
         Args:
             stmt: The SQL select statement to execute.
-            column: The column to use for windowing and sorting.
+            order_by: The columns to use for ordering.
             windowsize: The number of rows to fetch in each window.
 
         Returns:
             An iterator of Result objects, each containing a window of data.
 
-        Source: https://github.com/sqlalchemy/sqlalchemy/wiki/RangeQuery-and-WindowedRangeQuery
+        More info: https://github.com/sqlalchemy/sqlalchemy/wiki/RangeQuery-and-WindowedRangeQuery
         """
+        # Add row_number over the specified order
+        row_number = func.row_number().over(order_by=order_by).label("row_number")
 
         # Add the windowing column to the statement and sort by it for deterministic results
-        stmt = stmt.add_columns(column).order_by(column)
-        last_id = None
+        inner_stmt = stmt.add_columns(row_number).order_by(row_number)
+        subq = inner_stmt.subquery()
 
-        while True:
-            subq = stmt
+        # Create an outer query that selects from the subquery
+        cols = [c for c in subq.c]
+        outer_query = select(*cols)
 
-            if last_id is not None:
-                subq = subq.filter(column > last_id)
+        last_row_number = 0
 
-            result: Result = self.session.execute(subq.limit(windowsize))
+        while True:
+            # Filter on row_number in the outer query
+            current_query = outer_query.where(subq.c.row_number > last_row_number).limit(windowsize)
+            result = self.session.execute(current_query)
 
             # Create a FrozenResult to allow peeking at the data without consuming
             frozen_result: FrozenResult = result.freeze()
@@ -216,13 +220,13 @@ class AlchemyInterface:
             if not chunk:
                 break
 
-            result_width = len(chunk)
-            last_id = chunk[-1][-1]
+            # Update for next iteration
+            last_row_number = chunk[-1].row_number - 1
 
             # Create a new Result object from the FrozenResult
-            yield_result: Result = frozen_result()
+            yield_result = frozen_result()
 
-            # Remove the windowing column from the result
-            yield_result = yield_result.columns(*list(range(0, result_width - 1)))
+            # Remove row_number from result
+            yield_result = yield_result.columns(*range(len(cols) - 1))
 
             yield yield_result

{datamarket-0.9.15.dist-info → datamarket-0.9.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.15
+Version: 0.9.16
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.15.dist-info → datamarket-0.9.16.dist-info}/RECORD

@@ -1,6 +1,6 @@
 datamarket/__init__.py,sha256=FHS77P9qNewKMoN-p0FLEUEC60oWIYup1QkbJZP4ays,12
 datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamarket/interfaces/alchemy.py,sha256=Y1wKSEBlWXCE3-6JypdaUL2i_7FAGa9PK4_zQHeM2SE,8563
+datamarket/interfaces/alchemy.py,sha256=z7VarlKZ-JfsXWtuDXCYnNp_pSzEYo5IvrD7wqoRpbI,8891
 datamarket/interfaces/aws.py,sha256=7KLUeBxmPN7avEMPsu5HC_KHB1N7W6Anp2X8fo43mlw,2383
 datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIgE,2915
 datamarket/interfaces/ftp.py,sha256=o0KlJxtksbop9OjCiQRzyAa2IeG_ExVXagS6apwrAQo,1881
@@ -17,7 +17,7 @@ datamarket/utils/main.py,sha256=O6rX-65h4h0j2zs9dofdTPlly5reKDnvgLtTwbLmbWg,6529
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
-datamarket-0.9.15.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.15.dist-info/METADATA,sha256=LzfDrq6DA-PUfIHU19ZOy0n_x_txIMqXu6PKl-pmmxI,6363
-datamarket-0.9.15.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-datamarket-0.9.15.dist-info/RECORD,,
+datamarket-0.9.16.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.16.dist-info/METADATA,sha256=DsHOm4WsenKNFphVyqbLI3A3WluNWGGgVFZRlE8U8NI,6363
+datamarket-0.9.16.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+datamarket-0.9.16.dist-info/RECORD,,