datamarket 0.9.22__py3-none-any.whl → 0.9.23__py3-none-any.whl

datamarket/interfaces/alchemy.py

@@ -188,7 +188,7 @@ class AlchemyInterface:
 
         query_results.update({column_name: default_value}, synchronize_session=False)
 
-    def insert_alchemy_obj(self, alchemy_obj: ModelType, silent: bool = False) -> None:
+    def insert_alchemy_obj(self, alchemy_obj: ModelType, silent: bool = False) -> bool:
         if self.session is None:
             raise RuntimeError("Session not active. Use 'with AlchemyInterface(...):' or call start()")
 
@@ -203,8 +203,11 @@ class AlchemyInterface:
             if not silent:
                 logger.info(f"{alchemy_obj} already in db (savepoint rolled back)")
             # Do not re-raise, allow outer transaction/loop to continue
+            return False
 
-    def upsert_alchemy_obj(self, alchemy_obj: ModelType, index_elements: List[str], silent: bool = False) -> None:
+        return True
+
+    def upsert_alchemy_obj(self, alchemy_obj: ModelType, index_elements: List[str], silent: bool = False) -> bool:
         if self.session is None:
             raise RuntimeError("Session not active. Use 'with AlchemyInterface(...):' or call start()")
 
@@ -233,6 +236,9 @@ class AlchemyInterface:
             if not silent:
                 logger.info(f"could not upsert {alchemy_obj} (savepoint rolled back)")
             # Do not re-raise, allow outer transaction/loop to continue
+            return False
+
+        return True
 
     def windowed_query(
         self,
@@ -254,19 +260,13 @@ class AlchemyInterface:
 
         More info: https://github.com/sqlalchemy/sqlalchemy/wiki/RangeQuery-and-WindowedRangeQuery
         """
-        # Add row_number over the specified order
-        row_number = func.row_number().over(order_by=order_by).label("row_number")
-
-        # Add the windowing column to the statement
-        inner_stmt = stmt.with_session(None).add_columns(row_number).order_by(row_number)
-        subq = inner_stmt.subquery()
 
-        # Create an outer query that selects from the subquery
-        cols = [c for c in subq.c]
-        outer_query = select(*cols)
-
-        last_row_number = 0
+        # Find id column in stmt
+        if not any(column.get("entity").id for column in stmt.column_descriptions):
+            raise Exception("Column 'id' not found in any entity of the query.")
+        id_column = stmt.column_descriptions[0]["entity"].id
 
+        last_id = 0
         while True:
             session_active = False
             commit_needed = False
@@ -275,7 +275,7 @@ class AlchemyInterface:
                 session_active = True
 
                 # Filter on row_number in the outer query
-                current_query = outer_query.where(subq.c.row_number > last_row_number).limit(windowsize)
+                current_query = stmt.where(id_column > last_id).order_by(order_by[0], *order_by[1:]).limit(windowsize)
                 result = self.session.execute(current_query)
 
                 # Create a FrozenResult to allow peeking at the data without consuming
@@ -286,16 +286,11 @@ class AlchemyInterface:
                     break
 
                 # Update for next iteration
-                last_row_number = chunk[-1].row_number
+                last_id = chunk[-1].id
 
                 # Create a new Result object from the FrozenResult
                 yield_result = frozen_result()
 
-                # Remove row_number from result before yielding
-                # Ensure we don't yield the row_number column itself
-                original_col_count = len(cols) - 1
-                yield_result = yield_result.columns(*range(original_col_count))
-
                 yield yield_result
                 commit_needed = True
 

{datamarket-0.9.22.dist-info → datamarket-0.9.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.22
+Version: 0.9.23
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.22.dist-info → datamarket-0.9.23.dist-info}/RECORD

@@ -1,6 +1,6 @@
 datamarket/__init__.py,sha256=FHS77P9qNewKMoN-p0FLEUEC60oWIYup1QkbJZP4ays,12
 datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamarket/interfaces/alchemy.py,sha256=Qxgi_-TSzlDlUPy9SV1GoEfg1VzLx4KnPcSRVT-BFdc,12365
+datamarket/interfaces/alchemy.py,sha256=-oO-ZBQcv6ixn-N4rEbJcfRjT_54YkMr37hRqc8QKfA,11993
 datamarket/interfaces/aws.py,sha256=7KLUeBxmPN7avEMPsu5HC_KHB1N7W6Anp2X8fo43mlw,2383
 datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIgE,2915
 datamarket/interfaces/ftp.py,sha256=o0KlJxtksbop9OjCiQRzyAa2IeG_ExVXagS6apwrAQo,1881
@@ -18,7 +18,7 @@ datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
 datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
-datamarket-0.9.22.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.22.dist-info/METADATA,sha256=P21dnfivUdzY529pRAh83IVfa48at80lm7NuomuMuF4,6459
-datamarket-0.9.22.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-datamarket-0.9.22.dist-info/RECORD,,
+datamarket-0.9.23.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.23.dist-info/METADATA,sha256=iHO101fsdYc5iuQJbhEeVe8x6LBIl_mkofNpRP4Vjew,6459
+datamarket-0.9.23.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datamarket-0.9.23.dist-info/RECORD,,

{datamarket-0.9.22.dist-info → datamarket-0.9.23.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.2
+Generator: poetry-core 2.1.3
 Root-Is-Purelib: true
 Tag: py3-none-any