datamarket 0.9.15__py3-none-any.whl → 0.9.17__py3-none-any.whl

datamarket/interfaces/alchemy.py

@@ -2,16 +2,15 @@
 # IMPORTS
 
 import logging
-from urllib.parse import quote_plus
-from typing import Any, Iterator, List, Type, TypeVar
 from collections.abc import MutableMapping
+from typing import Any, Iterator, List, Type, TypeVar
+from urllib.parse import quote_plus
 
-from sqlalchemy import DDL, Result, Select, SQLColumnExpression, create_engine, text
+from sqlalchemy import DDL, FrozenResult, Result, Select, SQLColumnExpression, create_engine, func, select, text
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.exc import IntegrityError
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy import FrozenResult
 from sqlalchemy.ext.declarative import DeclarativeMeta
+from sqlalchemy.orm import sessionmaker
 
 ########################################################################################################################
 # CLASSES
@@ -180,7 +179,7 @@ class AlchemyInterface:
     def windowed_query(
         self,
         stmt: Select[Any],
-        column: SQLColumnExpression[Any],
+        order_by: List[SQLColumnExpression[Any]],
         windowsize: int,
     ) -> Iterator[Result[Any]]:
         """
@@ -188,26 +187,31 @@ class AlchemyInterface:
 
         Args:
             stmt: The SQL select statement to execute.
-            column: The column to use for windowing and sorting.
+            order_by: The columns to use for ordering.
             windowsize: The number of rows to fetch in each window.
 
         Returns:
             An iterator of Result objects, each containing a window of data.
 
-        Source: https://github.com/sqlalchemy/sqlalchemy/wiki/RangeQuery-and-WindowedRangeQuery
+        More info: https://github.com/sqlalchemy/sqlalchemy/wiki/RangeQuery-and-WindowedRangeQuery
         """
+        # Add row_number over the specified order
+        row_number = func.row_number().over(order_by=order_by).label("row_number")
 
         # Add the windowing column to the statement and sort by it for deterministic results
-        stmt = stmt.add_columns(column).order_by(column)
-        last_id = None
+        inner_stmt = stmt.add_columns(row_number).order_by(row_number)
+        subq = inner_stmt.subquery()
 
-        while True:
-            subq = stmt
+        # Create an outer query that selects from the subquery
+        cols = [c for c in subq.c]
+        outer_query = select(*cols)
 
-            if last_id is not None:
-                subq = subq.filter(column > last_id)
+        last_row_number = 0
 
-            result: Result = self.session.execute(subq.limit(windowsize))
+        while True:
+            # Filter on row_number in the outer query
+            current_query = outer_query.where(subq.c.row_number > last_row_number).limit(windowsize)
+            result = self.session.execute(current_query)
 
             # Create a FrozenResult to allow peeking at the data without consuming
             frozen_result: FrozenResult = result.freeze()
@@ -216,13 +220,13 @@ class AlchemyInterface:
             if not chunk:
                 break
 
-            result_width = len(chunk)
-            last_id = chunk[-1][-1]
+            # Update for next iteration
+            last_row_number = chunk[-1].row_number - 1
 
             # Create a new Result object from the FrozenResult
-            yield_result: Result = frozen_result()
+            yield_result = frozen_result()
 
-            # Remove the windowing column from the result
-            yield_result = yield_result.columns(*list(range(0, result_width - 1)))
+            # Remove row_number from result
+            yield_result = yield_result.columns(*range(len(cols) - 1))
 
             yield yield_result

datamarket/interfaces/peerdb.py

@@ -18,10 +18,13 @@ from .alchemy import AlchemyInterface
 ########################################################################################################################
 # EXCEPTIONS
 
+
 class DatabaseNotConnectedError(Exception):
     """Custom error for when database is not connected."""
+
     pass
 
+
 ########################################################################################################################
 # CLASSES
 
@@ -78,10 +81,17 @@ class PostgresPeer:
         with self.engine.connect() as conn:
             conn.execute(text(f"DROP PUBLICATION IF EXISTS {schema_name}_peerdb"))
 
-            table_list = ", ".join([f"{schema_name}.{table}" for table in table_names])
+            table_list = []
+            for table in table_names:
+                full_table_name = f'"{schema_name}"."{table}"'
+                logger.info(f"Setting REPLICA IDENTITY FULL for table: {full_table_name}")
+                conn.execute(text(f"ALTER TABLE {full_table_name} REPLICA IDENTITY FULL;"))
+                table_list.append(full_table_name)
+
+            table_list_str = ", ".join(table_list)
             conn.execute(
                 text(f"""
-                CREATE PUBLICATION {schema_name}_peerdb FOR TABLE {table_list};
+                CREATE PUBLICATION {schema_name}_peerdb FOR TABLE {table_list_str};
                 """)
             )
             conn.commit()
@@ -319,11 +329,11 @@ class PeerDBInterface:
         if not self.docker_host_mapping or not host:
             return host
 
-        if host in ['localhost', '127.0.0.1']:
+        if host in ["localhost", "127.0.0.1"]:
             logger.debug(f"Mapping host {host} to {self.docker_host_mapping} for Docker environment")
             return self.docker_host_mapping
 
-        url_pattern = r'(localhost|127\.0\.0\.1)'
+        url_pattern = r"(localhost|127\.0\.0\.1)"
         match = re.search(url_pattern, host)
         if match:
             original_host = match.group(1)

{datamarket-0.9.15.dist-info → datamarket-0.9.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.15
+Version: 0.9.17
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.15.dist-info → datamarket-0.9.17.dist-info}/RECORD

@@ -1,11 +1,11 @@
 datamarket/__init__.py,sha256=FHS77P9qNewKMoN-p0FLEUEC60oWIYup1QkbJZP4ays,12
 datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamarket/interfaces/alchemy.py,sha256=Y1wKSEBlWXCE3-6JypdaUL2i_7FAGa9PK4_zQHeM2SE,8563
+datamarket/interfaces/alchemy.py,sha256=z7VarlKZ-JfsXWtuDXCYnNp_pSzEYo5IvrD7wqoRpbI,8891
 datamarket/interfaces/aws.py,sha256=7KLUeBxmPN7avEMPsu5HC_KHB1N7W6Anp2X8fo43mlw,2383
 datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIgE,2915
 datamarket/interfaces/ftp.py,sha256=o0KlJxtksbop9OjCiQRzyAa2IeG_ExVXagS6apwrAQo,1881
 datamarket/interfaces/nominatim.py,sha256=ysIA2J1GhsZ0TJxD6B8N1_a7dkMEqtZQV6mT4Hayecg,3672
-datamarket/interfaces/peerdb.py,sha256=2FrG7E5BAGxhwoxjjlPP7ceE7bmUpQi54jtTPtjQul0,22232
+datamarket/interfaces/peerdb.py,sha256=QkX8pyT5pi6g0wT5yoPiBAkCTJB0NutkxNPPxYOVqyY,22566
 datamarket/interfaces/proxy.py,sha256=updoOStKd8-nQBbxWbnD9eOt6HksnYi-5dQ0rEySf5M,3152
 datamarket/interfaces/tinybird.py,sha256=AYrcRGNOCoCt7ojilkWa27POROee9sTCwZ61GGHEPeM,2698
 datamarket/params/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -17,7 +17,7 @@ datamarket/utils/main.py,sha256=O6rX-65h4h0j2zs9dofdTPlly5reKDnvgLtTwbLmbWg,6529
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
-datamarket-0.9.15.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.15.dist-info/METADATA,sha256=LzfDrq6DA-PUfIHU19ZOy0n_x_txIMqXu6PKl-pmmxI,6363
-datamarket-0.9.15.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-datamarket-0.9.15.dist-info/RECORD,,
+datamarket-0.9.17.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.17.dist-info/METADATA,sha256=hNvKYKsU4pTLrpjGQ6HFBn8IBicz1-uvjzuAcKE7yy0,6363
+datamarket-0.9.17.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+datamarket-0.9.17.dist-info/RECORD,,