datamarket 0.9.14__py3-none-any.whl → 0.9.16__py3-none-any.whl

datamarket/interfaces/alchemy.py

@@ -2,11 +2,14 @@
 # IMPORTS
 
 import logging
+from collections.abc import MutableMapping
+from typing import Any, Iterator, List, Type, TypeVar
 from urllib.parse import quote_plus
 
-from sqlalchemy import DDL, create_engine, text
+from sqlalchemy import DDL, FrozenResult, Result, Select, SQLColumnExpression, create_engine, func, select, text
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.declarative import DeclarativeMeta
 from sqlalchemy.orm import sessionmaker
 
 ########################################################################################################################
@@ -14,16 +17,18 @@ from sqlalchemy.orm import sessionmaker
 
 logger = logging.getLogger(__name__)
 
+ModelType = TypeVar("ModelType", bound=DeclarativeMeta)
+
 
 class MockContext:
-    def __init__(self, column):
+    def __init__(self, column: SQLColumnExpression) -> None:
         self.current_parameters = {}
         self.current_column = column
         self.connection = None
 
 
 class AlchemyInterface:
-    def __init__(self, config):
+    def __init__(self, config: MutableMapping) -> None:
         if "db" in config:
             self.config = config["db"]
 
@@ -43,7 +48,7 @@ class AlchemyInterface:
         )
 
     @staticmethod
-    def get_schema_from_table(table):
+    def get_schema_from_table(table: Type[ModelType]) -> str:
         schema = "public"
 
         if isinstance(table.__table_args__, tuple):
@@ -59,7 +64,7 @@ class AlchemyInterface:
 
         return schema
 
-    def create_tables(self, tables):
+    def create_tables(self, tables: List[Type[ModelType]]) -> None:
         for table in tables:
             schema = self.get_schema_from_table(table)
 
@@ -82,7 +87,7 @@ class AlchemyInterface:
                     else:
                         logger.info(f"table {table.__tablename__} already exists")
 
-    def drop_tables(self, tables):
+    def drop_tables(self, tables: List[Type[ModelType]]) -> None:
         for table in tables:
             schema = self.get_schema_from_table(table)
 
@@ -98,13 +103,40 @@ class AlchemyInterface:
                         conn.execute(DDL(f"DROP TABLE {schema}.{table.__tablename__} CASCADE"))
                         conn.commit()
 
-    def reset_db(self, tables, drop):
+    def reset_db(self, tables: List[Type[ModelType]], drop: bool = False) -> None:
         if drop:
             self.drop_tables(tables)
 
         self.create_tables(tables)
 
-    def insert_alchemy_obj(self, alchemy_obj, silent=False):
+    def reset_column(self, query_results: List[Result[Any]], column_name: str) -> None:
+        if not query_results:
+            logger.warning("No objects to reset column for.")
+            return
+
+        first_obj = query_results[0]
+        model_class = first_obj.__class__
+        table = model_class.__table__
+
+        if column_name not in table.columns:
+            logger.warning(f"Column {column_name} does not exist in table {table.name}.")
+            return
+
+        column = table.columns[column_name]
+
+        if column.server_default is not None:
+            query_results.update({column_name: text("DEFAULT")}, synchronize_session=False)
+        elif column.default is not None:
+            default_value = column.default.arg
+            if callable(default_value):
+                default_value = default_value(MockContext(column))
+            query_results.update({column_name: default_value}, synchronize_session=False)
+        else:
+            raise ValueError(f"Column '{column_name}' doesn't have a default value defined.")
+
+        self.session.commit()
+
+    def insert_alchemy_obj(self, alchemy_obj: ModelType, silent: bool = False) -> None:
         try:
             if not silent:
                 logger.info(f"adding {alchemy_obj}...")
@@ -118,7 +150,7 @@ class AlchemyInterface:
 
             self.session.rollback()
 
-    def upsert_alchemy_obj(self, alchemy_obj, index_elements, silent=False):
+    def upsert_alchemy_obj(self, alchemy_obj: ModelType, index_elements: List[str], silent: bool = False) -> None:
         if not silent:
             logger.info(f"upserting {alchemy_obj}")
 
@@ -144,29 +176,57 @@ class AlchemyInterface:
 
             self.session.rollback()
 
-    def reset_column(self, query_results, column_name):
-        if not query_results:
-            logger.warning("No objects to reset column for.")
-            return
+    def windowed_query(
+        self,
+        stmt: Select[Any],
+        order_by: List[SQLColumnExpression[Any]],
+        windowsize: int,
+    ) -> Iterator[Result[Any]]:
+        """
+        Executes a windowed query on the given statement.
 
-        first_obj = query_results[0]
-        model_class = first_obj.__class__
-        table = model_class.__table__
+        Args:
+            stmt: The SQL select statement to execute.
+            order_by: The columns to use for ordering.
+            windowsize: The number of rows to fetch in each window.
 
-        if column_name not in table.columns:
-            logger.warning(f"Column {column_name} does not exist in table {table.name}.")
-            return
+        Returns:
+            An iterator of Result objects, each containing a window of data.
 
-        column = table.columns[column_name]
+        More info: https://github.com/sqlalchemy/sqlalchemy/wiki/RangeQuery-and-WindowedRangeQuery
+        """
+        # Add row_number over the specified order
+        row_number = func.row_number().over(order_by=order_by).label("row_number")
 
-        if column.server_default is not None:
-            query_results.update({column_name: text("DEFAULT")}, synchronize_session=False)
-        elif column.default is not None:
-            default_value = column.default.arg
-            if callable(default_value):
-                default_value = default_value(MockContext(column))
-            query_results.update({column_name: default_value}, synchronize_session=False)
-        else:
-            raise ValueError(f"Column '{column_name}' doesn't have a default value defined.")
+        # Add the windowing column to the statement and sort by it for deterministic results
+        inner_stmt = stmt.add_columns(row_number).order_by(row_number)
+        subq = inner_stmt.subquery()
 
-        self.session.commit()
+        # Create an outer query that selects from the subquery
+        cols = [c for c in subq.c]
+        outer_query = select(*cols)
+
+        last_row_number = 0
+
+        while True:
+            # Filter on row_number in the outer query
+            current_query = outer_query.where(subq.c.row_number > last_row_number).limit(windowsize)
+            result = self.session.execute(current_query)
+
+            # Create a FrozenResult to allow peeking at the data without consuming
+            frozen_result: FrozenResult = result.freeze()
+            chunk = frozen_result().all()
+
+            if not chunk:
+                break
+
+            # Update for next iteration
+            last_row_number = chunk[-1].row_number - 1
+
+            # Create a new Result object from the FrozenResult
+            yield_result = frozen_result()
+
+            # Remove row_number from result
+            yield_result = yield_result.columns(*range(len(cols) - 1))
+
+            yield yield_result

{datamarket-0.9.14.dist-info → datamarket-0.9.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.14
+Version: 0.9.16
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.14.dist-info → datamarket-0.9.16.dist-info}/RECORD

@@ -1,6 +1,6 @@
 datamarket/__init__.py,sha256=FHS77P9qNewKMoN-p0FLEUEC60oWIYup1QkbJZP4ays,12
 datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamarket/interfaces/alchemy.py,sha256=Gkb-55kgHz-07wpfxQfCHP-jCtj7k-WcRAaQWqDaNRs,6377
+datamarket/interfaces/alchemy.py,sha256=z7VarlKZ-JfsXWtuDXCYnNp_pSzEYo5IvrD7wqoRpbI,8891
 datamarket/interfaces/aws.py,sha256=7KLUeBxmPN7avEMPsu5HC_KHB1N7W6Anp2X8fo43mlw,2383
 datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIgE,2915
 datamarket/interfaces/ftp.py,sha256=o0KlJxtksbop9OjCiQRzyAa2IeG_ExVXagS6apwrAQo,1881
@@ -17,7 +17,7 @@ datamarket/utils/main.py,sha256=O6rX-65h4h0j2zs9dofdTPlly5reKDnvgLtTwbLmbWg,6529
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
-datamarket-0.9.14.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.14.dist-info/METADATA,sha256=NnDtsHllSavKwk68xUBM9slmP8w6ZzvZgGjyg2uOrGQ,6363
-datamarket-0.9.14.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-datamarket-0.9.14.dist-info/RECORD,,
+datamarket-0.9.16.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.16.dist-info/METADATA,sha256=DsHOm4WsenKNFphVyqbLI3A3WluNWGGgVFZRlE8U8NI,6363
+datamarket-0.9.16.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+datamarket-0.9.16.dist-info/RECORD,,