datamarket 0.9.13__py3-none-any.whl → 0.9.15__py3-none-any.whl

datamarket/interfaces/alchemy.py

@@ -3,27 +3,33 @@
 
 import logging
 from urllib.parse import quote_plus
+from typing import Any, Iterator, List, Type, TypeVar
+from collections.abc import MutableMapping
 
-from sqlalchemy import DDL, create_engine, text
+from sqlalchemy import DDL, Result, Select, SQLColumnExpression, create_engine, text
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import sessionmaker
+from sqlalchemy import FrozenResult
+from sqlalchemy.ext.declarative import DeclarativeMeta
 
 ########################################################################################################################
 # CLASSES
 
 logger = logging.getLogger(__name__)
 
+ModelType = TypeVar("ModelType", bound=DeclarativeMeta)
+
 
 class MockContext:
-    def __init__(self, column):
+    def __init__(self, column: SQLColumnExpression) -> None:
         self.current_parameters = {}
         self.current_column = column
         self.connection = None
 
 
 class AlchemyInterface:
-    def __init__(self, config):
+    def __init__(self, config: MutableMapping) -> None:
         if "db" in config:
             self.config = config["db"]
 
@@ -43,7 +49,7 @@ class AlchemyInterface:
         )
 
     @staticmethod
-    def get_schema_from_table(table):
+    def get_schema_from_table(table: Type[ModelType]) -> str:
         schema = "public"
 
         if isinstance(table.__table_args__, tuple):
@@ -59,7 +65,7 @@ class AlchemyInterface:
 
         return schema
 
-    def create_tables(self, tables):
+    def create_tables(self, tables: List[Type[ModelType]]) -> None:
         for table in tables:
             schema = self.get_schema_from_table(table)
 
@@ -82,7 +88,7 @@ class AlchemyInterface:
                     else:
                         logger.info(f"table {table.__tablename__} already exists")
 
-    def drop_tables(self, tables):
+    def drop_tables(self, tables: List[Type[ModelType]]) -> None:
         for table in tables:
             schema = self.get_schema_from_table(table)
 
@@ -98,13 +104,40 @@ class AlchemyInterface:
                         conn.execute(DDL(f"DROP TABLE {schema}.{table.__tablename__} CASCADE"))
                         conn.commit()
 
-    def reset_db(self, tables, drop):
+    def reset_db(self, tables: List[Type[ModelType]], drop: bool = False) -> None:
         if drop:
             self.drop_tables(tables)
 
         self.create_tables(tables)
 
-    def insert_alchemy_obj(self, alchemy_obj, silent=False):
+    def reset_column(self, query_results: List[Result[Any]], column_name: str) -> None:
+        if not query_results:
+            logger.warning("No objects to reset column for.")
+            return
+
+        first_obj = query_results[0]
+        model_class = first_obj.__class__
+        table = model_class.__table__
+
+        if column_name not in table.columns:
+            logger.warning(f"Column {column_name} does not exist in table {table.name}.")
+            return
+
+        column = table.columns[column_name]
+
+        if column.server_default is not None:
+            query_results.update({column_name: text("DEFAULT")}, synchronize_session=False)
+        elif column.default is not None:
+            default_value = column.default.arg
+            if callable(default_value):
+                default_value = default_value(MockContext(column))
+            query_results.update({column_name: default_value}, synchronize_session=False)
+        else:
+            raise ValueError(f"Column '{column_name}' doesn't have a default value defined.")
+
+        self.session.commit()
+
+    def insert_alchemy_obj(self, alchemy_obj: ModelType, silent: bool = False) -> None:
         try:
             if not silent:
                 logger.info(f"adding {alchemy_obj}...")
@@ -118,7 +151,7 @@ class AlchemyInterface:
 
             self.session.rollback()
 
-    def upsert_alchemy_obj(self, alchemy_obj, index_elements, silent=False):
+    def upsert_alchemy_obj(self, alchemy_obj: ModelType, index_elements: List[str], silent: bool = False) -> None:
         if not silent:
             logger.info(f"upserting {alchemy_obj}")
 
@@ -144,29 +177,52 @@ class AlchemyInterface:
 
             self.session.rollback()
 
-    def reset_column(self, query_results, column_name):
-        if not query_results:
-            logger.warning("No objects to reset column for.")
-            return
+    def windowed_query(
+        self,
+        stmt: Select[Any],
+        column: SQLColumnExpression[Any],
+        windowsize: int,
+    ) -> Iterator[Result[Any]]:
+        """
+        Executes a windowed query on the given statement.
 
-        first_obj = query_results[0]
-        model_class = first_obj.__class__
-        table = model_class.__table__
+        Args:
+            stmt: The SQL select statement to execute.
+            column: The column to use for windowing and sorting.
+            windowsize: The number of rows to fetch in each window.
 
-        if column_name not in table.columns:
-            logger.warning(f"Column {column_name} does not exist in table {table.name}.")
-            return
+        Returns:
+            An iterator of Result objects, each containing a window of data.
 
-        column = table.columns[column_name]
+        Source: https://github.com/sqlalchemy/sqlalchemy/wiki/RangeQuery-and-WindowedRangeQuery
+        """
 
-        if column.server_default is not None:
-            query_results.update({column_name: text("DEFAULT")}, synchronize_session=False)
-        elif column.default is not None:
-            default_value = column.default.arg
-            if callable(default_value):
-                default_value = default_value(MockContext(column))
-            query_results.update({column_name: default_value}, synchronize_session=False)
-        else:
-            raise ValueError(f"Column '{column_name}' doesn't have a default value defined.")
+        # Add the windowing column to the statement and sort by it for deterministic results
+        stmt = stmt.add_columns(column).order_by(column)
+        last_id = None
 
-        self.session.commit()
+        while True:
+            subq = stmt
+
+            if last_id is not None:
+                subq = subq.filter(column > last_id)
+
+            result: Result = self.session.execute(subq.limit(windowsize))
+
+            # Create a FrozenResult to allow peeking at the data without consuming
+            frozen_result: FrozenResult = result.freeze()
+            chunk = frozen_result().all()
+
+            if not chunk:
+                break
+
+            result_width = len(chunk)
+            last_id = chunk[-1][-1]
+
+            # Create a new Result object from the FrozenResult
+            yield_result: Result = frozen_result()
+
+            # Remove the windowing column from the result
+            yield_result = yield_result.columns(*list(range(0, result_width - 1)))
+
+            yield yield_result

{datamarket-0.9.13.dist-info → datamarket-0.9.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.13
+Version: 0.9.15
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket
@@ -85,7 +85,7 @@ Requires-Dist: httpx[http2] (>=0.28.0,<0.29.0) ; extra == "httpx"
 Requires-Dist: jinja2 (>=3.0.0,<4.0.0)
 Requires-Dist: json5 (>=0.10.0,<0.11.0) ; extra == "json5"
 Requires-Dist: lxml[html-clean] (>=5.0.0,<6.0.0) ; extra == "lxml"
-Requires-Dist: nodriver (==0.38.post1) ; extra == "nodriver"
+Requires-Dist: nodriver (>=0.44,<0.45) ; extra == "nodriver"
 Requires-Dist: openpyxl (>=3.0.0,<4.0.0) ; extra == "openpyxl"
 Requires-Dist: pandas (>=2.0.0,<3.0.0) ; extra == "pandas"
 Requires-Dist: pandera (>=0.22.0,<0.23.0) ; extra == "pandera"

{datamarket-0.9.13.dist-info → datamarket-0.9.15.dist-info}/RECORD

@@ -1,6 +1,6 @@
 datamarket/__init__.py,sha256=FHS77P9qNewKMoN-p0FLEUEC60oWIYup1QkbJZP4ays,12
 datamarket/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamarket/interfaces/alchemy.py,sha256=Gkb-55kgHz-07wpfxQfCHP-jCtj7k-WcRAaQWqDaNRs,6377
+datamarket/interfaces/alchemy.py,sha256=Y1wKSEBlWXCE3-6JypdaUL2i_7FAGa9PK4_zQHeM2SE,8563
 datamarket/interfaces/aws.py,sha256=7KLUeBxmPN7avEMPsu5HC_KHB1N7W6Anp2X8fo43mlw,2383
 datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIgE,2915
 datamarket/interfaces/ftp.py,sha256=o0KlJxtksbop9OjCiQRzyAa2IeG_ExVXagS6apwrAQo,1881
@@ -17,7 +17,7 @@ datamarket/utils/main.py,sha256=O6rX-65h4h0j2zs9dofdTPlly5reKDnvgLtTwbLmbWg,6529
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
-datamarket-0.9.13.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.13.dist-info/METADATA,sha256=yecwYlfg1DmNoPRN6TE2rJ3MBRK09LieGMJJ4PUlNIU,6363
-datamarket-0.9.13.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-datamarket-0.9.13.dist-info/RECORD,,
+datamarket-0.9.15.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.15.dist-info/METADATA,sha256=LzfDrq6DA-PUfIHU19ZOy0n_x_txIMqXu6PKl-pmmxI,6363
+datamarket-0.9.15.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+datamarket-0.9.15.dist-info/RECORD,,