datamarket 0.9.17__tar.gz → 0.9.19__tar.gz

{datamarket-0.9.17 → datamarket-0.9.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.17
+Version: 0.9.19
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket
@@ -16,6 +16,7 @@ Provides-Extra: alchemy
 Provides-Extra: aws
 Provides-Extra: azure-storage-blob
 Provides-Extra: boto3
+Provides-Extra: camoufox
 Provides-Extra: chompjs
 Provides-Extra: click
 Provides-Extra: clickhouse-driver
@@ -64,6 +65,7 @@ Requires-Dist: SQLAlchemy (>=2.0.0,<3.0.0) ; extra == "alchemy"
 Requires-Dist: azure-storage-blob (>=12.0.0,<13.0.0) ; extra == "azure-storage-blob"
 Requires-Dist: beautifulsoup4 (>=4.0.0,<5.0.0)
 Requires-Dist: boto3 (>=1.35.0,<1.36.0) ; extra == "boto3" or extra == "aws" or extra == "peerdb"
+Requires-Dist: camoufox[geoip] (>=0.4.11,<0.5.0) ; extra == "camoufox"
 Requires-Dist: chompjs (>=1.0.0,<2.0.0) ; extra == "chompjs"
 Requires-Dist: click (>=8.0.0,<9.0.0) ; extra == "click"
 Requires-Dist: clickhouse-driver (>=0.2.0,<0.3.0) ; extra == "clickhouse-driver" or extra == "peerdb"

{datamarket-0.9.17 → datamarket-0.9.19}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.17"
+version = "0.9.19"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"
@@ -70,6 +70,7 @@ dnspython = { version = "^2.0.0", optional = true }
 openpyxl = { version = "^3.0.0", optional = true }
 httpx = { extras = ["http2"], version = "~0.28.0", optional = true }
 SQLAlchemy = { version = "^2.0.0", optional = true }
+camoufox = { extras = ["geoip"], version = "~0.4.11", optional = true }
 
 [tool.poetry.extras]
 boto3 = ["boto3"]
@@ -115,6 +116,7 @@ google-auth-oauthlib = ["google-auth-oauthlib"]
 dnspython = ["dnspython"]
 openpyxl = ["openpyxl"]
 httpx = ["httpx"]
+camoufox = ["camoufox"]
 
 # Interface groups
 aws = ["boto3"]

{datamarket-0.9.17 → datamarket-0.9.19}/src/datamarket/interfaces/peerdb.py

@@ -84,8 +84,28 @@ class PostgresPeer:
             table_list = []
             for table in table_names:
                 full_table_name = f'"{schema_name}"."{table}"'
-                logger.info(f"Setting REPLICA IDENTITY FULL for table: {full_table_name}")
-                conn.execute(text(f"ALTER TABLE {full_table_name} REPLICA IDENTITY FULL;"))
+
+                # Check current replica identity
+                query = text("""
+                    SELECT CASE c.relreplident
+                             WHEN 'd' THEN 'DEFAULT'
+                             WHEN 'n' THEN 'NOTHING'
+                             WHEN 'f' THEN 'FULL'
+                             WHEN 'i' THEN 'INDEX'
+                           END AS replica_identity
+                    FROM pg_class c
+                    JOIN pg_namespace n ON c.relnamespace = n.oid
+                    WHERE c.relname = :table_name
+                      AND n.nspname = :schema_name;
+                """)
+                result = conn.execute(query, {"table_name": table, "schema_name": schema_name}).scalar_one_or_none()
+
+                if result != "FULL":
+                    logger.info(f"Setting REPLICA IDENTITY FULL for table: {full_table_name}")
+                    conn.execute(text(f"ALTER TABLE {full_table_name} REPLICA IDENTITY FULL;"))
+                else:
+                    logger.info(f"REPLICA IDENTITY for table {full_table_name} is already FULL. Skipping ALTER TABLE.")
+
                 table_list.append(full_table_name)
 
             table_list_str = ", ".join(table_list)
@@ -312,12 +332,17 @@ class PeerDBInterface:
         logger.debug(f"Making API request to PeerDB endpoint: {endpoint}")
         try:
             r = requests.post(url, headers=headers, json=payload, timeout=30)
+            response = r.json()
             r.raise_for_status()
             logger.debug(f"API request to {endpoint} completed successfully")
-            return r.json()
+            return response
         except HTTPError as e:
             logger.error(f"HTTP error occurred: {e}")
             logger.error(f"Response JSON: {r.json() if 'r' in locals() else 'N/A'}")
+
+            if "no rows in result set" in response.get("message", ""):
+                return {"currentFlowState": "STATUS_UNKNOWN"}
+
             raise
 
     def _resolve_host_mapping(self, host):