datamarket 0.9.17__py3-none-any.whl → 0.9.19__py3-none-any.whl

datamarket/interfaces/peerdb.py

@@ -84,8 +84,28 @@ class PostgresPeer:
             table_list = []
             for table in table_names:
                 full_table_name = f'"{schema_name}"."{table}"'
-                logger.info(f"Setting REPLICA IDENTITY FULL for table: {full_table_name}")
-                conn.execute(text(f"ALTER TABLE {full_table_name} REPLICA IDENTITY FULL;"))
+
+                # Check current replica identity
+                query = text("""
+                    SELECT CASE c.relreplident
+                             WHEN 'd' THEN 'DEFAULT'
+                             WHEN 'n' THEN 'NOTHING'
+                             WHEN 'f' THEN 'FULL'
+                             WHEN 'i' THEN 'INDEX'
+                           END AS replica_identity
+                    FROM pg_class c
+                    JOIN pg_namespace n ON c.relnamespace = n.oid
+                    WHERE c.relname = :table_name
+                      AND n.nspname = :schema_name;
+                """)
+                result = conn.execute(query, {"table_name": table, "schema_name": schema_name}).scalar_one_or_none()
+
+                if result != "FULL":
+                    logger.info(f"Setting REPLICA IDENTITY FULL for table: {full_table_name}")
+                    conn.execute(text(f"ALTER TABLE {full_table_name} REPLICA IDENTITY FULL;"))
+                else:
+                    logger.info(f"REPLICA IDENTITY for table {full_table_name} is already FULL. Skipping ALTER TABLE.")
+
                 table_list.append(full_table_name)
 
             table_list_str = ", ".join(table_list)
@@ -312,12 +332,17 @@ class PeerDBInterface:
         logger.debug(f"Making API request to PeerDB endpoint: {endpoint}")
         try:
             r = requests.post(url, headers=headers, json=payload, timeout=30)
+            response = r.json()
             r.raise_for_status()
             logger.debug(f"API request to {endpoint} completed successfully")
-            return r.json()
+            return response
         except HTTPError as e:
             logger.error(f"HTTP error occurred: {e}")
             logger.error(f"Response JSON: {r.json() if 'r' in locals() else 'N/A'}")
+
+            if "no rows in result set" in response.get("message", ""):
+                return {"currentFlowState": "STATUS_UNKNOWN"}
+
             raise
 
     def _resolve_host_mapping(self, host):

{datamarket-0.9.17.dist-info → datamarket-0.9.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.17
+Version: 0.9.19
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket
@@ -16,6 +16,7 @@ Provides-Extra: alchemy
 Provides-Extra: aws
 Provides-Extra: azure-storage-blob
 Provides-Extra: boto3
+Provides-Extra: camoufox
 Provides-Extra: chompjs
 Provides-Extra: click
 Provides-Extra: clickhouse-driver
@@ -64,6 +65,7 @@ Requires-Dist: SQLAlchemy (>=2.0.0,<3.0.0) ; extra == "alchemy"
 Requires-Dist: azure-storage-blob (>=12.0.0,<13.0.0) ; extra == "azure-storage-blob"
 Requires-Dist: beautifulsoup4 (>=4.0.0,<5.0.0)
 Requires-Dist: boto3 (>=1.35.0,<1.36.0) ; extra == "boto3" or extra == "aws" or extra == "peerdb"
+Requires-Dist: camoufox[geoip] (>=0.4.11,<0.5.0) ; extra == "camoufox"
 Requires-Dist: chompjs (>=1.0.0,<2.0.0) ; extra == "chompjs"
 Requires-Dist: click (>=8.0.0,<9.0.0) ; extra == "click"
 Requires-Dist: clickhouse-driver (>=0.2.0,<0.3.0) ; extra == "clickhouse-driver" or extra == "peerdb"

{datamarket-0.9.17.dist-info → datamarket-0.9.19.dist-info}/RECORD

@@ -5,7 +5,7 @@ datamarket/interfaces/aws.py,sha256=7KLUeBxmPN7avEMPsu5HC_KHB1N7W6Anp2X8fo43mlw,
 datamarket/interfaces/drive.py,sha256=shbV5jpQVe_KPE-8Idx6Z9te5Zu1SmVfrvSAyd9ZIgE,2915
 datamarket/interfaces/ftp.py,sha256=o0KlJxtksbop9OjCiQRzyAa2IeG_ExVXagS6apwrAQo,1881
 datamarket/interfaces/nominatim.py,sha256=ysIA2J1GhsZ0TJxD6B8N1_a7dkMEqtZQV6mT4Hayecg,3672
-datamarket/interfaces/peerdb.py,sha256=QkX8pyT5pi6g0wT5yoPiBAkCTJB0NutkxNPPxYOVqyY,22566
+datamarket/interfaces/peerdb.py,sha256=cwYwvO740GyaPo9zLAwJsf3UeJDGDiYzjQVM9Q6s-_g,23652
 datamarket/interfaces/proxy.py,sha256=updoOStKd8-nQBbxWbnD9eOt6HksnYi-5dQ0rEySf5M,3152
 datamarket/interfaces/tinybird.py,sha256=AYrcRGNOCoCt7ojilkWa27POROee9sTCwZ61GGHEPeM,2698
 datamarket/params/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -17,7 +17,7 @@ datamarket/utils/main.py,sha256=O6rX-65h4h0j2zs9dofdTPlly5reKDnvgLtTwbLmbWg,6529
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
-datamarket-0.9.17.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.17.dist-info/METADATA,sha256=hNvKYKsU4pTLrpjGQ6HFBn8IBicz1-uvjzuAcKE7yy0,6363
-datamarket-0.9.17.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-datamarket-0.9.17.dist-info/RECORD,,
+datamarket-0.9.19.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.19.dist-info/METADATA,sha256=yMqp6i6mEyKhwB6VOxXF5Gg1sldJSbkuB9HH2w3NdPQ,6459
+datamarket-0.9.19.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+datamarket-0.9.19.dist-info/RECORD,,