datamarket 0.9.16__tar.gz → 0.9.18__tar.gz

{datamarket-0.9.16 → datamarket-0.9.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.16
+Version: 0.9.18
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.16 → datamarket-0.9.18}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datamarket"
-version = "0.9.16"
+version = "0.9.18"
 description = "Utilities that integrate advanced scraping knowledge into just one library."
 authors = ["DataMarket <techsupport@datamarket.es>"]
 license = "GPL-3.0-or-later"

{datamarket-0.9.16 → datamarket-0.9.18}/src/datamarket/interfaces/peerdb.py

@@ -18,10 +18,13 @@ from .alchemy import AlchemyInterface
 ########################################################################################################################
 # EXCEPTIONS
 
+
 class DatabaseNotConnectedError(Exception):
     """Custom error for when database is not connected."""
+
     pass
 
+
 ########################################################################################################################
 # CLASSES
 
@@ -78,10 +81,37 @@ class PostgresPeer:
         with self.engine.connect() as conn:
             conn.execute(text(f"DROP PUBLICATION IF EXISTS {schema_name}_peerdb"))
 
-            table_list = ", ".join([f"{schema_name}.{table}" for table in table_names])
+            table_list = []
+            for table in table_names:
+                full_table_name = f'"{schema_name}"."{table}"'
+
+                # Check current replica identity
+                query = text("""
+                    SELECT CASE c.relreplident
+                             WHEN 'd' THEN 'DEFAULT'
+                             WHEN 'n' THEN 'NOTHING'
+                             WHEN 'f' THEN 'FULL'
+                             WHEN 'i' THEN 'INDEX'
+                           END AS replica_identity
+                    FROM pg_class c
+                    JOIN pg_namespace n ON c.relnamespace = n.oid
+                    WHERE c.relname = :table_name
+                      AND n.nspname = :schema_name;
+                """)
+                result = conn.execute(query, {"table_name": table, "schema_name": schema_name}).scalar_one_or_none()
+
+                if result != "FULL":
+                    logger.info(f"Setting REPLICA IDENTITY FULL for table: {full_table_name}")
+                    conn.execute(text(f"ALTER TABLE {full_table_name} REPLICA IDENTITY FULL;"))
+                else:
+                    logger.info(f"REPLICA IDENTITY for table {full_table_name} is already FULL. Skipping ALTER TABLE.")
+
+                table_list.append(full_table_name)
+
+            table_list_str = ", ".join(table_list)
             conn.execute(
                 text(f"""
-                CREATE PUBLICATION {schema_name}_peerdb FOR TABLE {table_list};
+                CREATE PUBLICATION {schema_name}_peerdb FOR TABLE {table_list_str};
                 """)
             )
             conn.commit()
@@ -302,12 +332,17 @@ class PeerDBInterface:
         logger.debug(f"Making API request to PeerDB endpoint: {endpoint}")
         try:
             r = requests.post(url, headers=headers, json=payload, timeout=30)
+            response = r.json()
             r.raise_for_status()
             logger.debug(f"API request to {endpoint} completed successfully")
-            return r.json()
+            return response
         except HTTPError as e:
             logger.error(f"HTTP error occurred: {e}")
             logger.error(f"Response JSON: {r.json() if 'r' in locals() else 'N/A'}")
+
+            if "no rows in result set" in response.get("message", ""):
+                return {"currentFlowState": "STATUS_UNKNOWN"}
+
             raise
 
     def _resolve_host_mapping(self, host):
@@ -319,11 +354,11 @@ class PeerDBInterface:
         if not self.docker_host_mapping or not host:
             return host
 
-        if host in ['localhost', '127.0.0.1']:
+        if host in ["localhost", "127.0.0.1"]:
             logger.debug(f"Mapping host {host} to {self.docker_host_mapping} for Docker environment")
             return self.docker_host_mapping
 
-        url_pattern = r'(localhost|127\.0\.0\.1)'
+        url_pattern = r"(localhost|127\.0\.0\.1)"
         match = re.search(url_pattern, host)
         if match:
             original_host = match.group(1)