udata-hydra 2.0.5.dev5485__tar.gz → 2.0.5.dev5527__tar.gz

{udata_hydra-2.0.5.dev5485 → udata_hydra-2.0.5.dev5527}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata-hydra
-Version: 2.0.5.dev5485
+Version: 2.0.5.dev5527
 Summary: Async crawler and parsing service for data.gouv.fr
 License: MIT
 Author: Opendata Team

{udata_hydra-2.0.5.dev5485 → udata_hydra-2.0.5.dev5527}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "udata-hydra"
-version = "2.0.5.dev5485"
+version = "2.0.5.dev5527"
 description = "Async crawler and parsing service for data.gouv.fr"
 authors = ["Opendata Team <opendatateam@data.gouv.fr>"]
 license = "MIT"

{udata_hydra-2.0.5.dev5485 → udata_hydra-2.0.5.dev5527}/udata_hydra/analysis/resource.py

@@ -31,7 +31,9 @@ class Change(Enum):
 log = logging.getLogger("udata-hydra")
 
 
-async def analyse_resource(check_id: int, is_first_check: bool) -> None:
+async def analyse_resource(
+    check_id: int, is_first_check: bool, force_analysis: bool = False
+) -> None:
     """
     Perform analysis on the resource designated by check_id:
     - change analysis
@@ -70,7 +72,7 @@ async def analyse_resource(check_id: int, is_first_check: bool) -> None:
     # if the change status is NO_GUESS or HAS_CHANGED, let's download the file to get more infos
     dl_analysis = {}
     tmp_file = None
-    if change_status != Change.HAS_NOT_CHANGED:
+    if change_status != Change.HAS_NOT_CHANGED or force_analysis:
         try:
             tmp_file = await download_resource(url, headers, max_size_allowed)
         except IOError:
@@ -107,7 +109,7 @@ async def analyse_resource(check_id: int, is_first_check: bool) -> None:
 
     analysis_results = {**dl_analysis, **(change_payload or {})}
 
-    if change_status == Change.HAS_CHANGED or is_first_check:
+    if change_status == Change.HAS_CHANGED or is_first_check or force_analysis:
         if is_tabular and tmp_file:
             # Change status to TO_ANALYSE_CSV
             await Resource.update(resource_id, data={"status": "TO_ANALYSE_CSV"})
@@ -260,7 +262,7 @@ async def detect_resource_change_on_early_hints(
 
 
 async def detect_resource_change_from_harvest(
-    checks_data: list, resource_id: str
+    checks_data: tuple, resource_id: str
 ) -> tuple[Change, dict | None]:
     """
     Checks if resource has a harvest.modified_at

{udata_hydra-2.0.5.dev5485 → udata_hydra-2.0.5.dev5527}/udata_hydra/cli.py

@@ -137,7 +137,7 @@ async def crawl_url(url: str, method: str = "get"):
 
 
 @cli
-async def check_resource(resource_id: str, method: str = "get"):
+async def check_resource(resource_id: str, method: str = "get", force_analysis: bool = True):
     """Trigger a complete check for a given resource_id"""
     resource: asyncpg.Record | None = await Resource.get(resource_id)
     if not resource:
@@ -149,6 +149,7 @@ async def check_resource(resource_id: str, method: str = "get"):
             resource_id=resource_id,
             session=session,
             method=method,
+            force_analysis=force_analysis,
             worker_priority="high",
         )
 

{udata_hydra-2.0.5.dev5485 → udata_hydra-2.0.5.dev5527}/udata_hydra/crawl/check_resources.py

@@ -61,6 +61,7 @@ async def check_resource(
     sleep: float = 0,
     method: str = "head",
     worker_priority: str = "default",
+    force_analysis: bool = False,
 ) -> str:
     log.debug(f"check {url}, sleep {sleep}, method {method}")
 
@@ -101,7 +102,12 @@ async def check_resource(
             end = time.time()
             if method != "get" and not has_nice_head(resp):
                 return await check_resource(
-                    url, resource_id, session, method="get", worker_priority=worker_priority
+                    url,
+                    resource_id,
+                    session,
+                    force_analysis=force_analysis,
+                    method="get",
+                    worker_priority=worker_priority,
                 )
             resp.raise_for_status()
 
@@ -122,7 +128,13 @@ async def check_resource(
             await Resource.update(resource_id, data={"status": "TO_ANALYSE_RESOURCE"})
 
             # Enqueue the resource for analysis
-            queue.enqueue(analyse_resource, check["id"], is_first_check, _priority=worker_priority)
+            queue.enqueue(
+                analyse_resource,
+                check["id"],
+                is_first_check,
+                force_analysis,
+                _priority=worker_priority,
+            )
 
             return RESOURCE_RESPONSE_STATUSES["OK"]
 

{udata_hydra-2.0.5.dev5485 → udata_hydra-2.0.5.dev5527}/udata_hydra/routes/checks.py

@@ -64,6 +64,7 @@ async def create_check(request: web.Request) -> web.Response:
     try:
         payload: dict = await request.json()
         resource_id: str = payload["resource_id"]
+        force_analysis: bool = payload.get("force_analysis", True)
     except Exception as err:
         raise web.HTTPBadRequest(text=json.dumps({"error": str(err)}))
 
@@ -80,7 +81,11 @@ async def create_check(request: web.Request) -> web.Response:
         timeout=None, headers={"user-agent": config.USER_AGENT}
     ) as session:
         status: str = await check_resource(
-            url=url, resource_id=resource_id, session=session, worker_priority="high"
+            url=url,
+            resource_id=resource_id,
+            force_analysis=force_analysis,
+            session=session,
+            worker_priority="high",
         )
         context.monitor().refresh(status)