udata-hydra 2.0.0.dev3146__tar.gz → 2.0.0.dev3158__tar.gz

{udata_hydra-2.0.0.dev3146 → udata_hydra-2.0.0.dev3158}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata-hydra
-Version: 2.0.0.dev3146
+Version: 2.0.0.dev3158
 Summary: Async crawler and parsing service for data.gouv.fr
 License: MIT
 Author: Opendata Team

{udata_hydra-2.0.0.dev3146 → udata_hydra-2.0.0.dev3158}/pyproject.toml

@@ -9,7 +9,7 @@ readme = "README.md"
 
 [tool.poetry]
 name = "udata-hydra"
-version = "2.0.0.dev3146"
+version = "2.0.0.dev3158"
 description = "Async crawler and parsing service for data.gouv.fr"
 authors = ["Opendata Team <opendatateam@data.gouv.fr>"]
 license = "MIT"

{udata_hydra-2.0.0.dev3146 → udata_hydra-2.0.0.dev3158}/udata_hydra/cli.py

@@ -144,9 +144,7 @@ async def check_resource(resource_id: str, method: str = "get"):
         log.error("Resource not found in catalog")
         return
     async with aiohttp.ClientSession(timeout=None) as session:
-        await crawl_check_url(
-            url=res[0], resource_id=None, session=session, method=method, worker_priority="high"
-        )
+        await crawl_check_url(url=res[0], resource_id=None, session=session, method=method)
 
 
 @cli(name="analyse-csv")

{udata_hydra-2.0.0.dev3146 → udata_hydra-2.0.0.dev3158}/udata_hydra/crawl.py

@@ -18,12 +18,10 @@ from udata_hydra.utils import queue, send
 
 results: defaultdict = defaultdict(int)
 
-RESOURCE_RESPONSE_STATUSES = {
-    "OK": "ok",
-    "TIMEOUT": "timeout",
-    "ERROR": "error",
-    "BACKOFF": "backoff",
-}
+STATUS_OK = "ok"
+STATUS_TIMEOUT = "timeout"
+STATUS_ERROR = "error"
+STATUS_BACKOFF = "backoff"
 
 log = setup_logging()
 
@@ -237,12 +235,7 @@ def has_nice_head(resp) -> bool:
 
 
 async def check_url(
-    url: str,
-    resource_id: str,
-    session,
-    sleep: float = 0,
-    method: str = "head",
-    worker_priority: str = "default",
+    url: str, resource_id: str, session, sleep: float = 0, method: str = "head"
 ) -> str:
     log.debug(f"check {url}, sleep {sleep}, method {method}")
 
@@ -261,14 +254,14 @@ async def check_url(
                 "timeout": False,
             }
         )
-        return RESOURCE_RESPONSE_STATUSES["ERROR"]
+        return STATUS_ERROR
 
     should_backoff, reason = await is_backoff(domain)
     if should_backoff:
         log.info(f"backoff {domain} ({reason})")
         # skip this URL, it will come back in a next batch
         await Resource.update(resource_id=resource_id, data={"status": None, "priority": False})
-        return RESOURCE_RESPONSE_STATUSES["BACKOFF"]
+        return STATUS_BACKOFF
 
     try:
         start = time.time()
@@ -277,9 +270,7 @@ async def check_url(
         async with _method(url, timeout=timeout, allow_redirects=True) as resp:
             end = time.time()
             if method != "get" and not has_nice_head(resp):
-                return await check_url(
-                    url, resource_id, session, method="get", worker_priority=worker_priority
-                )
+                return await check_url(url, resource_id, session, method="get")
             resp.raise_for_status()
 
             check_id, is_first_check = await process_check_data(
@@ -291,12 +282,13 @@ async def check_url(
                     "headers": convert_headers(resp.headers),
                     "timeout": False,
                     "response_time": end - start,
-                },
+                }
             )
 
-            queue.enqueue(process_resource, check_id, is_first_check, _priority=worker_priority)
+            # Analyse the resource
+            queue.enqueue(process_resource, check_id, is_first_check, _priority="low")
 
-            return RESOURCE_RESPONSE_STATUSES["OK"]
+            return STATUS_OK
     except asyncio.exceptions.TimeoutError:
         await process_check_data(
             {
@@ -306,7 +298,7 @@ async def check_url(
                 "timeout": True,
             }
         )
-        return RESOURCE_RESPONSE_STATUSES["TIMEOUT"]
+        return STATUS_TIMEOUT
     # TODO: debug AssertionError, should be caught in DB now
     # File "[...]aiohttp/connector.py", line 991, in _create_direct_connection
     # assert port is not None
@@ -330,7 +322,7 @@ async def check_url(
             }
         )
         log.warning(f"Crawling error for url {url}", exc_info=e)
-        return RESOURCE_RESPONSE_STATUSES["ERROR"]
+        return STATUS_ERROR
 
 
 async def crawl_urls(to_parse: list[str]) -> None:
@@ -340,14 +332,7 @@ async def crawl_urls(to_parse: list[str]) -> None:
         timeout=None, headers={"user-agent": config.USER_AGENT}
     ) as session:
         for row in to_parse:
-            tasks.append(
-                check_url(
-                    url=row["url"],
-                    resource_id=row["resource_id"],
-                    session=session,
-                    worker_priority="low",
-                )
-            )
+            tasks.append(check_url(url=row["url"], resource_id=row["resource_id"], session=session))
         for task in asyncio.as_completed(tasks):
             result = await task
             results[result] += 1

{udata_hydra-2.0.0.dev3146 → udata_hydra-2.0.0.dev3158}/udata_hydra/db/check.py

@@ -62,12 +62,12 @@ class Check:
         This use the info from the last check of the same resource
         """
         data = convert_dict_values_to_json(data)
-        q1: str = compute_insert_query(table_name="checks", data=data)
+        q = compute_insert_query(table_name="checks", data=data)
         pool = await context.pool()
         async with pool.acquire() as connection:
-            last_check = await connection.fetchrow(q1, *data.values())
-            q2 = """UPDATE catalog SET last_check = $1 WHERE resource_id = $2"""
-            await connection.execute(q2, last_check["id"], data["resource_id"])
+            last_check = await connection.fetchrow(q, *data.values())
+            q = """UPDATE catalog SET last_check = $1 WHERE resource_id = $2"""
+            await connection.execute(q, last_check["id"], data["resource_id"])
         return last_check["id"]
 
     @classmethod

{udata_hydra-2.0.0.dev3146 → udata_hydra-2.0.0.dev3158}/udata_hydra/routes/__init__.py

@@ -1,6 +1,6 @@
 from aiohttp import web
 
-from udata_hydra.routes.checks import create_check, get_all_checks, get_latest_check
+from udata_hydra.routes.checks import get_all_checks, get_latest_check
 from udata_hydra.routes.resources import (
     create_resource,
     delete_resource,
@@ -14,7 +14,6 @@ routes: list = [
     # Routes for checks
     web.get("/api/checks/latest/", get_latest_check),
     web.get("/api/checks/all/", get_all_checks),
-    web.post("/api/checks/", create_check),
     # Routes for resources
     web.get("/api/resources/", get_resource),
     web.post("/api/resources/", create_resource),

udata_hydra-2.0.0.dev3158/udata_hydra/routes/checks.py

@@ -0,0 +1,26 @@
+from typing import Optional
+
+from aiohttp import web
+
+from udata_hydra.db.check import Check
+from udata_hydra.schemas import CheckSchema
+from udata_hydra.utils import get_request_params
+
+
+async def get_latest_check(request: web.Request) -> web.Response:
+    """Get the latest check for a given URL or resource_id"""
+    url, resource_id = get_request_params(request, params_names=["url", "resource_id"])
+    data: Optional[dict] = await Check.get_latest(url, resource_id)
+    if not data:
+        raise web.HTTPNotFound()
+    if data["deleted"]:
+        raise web.HTTPGone()
+    return web.json_response(CheckSchema().dump(dict(data)))
+
+
+async def get_all_checks(request: web.Request) -> web.Response:
+    url, resource_id = get_request_params(request, params_names=["url", "resource_id"])
+    data: Optional[list] = await Check.get_all(url, resource_id)
+    if not data:
+        raise web.HTTPNotFound()
+    return web.json_response([CheckSchema().dump(dict(r)) for r in data])

udata_hydra-2.0.0.dev3146/udata_hydra/routes/checks.py

@@ -1,68 +0,0 @@
-import json
-from typing import Optional
-
-import aiohttp
-from aiohttp import web
-from marshmallow import ValidationError
-
-from udata_hydra import config, context
-from udata_hydra.crawl import RESOURCE_RESPONSE_STATUSES, check_url
-from udata_hydra.db.check import Check
-from udata_hydra.db.resource import Resource
-from udata_hydra.schemas import CheckSchema
-from udata_hydra.utils import get_request_params
-
-
-async def get_latest_check(request: web.Request) -> web.Response:
-    """Get the latest check for a given URL or resource_id"""
-    url, resource_id = get_request_params(request, params_names=["url", "resource_id"])
-    data: Optional[dict] = await Check.get_latest(url, resource_id)
-    if not data:
-        raise web.HTTPNotFound()
-    if data["deleted"]:
-        raise web.HTTPGone()
-    return web.json_response(CheckSchema().dump(dict(data)))
-
-
-async def get_all_checks(request: web.Request) -> web.Response:
-    url, resource_id = get_request_params(request, params_names=["url", "resource_id"])
-    data: Optional[list] = await Check.get_all(url, resource_id)
-    if not data:
-        raise web.HTTPNotFound()
-    return web.json_response([CheckSchema().dump(dict(r)) for r in data])
-
-
-async def create_check(request: web.Request) -> web.Response:
-    """Create a new check"""
-
-    # Get resource_id from request
-    try:
-        payload: dict = await request.json()
-        resource_id: str = payload["resource_id"]
-    except ValidationError as err:
-        raise web.HTTPBadRequest(text=json.dumps(err.messages))
-    except KeyError as e:
-        raise web.HTTPBadRequest(text=f"Missing key: {e}")
-
-    # Get URL from resource_id
-    try:
-        resource: dict = await Resource.get(resource_id, "url")
-        url: str = resource["url"]
-    except Exception:
-        raise web.HTTPNotFound(text=f"Couldn't find URL for resource {resource_id}")
-
-    context.monitor().set_status(f'Crawling url "{url}"...')
-
-    async with aiohttp.ClientSession(
-        timeout=None, headers={"user-agent": config.USER_AGENT}
-    ) as session:
-        status: str = await check_url(
-            url=url, resource_id=resource_id, session=session, worker_priority="high"
-        )
-        context.monitor().refresh(status)
-
-    check: Optional[dict] = await Check.get_latest(url, resource_id)
-    if not check:
-        raise web.HTTPBadRequest(text=f"Check not created, status: {status}")
-
-    return web.json_response(CheckSchema().dump(dict(check)))