udata-hydra 2.0.0.dev3084__tar.gz → 2.0.0.dev3146__tar.gz

{udata_hydra-2.0.0.dev3084 → udata_hydra-2.0.0.dev3146}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata-hydra
-Version: 2.0.0.dev3084
+Version: 2.0.0.dev3146
 Summary: Async crawler and parsing service for data.gouv.fr
 License: MIT
 Author: Opendata Team
@@ -124,6 +124,22 @@ You can also override the coverage report configuration when running the tests b
 
 ## API
 
+The API will need a Bearer token for each request on protected endpoints (any endpoint that isn't a `GET`).
+The token is configured in the `config.toml` file as `API_KEY`, and has a default value set in the `udata_hydra/config_default.toml` file.
+
+If you're using hydra as an external service to receive resource events from [udata](https://github.com/opendatateam/udata), then udata needs to also configure this
+API key in its `udata.cfg` file:
+
+```
+# Wether udata should publish the resource events
+PUBLISH_ON_RESOURCE_EVENTS = True
+# Where to publish the events
+RESOURCES_ANALYSER_URI = "http://localhost:8000"
+# The API key that hydra needs
+RESOURCES_ANALYSER_API_KEY = "api_key_to_change"
+
+```
+
 ### Run
 
 ```bash

{udata_hydra-2.0.0.dev3084 → udata_hydra-2.0.0.dev3146}/README.md

@@ -87,6 +87,22 @@ You can also override the coverage report configuration when running the tests b
 
 ## API
 
+The API will need a Bearer token for each request on protected endpoints (any endpoint that isn't a `GET`).
+The token is configured in the `config.toml` file as `API_KEY`, and has a default value set in the `udata_hydra/config_default.toml` file.
+
+If you're using hydra as an external service to receive resource events from [udata](https://github.com/opendatateam/udata), then udata needs to also configure this
+API key in its `udata.cfg` file:
+
+```
+# Wether udata should publish the resource events
+PUBLISH_ON_RESOURCE_EVENTS = True
+# Where to publish the events
+RESOURCES_ANALYSER_URI = "http://localhost:8000"
+# The API key that hydra needs
+RESOURCES_ANALYSER_API_KEY = "api_key_to_change"
+
+```
+
 ### Run
 
 ```bash

{udata_hydra-2.0.0.dev3084 → udata_hydra-2.0.0.dev3146}/pyproject.toml

@@ -9,7 +9,7 @@ readme = "README.md"
 
 [tool.poetry]
 name = "udata-hydra"
-version = "2.0.0.dev3084"
+version = "2.0.0.dev3146"
 description = "Async crawler and parsing service for data.gouv.fr"
 authors = ["Opendata Team <opendatateam@data.gouv.fr>"]
 license = "MIT"

{udata_hydra-2.0.0.dev3084 → udata_hydra-2.0.0.dev3146}/udata_hydra/cli.py

@@ -144,7 +144,9 @@ async def check_resource(resource_id: str, method: str = "get"):
         log.error("Resource not found in catalog")
         return
     async with aiohttp.ClientSession(timeout=None) as session:
-        await crawl_check_url(url=res[0], resource_id=None, session=session, method=method)
+        await crawl_check_url(
+            url=res[0], resource_id=None, session=session, method=method, worker_priority="high"
+        )
 
 
 @cli(name="analyse-csv")

{udata_hydra-2.0.0.dev3084 → udata_hydra-2.0.0.dev3146}/udata_hydra/crawl.py

@@ -18,10 +18,12 @@ from udata_hydra.utils import queue, send
 
 results: defaultdict = defaultdict(int)
 
-STATUS_OK = "ok"
-STATUS_TIMEOUT = "timeout"
-STATUS_ERROR = "error"
-STATUS_BACKOFF = "backoff"
+RESOURCE_RESPONSE_STATUSES = {
+    "OK": "ok",
+    "TIMEOUT": "timeout",
+    "ERROR": "error",
+    "BACKOFF": "backoff",
+}
 
 log = setup_logging()
 
@@ -235,7 +237,12 @@ def has_nice_head(resp) -> bool:
 
 
 async def check_url(
-    url: str, resource_id: str, session, sleep: float = 0, method: str = "head"
+    url: str,
+    resource_id: str,
+    session,
+    sleep: float = 0,
+    method: str = "head",
+    worker_priority: str = "default",
 ) -> str:
     log.debug(f"check {url}, sleep {sleep}, method {method}")
 
@@ -254,14 +261,14 @@ async def check_url(
                 "timeout": False,
             }
         )
-        return STATUS_ERROR
+        return RESOURCE_RESPONSE_STATUSES["ERROR"]
 
     should_backoff, reason = await is_backoff(domain)
     if should_backoff:
         log.info(f"backoff {domain} ({reason})")
         # skip this URL, it will come back in a next batch
         await Resource.update(resource_id=resource_id, data={"status": None, "priority": False})
-        return STATUS_BACKOFF
+        return RESOURCE_RESPONSE_STATUSES["BACKOFF"]
 
     try:
         start = time.time()
@@ -270,7 +277,9 @@ async def check_url(
         async with _method(url, timeout=timeout, allow_redirects=True) as resp:
             end = time.time()
             if method != "get" and not has_nice_head(resp):
-                return await check_url(url, resource_id, session, method="get")
+                return await check_url(
+                    url, resource_id, session, method="get", worker_priority=worker_priority
+                )
             resp.raise_for_status()
 
             check_id, is_first_check = await process_check_data(
@@ -282,13 +291,12 @@ async def check_url(
                     "headers": convert_headers(resp.headers),
                     "timeout": False,
                     "response_time": end - start,
-                }
+                },
             )
 
-            # Analyse the resource
-            queue.enqueue(process_resource, check_id, is_first_check, _priority="low")
+            queue.enqueue(process_resource, check_id, is_first_check, _priority=worker_priority)
 
-            return STATUS_OK
+            return RESOURCE_RESPONSE_STATUSES["OK"]
     except asyncio.exceptions.TimeoutError:
         await process_check_data(
             {
@@ -298,7 +306,7 @@ async def check_url(
                 "timeout": True,
             }
         )
-        return STATUS_TIMEOUT
+        return RESOURCE_RESPONSE_STATUSES["TIMEOUT"]
     # TODO: debug AssertionError, should be caught in DB now
     # File "[...]aiohttp/connector.py", line 991, in _create_direct_connection
     # assert port is not None
@@ -322,7 +330,7 @@ async def check_url(
             }
         )
         log.warning(f"Crawling error for url {url}", exc_info=e)
-        return STATUS_ERROR
+        return RESOURCE_RESPONSE_STATUSES["ERROR"]
 
 
 async def crawl_urls(to_parse: list[str]) -> None:
@@ -332,7 +340,14 @@ async def crawl_urls(to_parse: list[str]) -> None:
         timeout=None, headers={"user-agent": config.USER_AGENT}
     ) as session:
         for row in to_parse:
-            tasks.append(check_url(url=row["url"], resource_id=row["resource_id"], session=session))
+            tasks.append(
+                check_url(
+                    url=row["url"],
+                    resource_id=row["resource_id"],
+                    session=session,
+                    worker_priority="low",
+                )
+            )
         for task in asyncio.as_completed(tasks):
             result = await task
             results[result] += 1

{udata_hydra-2.0.0.dev3084 → udata_hydra-2.0.0.dev3146}/udata_hydra/db/check.py

@@ -62,12 +62,12 @@ class Check:
         This use the info from the last check of the same resource
         """
         data = convert_dict_values_to_json(data)
-        q = compute_insert_query(table_name="checks", data=data)
+        q1: str = compute_insert_query(table_name="checks", data=data)
         pool = await context.pool()
         async with pool.acquire() as connection:
-            last_check = await connection.fetchrow(q, *data.values())
-            q = """UPDATE catalog SET last_check = $1 WHERE resource_id = $2"""
-            await connection.execute(q, last_check["id"], data["resource_id"])
+            last_check = await connection.fetchrow(q1, *data.values())
+            q2 = """UPDATE catalog SET last_check = $1 WHERE resource_id = $2"""
+            await connection.execute(q2, last_check["id"], data["resource_id"])
         return last_check["id"]
 
     @classmethod

{udata_hydra-2.0.0.dev3084 → udata_hydra-2.0.0.dev3146}/udata_hydra/routes/__init__.py

@@ -1,6 +1,6 @@
 from aiohttp import web
 
-from udata_hydra.routes.checks import get_all_checks, get_latest_check
+from udata_hydra.routes.checks import create_check, get_all_checks, get_latest_check
 from udata_hydra.routes.resources import (
     create_resource,
     delete_resource,
@@ -14,6 +14,7 @@ routes: list = [
     # Routes for checks
     web.get("/api/checks/latest/", get_latest_check),
     web.get("/api/checks/all/", get_all_checks),
+    web.post("/api/checks/", create_check),
     # Routes for resources
     web.get("/api/resources/", get_resource),
     web.post("/api/resources/", create_resource),

udata_hydra-2.0.0.dev3146/udata_hydra/routes/checks.py

@@ -0,0 +1,68 @@
+import json
+from typing import Optional
+
+import aiohttp
+from aiohttp import web
+from marshmallow import ValidationError
+
+from udata_hydra import config, context
+from udata_hydra.crawl import RESOURCE_RESPONSE_STATUSES, check_url
+from udata_hydra.db.check import Check
+from udata_hydra.db.resource import Resource
+from udata_hydra.schemas import CheckSchema
+from udata_hydra.utils import get_request_params
+
+
+async def get_latest_check(request: web.Request) -> web.Response:
+    """Get the latest check for a given URL or resource_id"""
+    url, resource_id = get_request_params(request, params_names=["url", "resource_id"])
+    data: Optional[dict] = await Check.get_latest(url, resource_id)
+    if not data:
+        raise web.HTTPNotFound()
+    if data["deleted"]:
+        raise web.HTTPGone()
+    return web.json_response(CheckSchema().dump(dict(data)))
+
+
+async def get_all_checks(request: web.Request) -> web.Response:
+    url, resource_id = get_request_params(request, params_names=["url", "resource_id"])
+    data: Optional[list] = await Check.get_all(url, resource_id)
+    if not data:
+        raise web.HTTPNotFound()
+    return web.json_response([CheckSchema().dump(dict(r)) for r in data])
+
+
+async def create_check(request: web.Request) -> web.Response:
+    """Create a new check"""
+
+    # Get resource_id from request
+    try:
+        payload: dict = await request.json()
+        resource_id: str = payload["resource_id"]
+    except ValidationError as err:
+        raise web.HTTPBadRequest(text=json.dumps(err.messages))
+    except KeyError as e:
+        raise web.HTTPBadRequest(text=f"Missing key: {e}")
+
+    # Get URL from resource_id
+    try:
+        resource: dict = await Resource.get(resource_id, "url")
+        url: str = resource["url"]
+    except Exception:
+        raise web.HTTPNotFound(text=f"Couldn't find URL for resource {resource_id}")
+
+    context.monitor().set_status(f'Crawling url "{url}"...')
+
+    async with aiohttp.ClientSession(
+        timeout=None, headers={"user-agent": config.USER_AGENT}
+    ) as session:
+        status: str = await check_url(
+            url=url, resource_id=resource_id, session=session, worker_priority="high"
+        )
+        context.monitor().refresh(status)
+
+    check: Optional[dict] = await Check.get_latest(url, resource_id)
+    if not check:
+        raise web.HTTPBadRequest(text=f"Check not created, status: {status}")
+
+    return web.json_response(CheckSchema().dump(dict(check)))

udata_hydra-2.0.0.dev3084/udata_hydra/routes/checks.py

@@ -1,26 +0,0 @@
-from typing import Optional
-
-from aiohttp import web
-
-from udata_hydra.db.check import Check
-from udata_hydra.schemas import CheckSchema
-from udata_hydra.utils import get_request_params
-
-
-async def get_latest_check(request: web.Request) -> web.Response:
-    """Get the latest check for a given URL or resource_id"""
-    url, resource_id = get_request_params(request, params_names=["url", "resource_id"])
-    data: Optional[dict] = await Check.get_latest(url, resource_id)
-    if not data:
-        raise web.HTTPNotFound()
-    if data["deleted"]:
-        raise web.HTTPGone()
-    return web.json_response(CheckSchema().dump(dict(data)))
-
-
-async def get_all_checks(request: web.Request) -> web.Response:
-    url, resource_id = get_request_params(request, params_names=["url", "resource_id"])
-    data: Optional[list] = await Check.get_all(url, resource_id)
-    if not data:
-        raise web.HTTPNotFound()
-    return web.json_response([CheckSchema().dump(dict(r)) for r in data])