udata-hydra 2.0.0.dev2124__tar.gz → 2.0.0.dev2347__tar.gz

{udata_hydra-2.0.0.dev2124 → udata_hydra-2.0.0.dev2347}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata-hydra
-Version: 2.0.0.dev2124
+Version: 2.0.0.dev2347
 Summary: Async crawler and parsing service for data.gouv.fr
 License: MIT
 Author: Opendata Team
@@ -105,6 +105,11 @@ Then you can run the tests with `poetry run pytest`.
 
 If you would like to see print statements as they are executed, you can pass the -s flag to pytest (`poetry run pytest -s`). However, note that this can sometimes be difficult to parse.
 
+### Tests coverage
+
+Pytest automatically uses the `coverage` package to generate a coverage report, which is displayed at the end of the test run in the terminal.
+The coverage is configured in the `pypoject.toml` file, in the `[tool.pytest.ini_options]` section.
+You can also override the coverage report configuration when running the tests by passing some flags like `--cov-report` to pytest. See [the pytest-cov documentation](https://pytest-cov.readthedocs.io/en/latest/config.html) for more information.
 
 ## API
 

{udata_hydra-2.0.0.dev2124 → udata_hydra-2.0.0.dev2347}/README.md

@@ -69,6 +69,11 @@ Then you can run the tests with `poetry run pytest`.
 
 If you would like to see print statements as they are executed, you can pass the -s flag to pytest (`poetry run pytest -s`). However, note that this can sometimes be difficult to parse.
 
+### Tests coverage
+
+Pytest automatically uses the `coverage` package to generate a coverage report, which is displayed at the end of the test run in the terminal.
+The coverage is configured in the `pypoject.toml` file, in the `[tool.pytest.ini_options]` section.
+You can also override the coverage report configuration when running the tests by passing some flags like `--cov-report` to pytest. See [the pytest-cov documentation](https://pytest-cov.readthedocs.io/en/latest/config.html) for more information.
 
 ## API
 

{udata_hydra-2.0.0.dev2124 → udata_hydra-2.0.0.dev2347}/pyproject.toml

@@ -9,7 +9,7 @@ readme = "README.md"
 
 [tool.poetry]
 name = "udata-hydra"
-version = "2.0.0.dev2124"
+version = "2.0.0.dev2347"
 description = "Async crawler and parsing service for data.gouv.fr"
 authors = ["Opendata Team <opendatateam@data.gouv.fr>"]
 license = "MIT"
@@ -47,6 +47,7 @@ mypy = "^1.11.0"
 nest_asyncio = "^1.5.5"
 pytest = "^7.1.1"
 pytest-asyncio = "^0.18.3"
+pytest-cov = "^5.0.0"
 pytest-mock = "^3.7.0"
 ruff = "^0.5.2"
 
@@ -66,6 +67,14 @@ module = [
 ]
 ignore_missing_imports = true
 
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+markers = [
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "catalog_harvested: use catalog_harvested.csv as source",
+]
+addopts = "--cov=udata_hydra/ --cov-report term-missing"
+
 [tool.ruff]
 lint = { select = ["I"] } # also sort imports with an isort rule
 line-length = 100
@@ -74,13 +83,6 @@ line-length = 100
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
-[tool.pytest.ini_options]
-asyncio_mode = "strict"
-markers = [
-    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
-    "catalog_harvested: use catalog_harvested.csv as source",
-]
-
 [tool.poetry.scripts]
 udata-hydra = "udata_hydra.cli:run"
 udata-hydra-crawl = "udata_hydra.crawl:run"

{udata_hydra-2.0.0.dev2124 → udata_hydra-2.0.0.dev2347}/udata_hydra/analysis/csv.py

@@ -32,8 +32,9 @@ from str2float import str2float
 from udata_hydra import config, context
 from udata_hydra.analysis import helpers
 from udata_hydra.analysis.errors import ParseException
+from udata_hydra.db import compute_insert_query
+from udata_hydra.db.check import Check
 from udata_hydra.utils import queue
-from udata_hydra.utils.db import compute_insert_query, get_check, update_check
 from udata_hydra.utils.file import download_resource
 from udata_hydra.utils.http import send
 from udata_hydra.utils.reader import Reader
@@ -76,7 +77,7 @@ RESERVED_COLS = ("__id", "tableoid", "xmin", "cmin", "xmax", "cmax", "ctid")
 
 async def notify_udata(check_id: int) -> None:
     """Notify udata of the result of a parsing"""
-    check = await get_check(check_id)
+    check = await Check.get(check_id)
     resource_id = check["resource_id"]
     db = await context.pool()
     record = await db.fetchrow("SELECT dataset_id FROM catalog WHERE resource_id = $1", resource_id)
@@ -112,7 +113,7 @@ async def analyse_csv(
 
     timer = Timer("analyse-csv")
     assert any(_ is not None for _ in (check_id, url))
-    check = await get_check(check_id) if check_id is not None else {}
+    check = await Check.get(check_id) if check_id is not None else {}
     url = check.get("url") or url
     exception_file = str(check.get("resource_id", "")) in exceptions
 
@@ -131,13 +132,13 @@ async def analyse_csv(
 
     try:
         if check_id:
-            await update_check(check_id, {"parsing_started_at": datetime.now(timezone.utc)})
+            await Check.update(check_id, {"parsing_started_at": datetime.now(timezone.utc)})
         csv_inspection = await perform_csv_inspection(tmp_file.name)
         timer.mark("csv-inspection")
         await csv_to_db(tmp_file.name, csv_inspection, table_name, debug_insert=debug_insert)
         timer.mark("csv-to-db")
         if check_id:
-            await update_check(
+            await Check.update(
                 check_id,
                 {
                     "parsing_table": table_name,
@@ -234,7 +235,7 @@ async def csv_to_db(
             for r in bar.iter(records):
                 data = {k: v for k, v in zip(columns.keys(), r)}
                 # NB: possible sql injection here, but should not be used in prod
-                q = compute_insert_query(data, table_name, returning="__id")
+                q = compute_insert_query(table_name=table_name, data=data, returning="__id")
                 await db.execute(q, *data.values())
 
 
@@ -274,7 +275,7 @@ async def handle_parse_exception(e: Exception, check_id: int, table_name: str) -
     await db.execute(f'DROP TABLE IF EXISTS "{table_name}"')
     if check_id:
         if config.SENTRY_DSN:
-            check = await get_check(check_id)
+            check = await Check.get(check_id)
             url = check["url"]
             with sentry_sdk.push_scope() as scope:
                 scope.set_extra("check_id", check_id)
@@ -284,7 +285,7 @@ async def handle_parse_exception(e: Exception, check_id: int, table_name: str) -
         # e.__cause__ let us access the "inherited" error of ParseException (raise e from cause)
         # it's called explicit exception chaining and it's very cool, look it up (PEP 3134)!
         err = f"{e.step}:sentry:{event_id}" if config.SENTRY_DSN else f"{e.step}:{str(e.__cause__)}"
-        await update_check(
+        await Check.update(
             check_id,
             {"parsing_error": err, "parsing_finished_at": datetime.now(timezone.utc)},
         )

{udata_hydra-2.0.0.dev2124 → udata_hydra-2.0.0.dev2347}/udata_hydra/analysis/resource.py

@@ -10,9 +10,9 @@ from dateparser import parse as date_parser
 
 from udata_hydra import config, context
 from udata_hydra.analysis.csv import analyse_csv
+from udata_hydra.db.check import Check
 from udata_hydra.utils import queue
 from udata_hydra.utils.csv import detect_tabular_from_headers
-from udata_hydra.utils.db import get_check, update_check
 from udata_hydra.utils.file import compute_checksum_from_file, download_resource
 from udata_hydra.utils.http import send
 
@@ -37,7 +37,7 @@ async def process_resource(check_id: int, is_first_check: bool) -> None:
 
     Will call udata if first check or changes found, and update check with optionnal infos
     """
-    check: dict = await get_check(check_id)
+    check: dict = await Check.get(check_id)
     if not check:
         log.error(f"Check not found by id {check_id}")
         return
@@ -85,7 +85,7 @@ async def process_resource(check_id: int, is_first_check: bool) -> None:
         finally:
             if tmp_file and not is_tabular:
                 os.remove(tmp_file.name)
-            await update_check(
+            await Check.update(
                 check_id,
                 {
                     "checksum": dl_analysis.get("analysis:checksum"),
@@ -96,7 +96,7 @@ async def process_resource(check_id: int, is_first_check: bool) -> None:
             )
 
     if change_status == Change.HAS_CHANGED:
-        await store_last_modified_date(change_payload or {}, resource_id, check_id)
+        await store_last_modified_date(change_payload or {}, check_id)
 
     analysis_results = {**dl_analysis, **(change_payload or {})}
     if change_status == Change.HAS_CHANGED or is_first_check:
@@ -111,20 +111,14 @@ async def process_resource(check_id: int, is_first_check: bool) -> None:
         )
 
 
-async def store_last_modified_date(change_analysis, resource_id, check_id) -> None:
+async def store_last_modified_date(change_analysis: dict, check_id: int) -> None:
     """
     Store last modified date in checks because it may be useful for later comparison
     """
-    pool = await context.pool()
     last_modified = change_analysis.get("analysis:last-modified-at")
     if last_modified:
         last_modified = datetime.fromisoformat(last_modified)
-        async with pool.acquire() as conn:
-            await conn.execute(
-                "UPDATE checks SET detected_last_modified_at = $1 WHERE id = $2",
-                last_modified,
-                check_id,
-            )
+        await Check.update(check_id, {"detected_last_modified_at": last_modified})
 
 
 async def detect_resource_change_from_checksum(

{udata_hydra-2.0.0.dev2124 → udata_hydra-2.0.0.dev2347}/udata_hydra/app.py

@@ -4,11 +4,14 @@ from datetime import datetime, timedelta, timezone
 
 from aiohttp import web
 from humanfriendly import parse_timespan
-from marshmallow import Schema, ValidationError, fields
+from marshmallow import ValidationError
 
 from udata_hydra import config, context
 from udata_hydra.crawl import get_excluded_clause
+from udata_hydra.db.check import Check
+from udata_hydra.db.resource import Resource
 from udata_hydra.logger import setup_logging
+from udata_hydra.schemas import CheckSchema, ResourceQuerySchema
 from udata_hydra.utils.minio import delete_resource_from_minio
 from udata_hydra.worker import QUEUES
 
@@ -16,52 +19,8 @@ log = setup_logging()
 routes = web.RouteTableDef()
 
 
-class CheckSchema(Schema):
-    check_id = fields.Integer(data_key="id")
-    catalog_id = fields.Integer()
-    url = fields.Str()
-    domain = fields.Str()
-    created_at = fields.DateTime()
-    check_status = fields.Integer(data_key="status")
-    headers = fields.Function(lambda obj: json.loads(obj["headers"]) if obj["headers"] else {})
-    timeout = fields.Boolean()
-    response_time = fields.Float()
-    error = fields.Str()
-    dataset_id = fields.Str()
-    resource_id = fields.UUID()
-    deleted = fields.Boolean()
-    parsing_started_at = fields.DateTime()
-    parsing_finished_at = fields.DateTime()
-    parsing_error = fields.Str()
-    parsing_table = fields.Str()
-
-
-class ResourceDocument(Schema):
-    id = fields.Str(required=True)
-    url = fields.Str(required=True)
-    format = fields.Str(allow_none=True)
-    title = fields.Str(required=True)
-    schema = fields.Dict(allow_none=True)
-    description = fields.Str(allow_none=True)
-    filetype = fields.Str(required=True)
-    type = fields.Str(required=True)
-    mime = fields.Str(allow_none=True)
-    filesize = fields.Int(allow_none=True)
-    checksum_type = fields.Str(allow_none=True)
-    checksum_value = fields.Str(allow_none=True)
-    created_at = fields.DateTime(required=True)
-    last_modified = fields.DateTime(required=True)
-    extras = fields.Dict()
-    harvest = fields.Dict()
-
-
-class ResourceQuery(Schema):
-    dataset_id = fields.Str(required=True)
-    resource_id = fields.Str(required=True)
-    document = fields.Nested(ResourceDocument(), allow_none=True)
-
-
 def _get_args(request, params=("url", "resource_id")) -> list:
+    """Get GET parameters from request"""
     data = [request.query.get(param) for param in params]
     if not any(data):
         raise web.HTTPBadRequest()
@@ -70,9 +29,14 @@ def _get_args(request, params=("url", "resource_id")) -> list:
 
 @routes.post("/api/resource/created/")
 async def resource_created(request: web.Request) -> web.Response:
+    """Endpoint to receive a resource creation event from a source
+    Will create a new resource in the DB "catalog" table and mark it as priority for next crawling
+    Respond with a 200 status code and a JSON body with a message key set to "created"
+    If error, respond with a 400 status code
+    """
     try:
         payload = await request.json()
-        valid_payload = ResourceQuery().load(payload)
+        valid_payload: dict = ResourceQuerySchema().load(payload)
     except ValidationError as err:
         raise web.HTTPBadRequest(text=json.dumps(err.messages))
 
@@ -83,26 +47,26 @@ async def resource_created(request: web.Request) -> web.Response:
     dataset_id = valid_payload["dataset_id"]
     resource_id = valid_payload["resource_id"]
 
-    pool = request.app["pool"]
-    async with pool.acquire() as connection:
-        # Insert new resource in catalog table and mark as high priority for crawling
-        q = f"""
-                INSERT INTO catalog (dataset_id, resource_id, url, deleted, priority)
-                VALUES ('{dataset_id}', '{resource_id}', '{resource["url"]}', FALSE, TRUE)
-                ON CONFLICT (resource_id) DO UPDATE SET
-                    priority = TRUE,
-                    url = '{resource["url"]}',
-                    dataset_id = '{dataset_id}';"""
-        await connection.execute(q)
+    await Resource.insert(
+        dataset_id=dataset_id,
+        resource_id=resource_id,
+        url=resource["url"],
+        priority=True,
+    )
 
     return web.json_response({"message": "created"})
 
 
 @routes.post("/api/resource/updated/")
 async def resource_updated(request: web.Request) -> web.Response:
+    """Endpoint to receive a resource update event from a source
+    Will update an existing resource in the DB "catalog" table and mark it as priority for next crawling
+    Respond with a 200 status code and a JSON body with a message key set to "updated"
+    If error, respond with a 400 status code
+    """
     try:
         payload = await request.json()
-        valid_payload = ResourceQuery().load(payload)
+        valid_payload: dict = ResourceQuerySchema().load(payload)
     except ValidationError as err:
         raise web.HTTPBadRequest(text=json.dumps(err.messages))
 
@@ -113,24 +77,7 @@ async def resource_updated(request: web.Request) -> web.Response:
     dataset_id = valid_payload["dataset_id"]
     resource_id = valid_payload["resource_id"]
 
-    pool = request.app["pool"]
-    async with pool.acquire() as connection:
-        # Make resource high priority for crawling
-        # Check if resource is in catalog then insert or update into table
-        q = f"""SELECT * FROM catalog WHERE resource_id = '{resource_id}';"""
-        res = await connection.fetch(q)
-        if len(res):
-            q = f"""UPDATE catalog SET priority = TRUE, url = '{resource["url"]}'
-            WHERE resource_id = '{resource_id}';"""
-        else:
-            q = f"""
-                    INSERT INTO catalog (dataset_id, resource_id, url, deleted, priority)
-                    VALUES ('{dataset_id}', '{resource_id}', '{resource["url"]}', FALSE, TRUE)
-                    ON CONFLICT (resource_id) DO UPDATE SET
-                        priority = TRUE,
-                        url = '{resource["url"]}',
-                        dataset_id = '{dataset_id}';"""
-        await connection.execute(q)
+    await Resource.update_or_insert(dataset_id, resource_id, resource["url"])
 
     return web.json_response({"message": "updated"})
 
@@ -139,7 +86,7 @@ async def resource_updated(request: web.Request) -> web.Response:
 async def resource_deleted(request: web.Request) -> web.Response:
     try:
         payload = await request.json()
-        valid_payload = ResourceQuery().load(payload)
+        valid_payload: dict = ResourceQuerySchema().load(payload)
     except ValidationError as err:
         raise web.HTTPBadRequest(text=json.dumps(err.messages))
 
@@ -159,16 +106,9 @@ async def resource_deleted(request: web.Request) -> web.Response:
 
 @routes.get("/api/checks/latest/")
 async def get_check(request: web.Request) -> web.Response:
+    """Get the latest check for a given URL or resource_id"""
     url, resource_id = _get_args(request)
-    column = "url" if url else "resource_id"
-    q = f"""
-    SELECT catalog.id as catalog_id, checks.id as check_id,
-           catalog.status as catalog_status, checks.status as check_status, *
-    FROM checks, catalog
-    WHERE checks.id = catalog.last_check
-    AND catalog.{column} = $1
-    """
-    data = await request.app["pool"].fetchrow(q, url or resource_id)
+    data = await Check.get_latest(url, resource_id)
     if not data:
         raise web.HTTPNotFound()
     if data["deleted"]:
@@ -179,16 +119,7 @@ async def get_check(request: web.Request) -> web.Response:
 @routes.get("/api/checks/all/")
 async def get_checks(request: web.Request) -> web.Response:
     url, resource_id = _get_args(request)
-    column = "url" if url else "resource_id"
-    q = f"""
-    SELECT catalog.id as catalog_id, checks.id as check_id,
-           catalog.status as catalog_status, checks.status as check_status, *
-    FROM checks, catalog
-    WHERE catalog.{column} = $1
-    AND catalog.url = checks.url
-    ORDER BY created_at DESC
-    """
-    data = await request.app["pool"].fetch(q, url or resource_id)
+    data = await Check.get_all(url, resource_id)
     if not data:
         raise web.HTTPNotFound()
     return web.json_response([CheckSchema().dump(dict(r)) for r in data])

{udata_hydra-2.0.0.dev2124 → udata_hydra-2.0.0.dev2347}/udata_hydra/cli.py

@@ -15,6 +15,7 @@ from progressist import ProgressBar
 from udata_hydra import config
 from udata_hydra.analysis.csv import analyse_csv, delete_table
 from udata_hydra.crawl import check_url as crawl_check_url
+from udata_hydra.db.resource import Resource
 from udata_hydra.logger import setup_logging
 from udata_hydra.migrations import Migrator
 
@@ -71,10 +72,10 @@ async def load_catalog(url=None, drop_meta=False, drop_all=False, quiet=False):
                 yield row
 
     try:
-        log.info(f"Downloading catalog from {url}...")
+        log.info(f"Downloading resources catalog from {url}...")
         with NamedTemporaryFile(dir=config.TEMPORARY_DOWNLOAD_FOLDER or None, delete=False) as fd:
             await download_file(url, fd)
-        log.info("Upserting catalog in database...")
+        log.info("Upserting resources catalog in database...")
         # consider everything deleted, deleted will be updated when loading new catalog
         conn = await connection()
         await conn.execute("UPDATE catalog SET deleted = TRUE")
@@ -106,7 +107,7 @@ async def load_catalog(url=None, drop_meta=False, drop_all=False, quiet=False):
                     if row["harvest.modified_at"]
                     else None,
                 )
-        log.info("Catalog successfully upserted into DB.")
+        log.info("Resources catalog successfully upserted into DB.")
     except Exception as e:
         raise e
     finally:
@@ -132,14 +133,12 @@ async def check_url(url: str, method: str = "get"):
 @cli
 async def check_resource(resource_id, method: str = "get"):
     """Trigger a complete check for a given resource_id"""
-    q = "SELECT * FROM catalog WHERE resource_id = $1"
-    conn = await connection()
-    res = await conn.fetch(q, resource_id)
+    res = await Resource.get(resource_id)
     if not res:
         log.error("Resource not found in catalog")
         return
     async with aiohttp.ClientSession(timeout=None) as session:
-        await crawl_check_url(res[0], session, method=method)
+        await crawl_check_url(url=res[0], resource_id=None, session=session, method=method)
 
 
 @cli(name="analyse-csv")
@@ -158,7 +157,7 @@ async def csv_sample(size=1000, download: bool = False, max_size: str = "100M"):
     :download: Download files or just list them
     :max_size: Maximum size for one file (from headers)
     """
-    max_size = parse_size(max_size)
+    max_size: int = parse_size(max_size)
     start_q = f"""
         SELECT catalog.resource_id, catalog.dataset_id, checks.url,
             checks.headers->>'content-type' as content_type,

{udata_hydra-2.0.0.dev2124 → udata_hydra-2.0.0.dev2347}/udata_hydra/crawl.py

@@ -11,12 +11,13 @@ from humanfriendly import parse_timespan
 
 from udata_hydra import config, context
 from udata_hydra.analysis.resource import process_resource
+from udata_hydra.db.check import Check
+from udata_hydra.db.resource import Resource
 from udata_hydra.logger import setup_logging
 from udata_hydra.utils import queue
-from udata_hydra.utils.db import insert_check
 from udata_hydra.utils.http import send
 
-results = defaultdict(int)
+results: defaultdict = defaultdict(int)
 
 STATUS_OK = "ok"
 STATUS_TIMEOUT = "timeout"
@@ -51,7 +52,7 @@ async def get_content_type_from_header(headers: dict) -> str:
     return content_type
 
 
-async def compute_check_has_changed(check_data, last_check) -> bool:
+async def compute_check_has_changed(check_data: dict, last_check: dict) -> bool:
     is_first_check = not last_check
     status_has_changed = last_check and check_data.get("status") != last_check.get("status")
     status_no_longer_available = (
@@ -95,13 +96,10 @@ async def compute_check_has_changed(check_data, last_check) -> bool:
             )
             or None,
         }
-        pool = await context.pool()
-        async with pool.acquire() as conn:
-            q = "SELECT dataset_id FROM CATALOG where resource_id = $1"
-            dataset = await conn.fetchrow(q, check_data["resource_id"])
+        res = await Resource.get(resource_id=check_data["resource_id"], column_name="dataset_id")
         queue.enqueue(
             send,
-            dataset_id=dataset["dataset_id"],
+            dataset_id=res["dataset_id"],
             resource_id=check_data["resource_id"],
             document=document,
             _priority="high",
@@ -110,15 +108,6 @@ async def compute_check_has_changed(check_data, last_check) -> bool:
     return has_changed
 
 
-async def update_catalog_following_check(resource_id: int):
-    pool = await context.pool()
-    async with pool.acquire() as connection:
-        await connection.execute(
-            "UPDATE catalog SET priority = FALSE, status = NULL WHERE resource_id = $1",
-            resource_id,
-        )
-
-
 async def process_check_data(check_data: dict) -> Tuple[int, bool]:
     """Preprocess a check before saving it"""
     check_data["resource_id"] = str(check_data["resource_id"])
@@ -135,13 +124,15 @@ async def process_check_data(check_data: dict) -> Tuple[int, bool]:
 
         await compute_check_has_changed(check_data, dict(last_check) if last_check else None)
 
-    await update_catalog_following_check(check_data["resource_id"])
+    await Resource.update(
+        resource_id=check_data["resource_id"], data={"status": None, "priority": False}
+    )
 
     is_first_check = last_check is None
-    return await insert_check(check_data), is_first_check
+    return await Check.insert(check_data), is_first_check
 
 
-async def is_backoff(domain) -> Tuple[bool, str]:
+async def is_backoff(domain: str) -> Tuple[bool, str]:
     backoff = False, ""
     no_backoff = [f"'{d}'" for d in config.NO_BACKOFF_DOMAINS]
     no_backoff = f"({','.join(no_backoff)})"
@@ -232,7 +223,7 @@ def convert_headers(headers):
     return _headers
 
 
-def has_nice_head(resp):
+def has_nice_head(resp) -> bool:
     """Check if a HEAD response looks useful to us"""
     if not is_valid_status(resp.status):
         return False
@@ -241,20 +232,22 @@ def has_nice_head(resp):
     return True
 
 
-async def check_url(row, session, sleep=0, method="head"):
-    log.debug(f"check {row}, sleep {sleep}, method {method}")
+async def check_url(
+    url: str, resource_id: str, session, sleep: float = 0, method: str = "head"
+) -> str:
+    log.debug(f"check {url}, sleep {sleep}, method {method}")
 
     if sleep:
         await asyncio.sleep(sleep)
 
-    url_parsed = urlparse(row["url"])
+    url_parsed = urlparse(url)
     domain = url_parsed.netloc
     if not domain:
-        log.warning(f"[warning] not netloc in url, skipping {row['url']}")
+        log.warning(f"[warning] not netloc in url, skipping {url}")
         await process_check_data(
             {
-                "resource_id": row["resource_id"],
-                "url": row["url"],
+                "resource_id": resource_id,
+                "url": url,
                 "error": "Not netloc in url",
                 "timeout": False,
             }
@@ -265,23 +258,23 @@ async def check_url(row, session, sleep=0, method="head"):
     if should_backoff:
         log.info(f"backoff {domain} ({reason})")
         # skip this URL, it will come back in a next batch
-        await update_catalog_following_check(row["resource_id"])
+        await Resource.update(resource_id=resource_id, data={"status": None, "priority": False})
         return STATUS_BACKOFF
 
     try:
         start = time.time()
         timeout = aiohttp.ClientTimeout(total=5)
         _method = getattr(session, method)
-        async with _method(row["url"], timeout=timeout, allow_redirects=True) as resp:
+        async with _method(url, timeout=timeout, allow_redirects=True) as resp:
             end = time.time()
             if method != "get" and not has_nice_head(resp):
-                return await check_url(row, session, method="get")
+                return await check_url(url, resource_id, session, method="get")
             resp.raise_for_status()
 
             check_id, is_first_check = await process_check_data(
                 {
-                    "resource_id": row["resource_id"],
-                    "url": row["url"],
+                    "resource_id": resource_id,
+                    "url": url,
                     "domain": domain,
                     "status": resp.status,
                     "headers": convert_headers(resp.headers),
@@ -296,8 +289,8 @@ async def check_url(row, session, sleep=0, method="head"):
     except asyncio.exceptions.TimeoutError:
         await process_check_data(
             {
-                "resource_id": row["resource_id"],
-                "url": row["url"],
+                "resource_id": resource_id,
+                "url": url,
                 "domain": domain,
                 "timeout": True,
             }
@@ -316,8 +309,8 @@ async def check_url(row, session, sleep=0, method="head"):
         error = getattr(e, "message", None) or str(e)
         await process_check_data(
             {
-                "resource_id": row["resource_id"],
-                "url": row["url"],
+                "resource_id": resource_id,
+                "url": url,
                 "domain": domain,
                 "timeout": False,
                 "error": fix_surrogates(error),
@@ -325,25 +318,25 @@ async def check_url(row, session, sleep=0, method="head"):
                 "status": getattr(e, "status", None),
             }
         )
-        log.warning(f"Crawling error for url {row['url']}", exc_info=e)
+        log.warning(f"Crawling error for url {url}", exc_info=e)
         return STATUS_ERROR
 
 
-async def crawl_urls(to_parse):
+async def crawl_urls(to_parse: list[str]) -> None:
     context.monitor().set_status("Crawling urls...")
-    tasks = []
+    tasks: list = []
     async with aiohttp.ClientSession(
         timeout=None, headers={"user-agent": config.USER_AGENT}
     ) as session:
         for row in to_parse:
-            tasks.append(check_url(row, session))
+            tasks.append(check_url(url=row["url"], resource_id=row["resource_id"], session=session))
         for task in asyncio.as_completed(tasks):
             result = await task
             results[result] += 1
             context.monitor().refresh(results)
 
 
-def get_excluded_clause():
+def get_excluded_clause() -> str:
     return " AND ".join(
         [f"catalog.url NOT LIKE '{p}'" for p in config.EXCLUDED_PATTERNS]
         + [
@@ -379,7 +372,7 @@ async def select_rows_based_on_query(connection, q, *args):
     return to_check
 
 
-async def crawl_batch():
+async def crawl_batch() -> None:
     """Crawl a batch from the catalog"""
     context.monitor().set_status("Getting a batch from catalog...")
     pool = await context.pool()
@@ -435,7 +428,7 @@ async def crawl_batch():
     await asyncio.sleep(config.SLEEP_BETWEEN_BATCHES)
 
 
-async def crawl(iterations=-1):
+async def crawl(iterations: int = -1) -> None:
     """Launch crawl batches
 
     :iterations: for testing purposes (break infinite loop)
@@ -455,7 +448,7 @@ async def crawl(iterations=-1):
         await pool.close()
 
 
-def run():
+def run() -> None:
     """Main function
 
     :iterations: for testing purposes (break infinite loop)

udata_hydra-2.0.0.dev2347/udata_hydra/db/__init__.py

@@ -0,0 +1,46 @@
+import json
+
+from udata_hydra import context
+
+
+def convert_dict_values_to_json(data: dict) -> dict:
+    """
+    Convert values in dict that are dict to json for DB serialization
+    TODO: this is suboptimal from asyncpg, dig into this
+    https://magicstack.github.io/asyncpg/current/usage.html#example-automatic-json-conversion
+    """
+    for k, v in data.items():
+        if type(v) is dict:
+            data[k] = json.dumps(v)
+    return data
+
+
+def compute_insert_query(table_name: str, data: dict, returning: str = "id") -> str:
+    columns = ",".join([f'"{k}"' for k in data.keys()])
+    # $1, $2...
+    placeholders = ",".join([f"${x + 1}" for x in range(len(data.values()))])
+    return f"""
+        INSERT INTO "{table_name}" ({columns})
+        VALUES ({placeholders})
+        RETURNING {returning}
+    """
+
+
+def compute_update_query(table_name: str, data: dict) -> str:
+    columns = data.keys()
+    # $1, $2...
+    placeholders = [f"${x + 1}" for x in range(len(data.values()))]
+    set_clause = ",".join([f"{c} = {v}" for c, v in zip(columns, placeholders)])
+    return f"""
+        UPDATE "{table_name}"
+        SET {set_clause}
+        WHERE id = ${len(placeholders) + 1}
+    """
+
+
+async def update_table_record(table_name: str, record_id: int, data: dict) -> int:
+    data = convert_dict_values_to_json(data)
+    q = compute_update_query(table_name, data)
+    pool = await context.pool()
+    await pool.execute(q, *data.values(), record_id)
+    return record_id

udata_hydra-2.0.0.dev2347/udata_hydra/db/check.py

@@ -0,0 +1,73 @@
+from typing import Union
+
+from udata_hydra import context
+from udata_hydra.db import (
+    compute_insert_query,
+    convert_dict_values_to_json,
+    update_table_record,
+)
+
+
+class Check:
+    """Represents a check in the "checks" DB table"""
+
+    @classmethod
+    async def get(cls, check_id: int) -> dict:
+        pool = await context.pool()
+        async with pool.acquire() as connection:
+            q = """
+                SELECT * FROM catalog JOIN checks
+                ON catalog.last_check = checks.id
+                WHERE checks.id = $1;
+            """
+            check = await connection.fetchrow(q, check_id)
+        return check
+
+    @classmethod
+    async def get_latest(cls, url: Union[str, None], resource_id: Union[str, None]) -> dict | None:
+        column: str = "url" if url else "resource_id"
+        pool = await context.pool()
+        async with pool.acquire() as connection:
+            q = f"""
+            SELECT catalog.id as catalog_id, checks.id as check_id,
+                catalog.status as catalog_status, checks.status as check_status, *
+            FROM checks, catalog
+            WHERE checks.id = catalog.last_check
+            AND catalog.{column} = $1
+            """
+            return await connection.fetchrow(q, url or resource_id)
+
+    @classmethod
+    async def get_all(cls, url: Union[str, None], resource_id: Union[str, None]) -> dict | None:
+        column: str = "url" if url else "resource_id"
+        pool = await context.pool()
+        async with pool.acquire() as connection:
+            q = f"""
+            SELECT catalog.id as catalog_id, checks.id as check_id,
+                catalog.status as catalog_status, checks.status as check_status, *
+            FROM checks, catalog
+            WHERE catalog.{column} = $1
+            AND catalog.url = checks.url
+            ORDER BY created_at DESC
+            """
+            return await connection.fetch(q, url or resource_id)
+
+    @classmethod
+    async def insert(cls, data: dict) -> int:
+        """
+        Insert a new check in DB and return the check id in DB
+        This use the info from the last check of the same resource
+        """
+        data = convert_dict_values_to_json(data)
+        q = compute_insert_query(table_name="checks", data=data)
+        pool = await context.pool()
+        async with pool.acquire() as connection:
+            last_check = await connection.fetchrow(q, *data.values())
+            q = """UPDATE catalog SET last_check = $1 WHERE resource_id = $2"""
+            await connection.execute(q, last_check["id"], data["resource_id"])
+        return last_check["id"]
+
+    @classmethod
+    async def update(cls, check_id: int, data: dict) -> int:
+        """Update a check in DB with new data and return the check id in DB"""
+        return await update_table_record(table_name="checks", record_id=check_id, data=data)

udata_hydra-2.0.0.dev2347/udata_hydra/db/resource.py

@@ -0,0 +1,74 @@
+from udata_hydra import context
+
+
+class Resource:
+    """Represents a resource in the "catalog" DB table"""
+
+    @classmethod
+    async def get(cls, resource_id: str, column_name: str = "*") -> dict:
+        pool = await context.pool()
+        async with pool.acquire() as connection:
+            q = f"""SELECT {column_name} FROM catalog WHERE resource_id = '{resource_id}';"""
+            resource = await connection.fetchrow(q)
+        return resource
+
+    @classmethod
+    async def insert(
+        cls,
+        dataset_id: str,
+        resource_id: str,
+        url: str,
+        priority: bool = True,
+    ) -> None:
+        pool = await context.pool()
+        async with pool.acquire() as connection:
+            # Insert new resource in catalog table and mark as high priority for crawling
+            q = f"""
+                    INSERT INTO catalog (dataset_id, resource_id, url, deleted, status, priority)
+                    VALUES ('{dataset_id}', '{resource_id}', '{url}', FALSE, 'TO_CHECK', '{priority}')
+                    ON CONFLICT (resource_id) DO UPDATE SET
+                        dataset_id = '{dataset_id}',
+                        url = '{url}',
+                        priority = '{priority}';"""
+            await connection.execute(q)
+
+    @classmethod
+    async def update(cls, resource_id: str, data: dict) -> str:
+        """Update a resource in DB with new data and return the updated resource id in DB"""
+        columns = data.keys()
+        # $1, $2...
+        placeholders = [f"${x + 1}" for x in range(len(data.values()))]
+        set_clause = ",".join([f"{c} = {v}" for c, v in zip(columns, placeholders)])
+        q = f"""
+                UPDATE catalog
+                SET {set_clause}
+                WHERE resource_id = ${len(placeholders) + 1};"""
+        pool = await context.pool()
+        await pool.execute(q, *data.values(), resource_id)
+        return resource_id
+
+    @classmethod
+    async def update_or_insert(
+        cls,
+        dataset_id: str,
+        resource_id: str,
+        url: str,
+        priority: bool = True,  # Make resource high priority by default for crawling
+    ) -> None:
+        pool = await context.pool()
+        async with pool.acquire() as connection:
+            # Check if resource is in catalog then insert or update into table
+            if await Resource.get(resource_id):
+                q = f"""
+                        UPDATE catalog
+                        SET url = '{url}', priority = '{priority}'
+                        WHERE resource_id = '{resource_id}';"""
+            else:
+                q = f"""
+                        INSERT INTO catalog (dataset_id, resource_id, url, deleted, priority)
+                        VALUES ('{dataset_id}', '{resource_id}', '{url}', FALSE, '{priority}')
+                        ON CONFLICT (resource_id) DO UPDATE SET
+                            dataset_id = '{dataset_id}',
+                            url = '{url}',
+                            priority = '{priority}';"""
+            await connection.execute(q)

udata_hydra-2.0.0.dev2347/udata_hydra/schemas/__init__.py

@@ -0,0 +1,2 @@
+from .check import CheckSchema
+from .resource_query import ResourceQuerySchema

udata_hydra-2.0.0.dev2347/udata_hydra/schemas/check.py

@@ -0,0 +1,27 @@
+import json
+
+from aiohttp import web
+from marshmallow import Schema, fields
+
+
+class CheckSchema(Schema):
+    check_id = fields.Integer(data_key="id")
+    catalog_id = fields.Integer()
+    url = fields.Str()
+    domain = fields.Str()
+    created_at = fields.DateTime()
+    check_status = fields.Integer(data_key="status")
+    headers = fields.Function(lambda obj: json.loads(obj["headers"]) if obj["headers"] else {})
+    timeout = fields.Boolean()
+    response_time = fields.Float()
+    error = fields.Str()
+    dataset_id = fields.Str()
+    resource_id = fields.UUID()
+    deleted = fields.Boolean()
+    parsing_started_at = fields.DateTime()
+    parsing_finished_at = fields.DateTime()
+    parsing_error = fields.Str()
+    parsing_table = fields.Str()
+
+    def create(self, data):
+        return self.load(data)

udata_hydra-2.0.0.dev2347/udata_hydra/schemas/resource_query.py

@@ -0,0 +1,26 @@
+from marshmallow import Schema, fields
+
+
+class ResourceDocumentSchema(Schema):
+    id = fields.Str(required=True)
+    url = fields.Str(required=True)
+    format = fields.Str(allow_none=True)
+    title = fields.Str(required=True)
+    schema = fields.Dict(allow_none=True)
+    description = fields.Str(allow_none=True)
+    filetype = fields.Str(required=True)
+    type = fields.Str(required=True)
+    mime = fields.Str(allow_none=True)
+    filesize = fields.Int(allow_none=True)
+    checksum_type = fields.Str(allow_none=True)
+    checksum_value = fields.Str(allow_none=True)
+    created_at = fields.DateTime(required=True)
+    last_modified = fields.DateTime(required=True)
+    extras = fields.Dict()
+    harvest = fields.Dict()
+
+
+class ResourceQuerySchema(Schema):
+    dataset_id = fields.Str(required=True)
+    resource_id = fields.Str(required=True)
+    document = fields.Nested(ResourceDocumentSchema(), allow_none=True)

udata_hydra-2.0.0.dev2124/udata_hydra/utils/db.py

@@ -1,73 +0,0 @@
-import json
-
-from udata_hydra import context
-
-
-def convert_dict_values_to_json(data: dict) -> dict:
-    """
-    Convert values in dict that are dict to json for DB serialization
-    TODO: this is suboptimal from asyncpg, dig into this
-    https://magicstack.github.io/asyncpg/current/usage.html#example-automatic-json-conversion
-    """
-    for k, v in data.items():
-        if type(v) is dict:
-            data[k] = json.dumps(v)
-    return data
-
-
-def compute_insert_query(data: dict, table: str, returning: str = "id") -> str:
-    columns = ",".join([f'"{k}"' for k in data.keys()])
-    # $1, $2...
-    placeholders = ",".join([f"${x + 1}" for x in range(len(data.values()))])
-    return f"""
-        INSERT INTO "{table}" ({columns})
-        VALUES ({placeholders})
-        RETURNING {returning}
-    """
-
-
-async def insert_check(data: dict) -> int:
-    data = convert_dict_values_to_json(data)
-    q = compute_insert_query(data, "checks")
-    pool = await context.pool()
-    async with pool.acquire() as connection:
-        last_check = await connection.fetchrow(q, *data.values())
-        q = """UPDATE catalog SET last_check = $1 WHERE resource_id = $2"""
-        await connection.execute(q, last_check["id"], data["resource_id"])
-    return last_check["id"]
-
-
-def compute_update_query(table: str, data: dict) -> str:
-    columns = data.keys()
-    # $1, $2...
-    placeholders = [f"${x + 1}" for x in range(len(data.values()))]
-    set_clause = ",".join([f"{c} = {v}" for c, v in zip(columns, placeholders)])
-    return f"""
-        UPDATE "{table}"
-        SET {set_clause}
-        WHERE id = ${len(placeholders) + 1}
-    """
-
-
-async def update_table_record(table: str, record_id: int, data: dict) -> int:
-    data = convert_dict_values_to_json(data)
-    q = compute_update_query(table, data)
-    pool = await context.pool()
-    await pool.execute(q, *data.values(), record_id)
-    return record_id
-
-
-async def update_check(check_id: int, data: dict) -> int:
-    return await update_table_record("checks", check_id, data)
-
-
-async def get_check(check_id: int) -> dict:
-    pool = await context.pool()
-    async with pool.acquire() as connection:
-        q = """
-            SELECT * FROM catalog JOIN checks
-            ON catalog.last_check = checks.id
-            WHERE checks.id = $1;
-        """
-        check = await connection.fetchrow(q, check_id)
-    return check

udata_hydra-2.0.0.dev2124/udata_hydra/utils/json.py

@@ -1,11 +0,0 @@
-import json
-
-
-def is_json_file(filename: str) -> bool:
-    """Checks if a string is a valid json"""
-    try:
-        with open(filename) as f:
-            json.load(f)
-            return True
-    except ValueError:
-        return False