udata-hydra 2.0.0.dev2998__tar.gz → 2.0.0.dev3046__tar.gz

{udata_hydra-2.0.0.dev2998 → udata_hydra-2.0.0.dev3046}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata-hydra
-Version: 2.0.0.dev2998
+Version: 2.0.0.dev3046
 Summary: Async crawler and parsing service for data.gouv.fr
 License: MIT
 Author: Opendata Team

{udata_hydra-2.0.0.dev2998 → udata_hydra-2.0.0.dev3046}/pyproject.toml

@@ -9,7 +9,7 @@ readme = "README.md"
 
 [tool.poetry]
 name = "udata-hydra"
-version = "2.0.0.dev2998"
+version = "2.0.0.dev3046"
 description = "Async crawler and parsing service for data.gouv.fr"
 authors = ["Opendata Team <opendatateam@data.gouv.fr>"]
 license = "MIT"

{udata_hydra-2.0.0.dev2998 → udata_hydra-2.0.0.dev3046}/udata_hydra/analysis/csv.py

@@ -5,7 +5,7 @@ import logging
 import os
 import sys
 from datetime import datetime, timezone
-from typing import Any, Iterator, Union
+from typing import Any, Iterator, Optional
 
 import sentry_sdk
 from csv_detective.detection import engine_to_file
@@ -100,9 +100,9 @@ async def notify_udata(check_id: int, table_name: str) -> None:
 
 
 async def analyse_csv(
-    check_id: Union[int, None] = None,
-    url: Union[str, None] = None,
-    file_path: Union[str, None] = None,
+    check_id: Optional[int] = None,
+    url: Optional[str] = None,
+    file_path: Optional[str] = None,
     debug_insert: bool = False,
 ) -> None:
     """Launch csv analysis from a check or an URL (debug), using previsously downloaded file at file_path if any"""
@@ -289,7 +289,7 @@ async def csv_to_db_index(table_name: str, inspection: dict, check: dict) -> Non
     )
 
 
-async def perform_csv_inspection(file_path: str) -> Union[dict, None]:
+async def perform_csv_inspection(file_path: str) -> Optional[dict]:
     """Launch csv-detective against given file"""
     try:
         return csv_detective_routine(

{udata_hydra-2.0.0.dev2998 → udata_hydra-2.0.0.dev3046}/udata_hydra/analysis/helpers.py

@@ -1,5 +1,5 @@
 from datetime import date, datetime
-from typing import Union
+from typing import Optional
 
 from dateparser import parse as date_parser
 from dateutil.parser import ParserError
@@ -11,7 +11,7 @@ def to_json(value: str) -> str:
     return value
 
 
-def _parse_dt(value: str) -> Union[datetime, None]:
+def _parse_dt(value: str) -> Optional[datetime]:
     """For performance reasons, we try first with dateutil and fallback on dateparser"""
     try:
         return dateutil_parser(value)
@@ -19,10 +19,10 @@ def _parse_dt(value: str) -> Union[datetime, None]:
         return date_parser(value)
 
 
-def to_date(value: str) -> Union[date, None]:
+def to_date(value: str) -> Optional[date]:
     parsed = _parse_dt(value)
     return parsed.date() if parsed else None
 
 
-def to_datetime(value: str) -> Union[datetime, None]:
+def to_datetime(value: str) -> Optional[datetime]:
     return _parse_dt(value)

{udata_hydra-2.0.0.dev2998 → udata_hydra-2.0.0.dev3046}/udata_hydra/analysis/resource.py

@@ -3,7 +3,7 @@ import logging
 import os
 from datetime import datetime, timezone
 from enum import Enum
-from typing import Tuple, Union
+from typing import Optional, Tuple
 
 import magic
 from dateparser import parse as date_parser
@@ -126,7 +126,7 @@ async def store_last_modified_date(change_analysis: dict, check_id: int) -> None
 
 async def detect_resource_change_from_checksum(
     resource_id, new_checksum
-) -> Tuple[Change, Union[dict, None]]:
+) -> Tuple[Change, Optional[dict]]:
     """
     Checks if resource checksum has changed over time
     Returns a tuple with a Change status and an optional payload:
@@ -155,7 +155,7 @@ async def detect_resource_change_from_checksum(
 
 async def detect_resource_change_from_last_modified_header(
     data: dict,
-) -> Tuple[Change, Union[dict, None]]:
+) -> Tuple[Change, Optional[dict]]:
     # last modified header check
 
     if len(data) == 1 and data[0]["last_modified"]:
@@ -179,7 +179,7 @@ async def detect_resource_change_from_last_modified_header(
 
 async def detect_resource_change_from_content_length_header(
     data: dict,
-) -> Tuple[Change, Union[dict, None]]:
+) -> Tuple[Change, Optional[dict]]:
     # content-length variation between current and last check
     if len(data) <= 1 or not data[0]["content_length"]:
         return Change.NO_GUESS, None
@@ -194,7 +194,7 @@ async def detect_resource_change_from_content_length_header(
 
 async def detect_resource_change_on_early_hints(
     resource_id: str,
-) -> Tuple[Change, Union[dict, None]]:
+) -> Tuple[Change, Optional[dict]]:
     """
     Try to guess if a resource has been modified from harvest and headers in check data:
     - last-modified header value if it can be found and parsed
@@ -245,7 +245,7 @@ async def detect_resource_change_on_early_hints(
 
 async def detect_resource_change_from_harvest(
     checks_data: dict, resource_id: str
-) -> Tuple[Change, Union[dict, None]]:
+) -> Tuple[Change, Optional[dict]]:
     """
     Checks if resource has a harvest.modified_at
     Returns a tuple with a Change status and an optional payload:

{udata_hydra-2.0.0.dev2998 → udata_hydra-2.0.0.dev3046}/udata_hydra/cli.py

@@ -4,7 +4,7 @@ import os
 from datetime import datetime, timezone
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Union
+from typing import Optional
 
 import aiohttp
 import asyncpg
@@ -47,7 +47,9 @@ async def connection(db_name: str = "main"):
 
 
 @cli
-async def load_catalog(url=None, drop_meta=False, drop_all=False, quiet=False):
+async def load_catalog(
+    url: Optional[str] = None, drop_meta: bool = False, drop_all: bool = False, quiet: bool = False
+):
     """Load the catalog into DB from CSV file
 
     :url: URL of the catalog to fetch, by default defined in config
@@ -135,7 +137,7 @@ async def check_url(url: str, method: str = "get"):
 
 
 @cli
-async def check_resource(resource_id, method: str = "get"):
+async def check_resource(resource_id: str, method: str = "get"):
     """Trigger a complete check for a given resource_id"""
     res = await Resource.get(resource_id)
     if not res:
@@ -147,14 +149,14 @@ async def check_resource(resource_id, method: str = "get"):
 
 @cli(name="analyse-csv")
 async def analyse_csv_cli(
-    check_id: Union[int, None] = None, url: Union[str, None] = None, debug_insert: bool = False
+    check_id: Optional[int] = None, url: Optional[str] = None, debug_insert: bool = False
 ):
     """Trigger a csv analysis from a check_id or an url"""
     await analyse_csv(check_id=check_id, url=url, debug_insert=debug_insert)
 
 
 @cli
-async def csv_sample(size=1000, download: bool = False, max_size: str = "100M"):
+async def csv_sample(size: int = 1000, download: bool = False, max_size: str = "100M"):
     """Get a csv sample from latest checks
 
     :size: Size of the sample (how many files to query)

{udata_hydra-2.0.0.dev2998 → udata_hydra-2.0.0.dev3046}/udata_hydra/config_default.toml

@@ -12,7 +12,7 @@ TESTING = false
 MAX_POOL_SIZE = 50
 USER_AGENT = "udata-hydra/1.0"
 
-API_TOKEN = "to-change"
+API_KEY = "hydra_api_key_to_change"
 
 # -- crawler settings -- #
 

{udata_hydra-2.0.0.dev2998 → udata_hydra-2.0.0.dev3046}/udata_hydra/context.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Union
+from typing import Optional
 from unittest.mock import MagicMock
 
 import asyncpg
@@ -36,7 +36,7 @@ async def pool(db: str = "main") -> asyncpg.pool.Pool:
     return context["databases"][db]
 
 
-def queue(name: str = "default") -> Union[Queue, None]:
+def queue(name: str = "default") -> Optional[Queue]:
     if not context["queues"].get(name):
         # we dont need a queue while testing, make sure we're not using a real Redis connection
         if config.TESTING:

{udata_hydra-2.0.0.dev2998 → udata_hydra-2.0.0.dev3046}/udata_hydra/crawl.py

@@ -3,7 +3,7 @@ import json
 import time
 from collections import defaultdict
 from datetime import datetime, timedelta, timezone
-from typing import Tuple, Union
+from typing import Optional, Tuple
 from urllib.parse import urlparse
 
 import aiohttp
@@ -26,7 +26,7 @@ STATUS_BACKOFF = "backoff"
 log = setup_logging()
 
 
-def is_valid_status(status: str) -> Union[bool, None]:
+def is_valid_status(status: str) -> Optional[bool]:
     if not status:
         return False
     status_nb = int(status)
@@ -52,7 +52,7 @@ async def get_content_type_from_header(headers: dict) -> str:
 
 
 async def compute_check_has_changed(check_data: dict, last_check: dict) -> bool:
-    is_first_check = not last_check
+    is_first_check: bool = last_check is None
     status_has_changed = last_check and check_data.get("status") != last_check.get("status")
     status_no_longer_available = (
         last_check
@@ -285,6 +285,7 @@ async def check_url(
                 }
             )
 
+            # Analyse the resource
             queue.enqueue(process_resource, check_id, is_first_check, _priority="low")
 
             return STATUS_OK

{udata_hydra-2.0.0.dev2998 → udata_hydra-2.0.0.dev3046}/udata_hydra/db/check.py

@@ -1,4 +1,4 @@
-from typing import Union
+from typing import Optional
 
 from udata_hydra import context
 from udata_hydra.db import (
@@ -12,7 +12,7 @@ class Check:
     """Represents a check in the "checks" DB table"""
 
     @classmethod
-    async def get(cls, check_id: int) -> dict:
+    async def get(cls, check_id: int) -> Optional[dict]:
         pool = await context.pool()
         async with pool.acquire() as connection:
             q = """
@@ -20,13 +20,12 @@ class Check:
                 ON catalog.last_check = checks.id
                 WHERE checks.id = $1;
             """
-            check = await connection.fetchrow(q, check_id)
-        return check
+            return await connection.fetchrow(q, check_id)
 
     @classmethod
     async def get_latest(
-        cls, url: Union[str, None], resource_id: Union[str, None]
-    ) -> Union[dict, None]:
+        cls, url: Optional[str] = None, resource_id: Optional[str] = None
+    ) -> Optional[dict]:
         column: str = "url" if url else "resource_id"
         pool = await context.pool()
         async with pool.acquire() as connection:
@@ -41,8 +40,8 @@ class Check:
 
     @classmethod
     async def get_all(
-        cls, url: Union[str, None], resource_id: Union[str, None]
-    ) -> Union[list, None]:
+        cls, url: Optional[str] = None, resource_id: Optional[str] = None
+    ) -> Optional[list]:
         column: str = "url" if url else "resource_id"
         pool = await context.pool()
         async with pool.acquire() as connection:

{udata_hydra-2.0.0.dev2998 → udata_hydra-2.0.0.dev3046}/udata_hydra/logger.py

@@ -1,6 +1,6 @@
 import logging
 import os
-from typing import Union
+from typing import Optional
 
 import coloredlogs
 import sentry_sdk
@@ -18,7 +18,7 @@ def setup_logging() -> logging.Logger:
     if context.get("inited"):
         return log
     release = "hydra@unknown"
-    app_version: Union[str, None] = get_app_version()
+    app_version: Optional[str] = get_app_version()
     if app_version:
         release = f"hydra@{app_version}"
     if config.SENTRY_DSN:

{udata_hydra-2.0.0.dev2998 → udata_hydra-2.0.0.dev3046}/udata_hydra/routes/checks.py

@@ -1,4 +1,4 @@
-from typing import Union
+from typing import Optional
 
 from aiohttp import web
 
@@ -10,7 +10,7 @@ from udata_hydra.utils import get_request_params
 async def get_latest_check(request: web.Request) -> web.Response:
     """Get the latest check for a given URL or resource_id"""
     url, resource_id = get_request_params(request, params_names=["url", "resource_id"])
-    data: Union[dict, None] = await Check.get_latest(url, resource_id)
+    data: Optional[dict] = await Check.get_latest(url, resource_id)
     if not data:
         raise web.HTTPNotFound()
     if data["deleted"]:
@@ -20,7 +20,7 @@ async def get_latest_check(request: web.Request) -> web.Response:
 
 async def get_all_checks(request: web.Request) -> web.Response:
     url, resource_id = get_request_params(request, params_names=["url", "resource_id"])
-    data: Union[list, None] = await Check.get_all(url, resource_id)
+    data: Optional[list] = await Check.get_all(url, resource_id)
     if not data:
         raise web.HTTPNotFound()
     return web.json_response([CheckSchema().dump(dict(r)) for r in data])

{udata_hydra-2.0.0.dev2998 → udata_hydra-2.0.0.dev3046}/udata_hydra/utils/app_version.py

@@ -1,12 +1,12 @@
 import logging
-from typing import Union
+from typing import Optional
 
 import toml
 
 log = logging.getLogger("udata-hydra")
 
 
-def get_app_version() -> Union[str, None]:
+def get_app_version() -> Optional[str]:
     """Get the app version from pyproject.toml"""
     try:
         pyproject: dict = toml.load("pyproject.toml")

{udata_hydra-2.0.0.dev2998 → udata_hydra-2.0.0.dev3046}/udata_hydra/utils/auth.py

@@ -62,8 +62,8 @@ def token_auth_middleware(
                 reason="Invalid token scheme",
             )
 
-        if token == config.API_TOKEN:
-            request[request_property] = {"username": "admin"}
+        if token == config.API_KEY:
+            request[request_property] = {"username": "udata"}
         else:
             raise web.HTTPForbidden(reason="Invalid authentication token")
 

{udata_hydra-2.0.0.dev2998 → udata_hydra-2.0.0.dev3046}/udata_hydra/utils/file.py

@@ -2,7 +2,7 @@ import gzip
 import hashlib
 import logging
 import tempfile
-from typing import IO, Union
+from typing import IO, Optional
 
 import aiohttp
 import magic
@@ -33,7 +33,7 @@ def read_csv_gz(file_path: str) -> IO[bytes]:
 async def download_resource(
     url: str,
     headers: dict,
-    max_size_allowed: Union[int, None],
+    max_size_allowed: Optional[int],
 ) -> IO[bytes]:
     """
     Attempts downloading a resource from a given url.