udata-hydra 2.0.0.dev2730__tar.gz → 2.0.0.dev2867__tar.gz

{udata_hydra-2.0.0.dev2730 → udata_hydra-2.0.0.dev2867}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata-hydra
-Version: 2.0.0.dev2730
+Version: 2.0.0.dev2867
 Summary: Async crawler and parsing service for data.gouv.fr
 License: MIT
 Author: Opendata Team

{udata_hydra-2.0.0.dev2730 → udata_hydra-2.0.0.dev2867}/pyproject.toml

@@ -9,7 +9,7 @@ readme = "README.md"
 
 [tool.poetry]
 name = "udata-hydra"
-version = "2.0.0.dev2730"
+version = "2.0.0.dev2867"
 description = "Async crawler and parsing service for data.gouv.fr"
 authors = ["Opendata Team <opendatateam@data.gouv.fr>"]
 license = "MIT"
@@ -17,27 +17,27 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "^3.9"
+aiocontextvars = "^0.2.2"
 aiohttp = "^3.10.3"
 asyncpg = "^0.29.0"
+coloredlogs = "^15.0.1"
+csv-detective = "0.7.2.dev800"
+dateparser = "^1.1.7"
 humanfriendly = "^10.0"
 marshmallow = "^3.14.1"
 minicli = "^0.5.3"
-progressist = "^0.1.0"
+minio = "7.2.7"
+pyarrow = "16.1.0"
+python-dateutil = "^2.8.2"
 python-magic = "^0.4.25"
+progressist = "^0.1.0"
 redis = "^4.1.4"
-sentry-sdk = "^2.10.0"
-aiocontextvars = "^0.2.2"
-coloredlogs = "^15.0.1"
 rq = "^1.11.1"
-toml = "^0.10.2"
+sentry-sdk = "^2.10.0"
+sqlalchemy = "^1.4.46"
 str2float = "^0.0.9"
 str2bool = "^1.1"
-sqlalchemy = "^1.4.46"
-dateparser = "^1.1.7"
-python-dateutil = "^2.8.2"
-csv-detective = "0.7.2.dev800"
-pyarrow = "16.1.0"
-minio = "7.2.7"
+toml = "^0.10.2"
 
 [tool.poetry.group.dev.dependencies]
 aiohttp-devtools = "^1.0.post0"

{udata_hydra-2.0.0.dev2730 → udata_hydra-2.0.0.dev2867}/udata_hydra/analysis/csv.py

@@ -74,7 +74,7 @@ RESERVED_COLS = ("__id", "tableoid", "xmin", "cmin", "xmax", "cmax", "ctid")
 minio_client = MinIOClient()
 
 
-async def notify_udata(check_id: str, table_name: str) -> None:
+async def notify_udata(check_id: int, table_name: str) -> None:
     """Notify udata of the result of a parsing"""
     check = await Check.get(check_id)
     resource_id = check["resource_id"]

{udata_hydra-2.0.0.dev2730 → udata_hydra-2.0.0.dev2867}/udata_hydra/app.py

@@ -4,6 +4,7 @@ from aiohttp import web
 
 from udata_hydra import context
 from udata_hydra.routes import routes
+from udata_hydra.utils import token_auth_middleware
 
 
 async def app_factory() -> web.Application:
@@ -14,16 +15,12 @@ async def app_factory() -> web.Application:
         if "pool" in app:
             await app["pool"].close()
 
-    app = web.Application()
+    app = web.Application(middlewares=[token_auth_middleware(exclude_methods=("GET",))])
     app.add_routes(routes)
     app.on_startup.append(app_startup)
     app.on_cleanup.append(app_cleanup)
     return app
 
 
-def run():
-    web.run_app(app_factory(), path=os.environ.get("HYDRA_APP_SOCKET_PATH"))
-
-
 if __name__ == "__main__":
-    run()
+    web.run_app(app_factory(), path=os.environ.get("HYDRA_APP_SOCKET_PATH"))

{udata_hydra-2.0.0.dev2730 → udata_hydra-2.0.0.dev2867}/udata_hydra/config_default.toml

@@ -12,6 +12,8 @@ TESTING = false
 MAX_POOL_SIZE = 50
 USER_AGENT = "udata-hydra/1.0"
 
+API_TOKEN = "to-change"
+
 # -- crawler settings -- #
 
 CATALOG_URL = "https://www.data.gouv.fr/fr/datasets/r/4babf5f2-6a9c-45b5-9144-ca5eae6a7a6d"
@@ -30,8 +32,8 @@ NO_BACKOFF_DOMAINS = [
 ]
 # max number of _completed_ requests per domain per period
 BACKOFF_NB_REQ = 180
-BACKOFF_PERIOD = 360  # in seconds
-COOL_OFF_PERIOD = 86400  # 1 day to cool off when we've messed up
+BACKOFF_PERIOD = 360    # in seconds
+COOL_OFF_PERIOD = 86400 # 1 day to cool off when we've messed up
 # ids of the resources that are too large to be processed normally
 # but that we want to have anyway
 LARGE_RESOURCES_EXCEPTIONS = [
@@ -50,9 +52,9 @@ SLEEP_BETWEEN_BATCHES = 60
 # max download filesize in bytes (100 MB)
 MAX_FILESIZE_ALLOWED.csv = 104857600
 MAX_FILESIZE_ALLOWED.csvgz = 104857600
-MAX_FILESIZE_ALLOWED.xls = 52428800 # /2
-MAX_FILESIZE_ALLOWED.xlsx = 13107200 # /8
-MAX_FILESIZE_ALLOWED.ods = 10485760 # /10
+MAX_FILESIZE_ALLOWED.xls = 52428800    # /2
+MAX_FILESIZE_ALLOWED.xlsx = 13107200   # /8
+MAX_FILESIZE_ALLOWED.ods = 10485760    # /10
 
 
 CSV_ANALYSIS = true

{udata_hydra-2.0.0.dev2730 → udata_hydra-2.0.0.dev2867}/udata_hydra/routes/resources.py

@@ -3,7 +3,6 @@ import json
 from aiohttp import web
 from marshmallow import ValidationError
 
-from udata_hydra import config
 from udata_hydra.db.resource import Resource
 from udata_hydra.schemas import ResourceSchema
 from udata_hydra.utils import get_request_params

{udata_hydra-2.0.0.dev2730 → udata_hydra-2.0.0.dev2867}/udata_hydra/utils/__init__.py

@@ -1,4 +1,5 @@
 from .app_version import get_app_version
+from .auth import token_auth_middleware
 from .csv import detect_tabular_from_headers
 from .file import compute_checksum_from_file, download_resource, read_csv_gz
 from .http import get_request_params, send

udata_hydra-2.0.0.dev2867/udata_hydra/utils/auth.py

@@ -0,0 +1,72 @@
+import re
+from typing import Coroutine, Tuple
+
+from aiohttp import web
+
+from udata_hydra import config
+
+
+def _is_exclude(request, exclude: Tuple[str]) -> bool:
+    for pattern in exclude:
+        if re.fullmatch(pattern, request.path):
+            return True
+    return False
+
+
+def token_auth_middleware(
+    request_property: str = "user",
+    auth_scheme: str = "Bearer",
+    exclude_routes: Tuple[str] = tuple(),
+    exclude_methods: Tuple[str] = tuple(),
+) -> Coroutine:
+    """Checks a auth token and adds a user in request.
+
+    Token auth middleware that checks the "Authorization" http header for token and, if the token in the requet headers is valid, then middleware adds the user to request with key that contain the "request_property" variable, else it will raise an HTTPForbiddenexception.
+
+    Args:
+        request_property (str, optional): Key for save in request object.
+            Defaults to 'user'.
+        auth_scheme (str, optional): Prefix for value in "Authorization" header.
+            Defaults to 'Bearer'.
+        exclude_routes: (Tuple, optional): Tuple of pathes that will be excluded.
+            Defaults to empty tuple.
+        exclude_methods(Tuple, optional): Tuple of http methods that will be
+            excluded. Defaults to empty tuple.
+
+    Raises:
+        web.HTTPUnauthorized: If "Authorization" token is missing.
+        web.HTTPForbidden: Wrong token, schema or header.
+
+    Returns:
+        Coroutine: Aiohttp middleware.
+    """
+
+    @web.middleware
+    async def middleware(request, handler):
+        if _is_exclude(request, exclude_routes) or request.method in exclude_methods:
+            return await handler(request)
+
+        try:
+            scheme, token = request.headers["Authorization"].strip().split(" ")
+        except KeyError:
+            raise web.HTTPUnauthorized(
+                reason="Missing authorization header",
+            )
+        except ValueError:
+            raise web.HTTPForbidden(
+                reason="Invalid authorization header",
+            )
+
+        if auth_scheme.lower() != scheme.lower():
+            raise web.HTTPForbidden(
+                reason="Invalid token scheme",
+            )
+
+        if token == config.API_TOKEN:
+            request[request_property] = {"username": "admin"}
+        else:
+            raise web.HTTPForbidden(reason="Invalid authentication token")
+
+        return await handler(request)
+
+    return middleware