udata-hydra 2.0.0.dev3017__tar.gz → 2.0.0.dev3084__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/PKG-INFO +1 -1
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/pyproject.toml +1 -1
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/__init__.py +15 -2
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/config_default.toml +1 -1
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/logger.py +1 -6
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/__init__.py +0 -1
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/auth.py +2 -2
- udata_hydra-2.0.0.dev3017/udata_hydra/utils/app_version.py +0 -17
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/README.md +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/analysis/__init__.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/analysis/csv.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/analysis/errors.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/analysis/helpers.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/analysis/resource.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/app.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/cli.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/context.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/crawl.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/db/__init__.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/db/check.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/db/resource.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/__init__.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/csv/20221205_initial_up_rev1.sql +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/csv/20230130_drop_migrations.sql +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/csv/20230206_datetime_aware.sql +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20221205_initial_up_rev1.sql +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20221206_rev1_up_rev2.sql +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20221206_rev2_up_rev3.sql +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20221208_rev3_up_rev4.sql +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20221208_rev4_up_rev5.sql +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20230119_rev5_up_rev6.sql +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20230121_rev6_up_rev7.sql +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20230121_rev7_up_rev8.sql +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20230130_drop_migrations.sql +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20230206_datetime_aware.sql +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20230515_rev8_up_rev9.sql +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20230606_rev9_up_rev10.sql +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20231102_drop_csv_analysis.sql +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/routes/__init__.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/routes/checks.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/routes/resources.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/routes/status.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/schemas/__init__.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/schemas/check.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/schemas/resource.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/csv.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/file.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/http.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/minio.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/parquet.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/queue.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/reader.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/timer.py +0 -0
- {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/worker.py +0 -0
|
@@ -9,7 +9,7 @@ readme = "README.md"
|
|
|
9
9
|
|
|
10
10
|
[tool.poetry]
|
|
11
11
|
name = "udata-hydra"
|
|
12
|
-
version = "2.0.0.
|
|
12
|
+
version = "2.0.0.dev3084"
|
|
13
13
|
description = "Async crawler and parsing service for data.gouv.fr"
|
|
14
14
|
authors = ["Opendata Team <opendatateam@data.gouv.fr>"]
|
|
15
15
|
license = "MIT"
|
|
@@ -1,21 +1,25 @@
|
|
|
1
|
+
import logging
|
|
1
2
|
import os
|
|
2
3
|
from pathlib import Path
|
|
3
4
|
|
|
4
5
|
import toml
|
|
5
6
|
|
|
7
|
+
log = logging.getLogger("udata-hydra")
|
|
8
|
+
|
|
6
9
|
|
|
7
10
|
class Configurator:
|
|
8
11
|
"""Loads a dict of config from TOML file(s) and behaves like an object, ie config.VALUE"""
|
|
9
12
|
|
|
10
|
-
configuration =
|
|
13
|
+
configuration: dict = {}
|
|
11
14
|
|
|
12
15
|
def __init__(self):
|
|
13
16
|
if not self.configuration:
|
|
14
17
|
self.configure()
|
|
18
|
+
self.load_pyproject_info()
|
|
15
19
|
|
|
16
20
|
def configure(self) -> None:
|
|
17
21
|
# load default settings
|
|
18
|
-
configuration = toml.load(Path(__file__).parent / "config_default.toml")
|
|
22
|
+
configuration: dict = toml.load(Path(__file__).parent / "config_default.toml")
|
|
19
23
|
|
|
20
24
|
# override with local settings
|
|
21
25
|
local_settings = os.environ.get("HYDRA_SETTINGS", Path.cwd() / "config.toml")
|
|
@@ -25,6 +29,15 @@ class Configurator:
|
|
|
25
29
|
self.configuration = configuration
|
|
26
30
|
self.check()
|
|
27
31
|
|
|
32
|
+
def load_pyproject_info(self) -> None:
|
|
33
|
+
"""Get more info about the app from pyproject.toml"""
|
|
34
|
+
try:
|
|
35
|
+
project_info: dict = toml.load("pyproject.toml")["project"]
|
|
36
|
+
except Exception as e:
|
|
37
|
+
log.error(f"Error while getting pyproject.toml info: {str(e)}")
|
|
38
|
+
self.configuration["APP_NAME"] = project_info.get("name")
|
|
39
|
+
self.configuration["APP_VERSION"] = project_info.get("version")
|
|
40
|
+
|
|
28
41
|
def override(self, **kwargs) -> None:
|
|
29
42
|
self.configuration.update(kwargs)
|
|
30
43
|
self.check()
|
|
@@ -8,7 +8,6 @@ from sentry_sdk.integrations.aiohttp import AioHttpIntegration
|
|
|
8
8
|
from sentry_sdk.integrations.rq import RqIntegration
|
|
9
9
|
|
|
10
10
|
from udata_hydra import config
|
|
11
|
-
from udata_hydra.utils import get_app_version
|
|
12
11
|
|
|
13
12
|
log = logging.getLogger("udata-hydra")
|
|
14
13
|
context = {"inited": False}
|
|
@@ -17,10 +16,6 @@ context = {"inited": False}
|
|
|
17
16
|
def setup_logging() -> logging.Logger:
|
|
18
17
|
if context.get("inited"):
|
|
19
18
|
return log
|
|
20
|
-
release = "hydra@unknown"
|
|
21
|
-
app_version: Optional[str] = get_app_version()
|
|
22
|
-
if app_version:
|
|
23
|
-
release = f"hydra@{app_version}"
|
|
24
19
|
if config.SENTRY_DSN:
|
|
25
20
|
sentry_sdk.init(
|
|
26
21
|
dsn=config.SENTRY_DSN,
|
|
@@ -28,7 +23,7 @@ def setup_logging() -> logging.Logger:
|
|
|
28
23
|
AioHttpIntegration(),
|
|
29
24
|
RqIntegration(),
|
|
30
25
|
],
|
|
31
|
-
release=
|
|
26
|
+
release=f"{config.APP_NAME}@{config.APP_VERSION}",
|
|
32
27
|
environment=os.getenv("HYDRA_ENV", "unknown"),
|
|
33
28
|
# Set traces_sample_rate to 1.0 to capture 100%
|
|
34
29
|
# of transactions for performance monitoring.
|
|
@@ -62,8 +62,8 @@ def token_auth_middleware(
|
|
|
62
62
|
reason="Invalid token scheme",
|
|
63
63
|
)
|
|
64
64
|
|
|
65
|
-
if token == config.
|
|
66
|
-
request[request_property] = {"username": "
|
|
65
|
+
if token == config.API_KEY:
|
|
66
|
+
request[request_property] = {"username": "udata"}
|
|
67
67
|
else:
|
|
68
68
|
raise web.HTTPForbidden(reason="Invalid authentication token")
|
|
69
69
|
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
from typing import Optional
|
|
3
|
-
|
|
4
|
-
import toml
|
|
5
|
-
|
|
6
|
-
log = logging.getLogger("udata-hydra")
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
def get_app_version() -> Optional[str]:
|
|
10
|
-
"""Get the app version from pyproject.toml"""
|
|
11
|
-
try:
|
|
12
|
-
pyproject: dict = toml.load("pyproject.toml")
|
|
13
|
-
app_version: str = pyproject["tool"]["poetry"]["version"]
|
|
14
|
-
return app_version
|
|
15
|
-
except Exception as e:
|
|
16
|
-
log.error(f"Error while getting app version: {e}")
|
|
17
|
-
return None
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|