udata-hydra 2.0.0.dev3017__tar.gz → 2.0.0.dev3084__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/PKG-INFO +1 -1
  2. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/pyproject.toml +1 -1
  3. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/__init__.py +15 -2
  4. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/config_default.toml +1 -1
  5. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/logger.py +1 -6
  6. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/__init__.py +0 -1
  7. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/auth.py +2 -2
  8. udata_hydra-2.0.0.dev3017/udata_hydra/utils/app_version.py +0 -17
  9. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/README.md +0 -0
  10. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/analysis/__init__.py +0 -0
  11. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/analysis/csv.py +0 -0
  12. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/analysis/errors.py +0 -0
  13. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/analysis/helpers.py +0 -0
  14. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/analysis/resource.py +0 -0
  15. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/app.py +0 -0
  16. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/cli.py +0 -0
  17. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/context.py +0 -0
  18. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/crawl.py +0 -0
  19. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/db/__init__.py +0 -0
  20. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/db/check.py +0 -0
  21. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/db/resource.py +0 -0
  22. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/__init__.py +0 -0
  23. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/csv/20221205_initial_up_rev1.sql +0 -0
  24. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/csv/20230130_drop_migrations.sql +0 -0
  25. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/csv/20230206_datetime_aware.sql +0 -0
  26. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20221205_initial_up_rev1.sql +0 -0
  27. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20221206_rev1_up_rev2.sql +0 -0
  28. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20221206_rev2_up_rev3.sql +0 -0
  29. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20221208_rev3_up_rev4.sql +0 -0
  30. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20221208_rev4_up_rev5.sql +0 -0
  31. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20230119_rev5_up_rev6.sql +0 -0
  32. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20230121_rev6_up_rev7.sql +0 -0
  33. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20230121_rev7_up_rev8.sql +0 -0
  34. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20230130_drop_migrations.sql +0 -0
  35. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20230206_datetime_aware.sql +0 -0
  36. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20230515_rev8_up_rev9.sql +0 -0
  37. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20230606_rev9_up_rev10.sql +0 -0
  38. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/migrations/main/20231102_drop_csv_analysis.sql +0 -0
  39. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/routes/__init__.py +0 -0
  40. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/routes/checks.py +0 -0
  41. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/routes/resources.py +0 -0
  42. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/routes/status.py +0 -0
  43. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/schemas/__init__.py +0 -0
  44. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/schemas/check.py +0 -0
  45. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/schemas/resource.py +0 -0
  46. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/csv.py +0 -0
  47. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/file.py +0 -0
  48. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/http.py +0 -0
  49. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/minio.py +0 -0
  50. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/parquet.py +0 -0
  51. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/queue.py +0 -0
  52. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/reader.py +0 -0
  53. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/utils/timer.py +0 -0
  54. {udata_hydra-2.0.0.dev3017 → udata_hydra-2.0.0.dev3084}/udata_hydra/worker.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: udata-hydra
3
- Version: 2.0.0.dev3017
3
+ Version: 2.0.0.dev3084
4
4
  Summary: Async crawler and parsing service for data.gouv.fr
5
5
  License: MIT
6
6
  Author: Opendata Team
@@ -9,7 +9,7 @@ readme = "README.md"
9
9
 
10
10
  [tool.poetry]
11
11
  name = "udata-hydra"
12
- version = "2.0.0.dev3017"
12
+ version = "2.0.0.dev3084"
13
13
  description = "Async crawler and parsing service for data.gouv.fr"
14
14
  authors = ["Opendata Team <opendatateam@data.gouv.fr>"]
15
15
  license = "MIT"
@@ -1,21 +1,25 @@
1
+ import logging
1
2
  import os
2
3
  from pathlib import Path
3
4
 
4
5
  import toml
5
6
 
7
+ log = logging.getLogger("udata-hydra")
8
+
6
9
 
7
10
  class Configurator:
8
11
  """Loads a dict of config from TOML file(s) and behaves like an object, ie config.VALUE"""
9
12
 
10
- configuration = None
13
+ configuration: dict = {}
11
14
 
12
15
  def __init__(self):
13
16
  if not self.configuration:
14
17
  self.configure()
18
+ self.load_pyproject_info()
15
19
 
16
20
  def configure(self) -> None:
17
21
  # load default settings
18
- configuration = toml.load(Path(__file__).parent / "config_default.toml")
22
+ configuration: dict = toml.load(Path(__file__).parent / "config_default.toml")
19
23
 
20
24
  # override with local settings
21
25
  local_settings = os.environ.get("HYDRA_SETTINGS", Path.cwd() / "config.toml")
@@ -25,6 +29,15 @@ class Configurator:
25
29
  self.configuration = configuration
26
30
  self.check()
27
31
 
32
+ def load_pyproject_info(self) -> None:
33
+ """Get more info about the app from pyproject.toml"""
34
+ try:
35
+ project_info: dict = toml.load("pyproject.toml")["project"]
36
+ except Exception as e:
37
+ log.error(f"Error while getting pyproject.toml info: {str(e)}")
38
+ self.configuration["APP_NAME"] = project_info.get("name")
39
+ self.configuration["APP_VERSION"] = project_info.get("version")
40
+
28
41
  def override(self, **kwargs) -> None:
29
42
  self.configuration.update(kwargs)
30
43
  self.check()
@@ -12,7 +12,7 @@ TESTING = false
12
12
  MAX_POOL_SIZE = 50
13
13
  USER_AGENT = "udata-hydra/1.0"
14
14
 
15
- API_TOKEN = "to-change"
15
+ API_KEY = "hydra_api_key_to_change"
16
16
 
17
17
  # -- crawler settings -- #
18
18
 
@@ -8,7 +8,6 @@ from sentry_sdk.integrations.aiohttp import AioHttpIntegration
8
8
  from sentry_sdk.integrations.rq import RqIntegration
9
9
 
10
10
  from udata_hydra import config
11
- from udata_hydra.utils import get_app_version
12
11
 
13
12
  log = logging.getLogger("udata-hydra")
14
13
  context = {"inited": False}
@@ -17,10 +16,6 @@ context = {"inited": False}
17
16
  def setup_logging() -> logging.Logger:
18
17
  if context.get("inited"):
19
18
  return log
20
- release = "hydra@unknown"
21
- app_version: Optional[str] = get_app_version()
22
- if app_version:
23
- release = f"hydra@{app_version}"
24
19
  if config.SENTRY_DSN:
25
20
  sentry_sdk.init(
26
21
  dsn=config.SENTRY_DSN,
@@ -28,7 +23,7 @@ def setup_logging() -> logging.Logger:
28
23
  AioHttpIntegration(),
29
24
  RqIntegration(),
30
25
  ],
31
- release=release,
26
+ release=f"{config.APP_NAME}@{config.APP_VERSION}",
32
27
  environment=os.getenv("HYDRA_ENV", "unknown"),
33
28
  # Set traces_sample_rate to 1.0 to capture 100%
34
29
  # of transactions for performance monitoring.
@@ -1,4 +1,3 @@
1
- from .app_version import get_app_version
2
1
  from .auth import token_auth_middleware
3
2
  from .csv import detect_tabular_from_headers
4
3
  from .file import compute_checksum_from_file, download_resource, read_csv_gz
@@ -62,8 +62,8 @@ def token_auth_middleware(
62
62
  reason="Invalid token scheme",
63
63
  )
64
64
 
65
- if token == config.API_TOKEN:
66
- request[request_property] = {"username": "admin"}
65
+ if token == config.API_KEY:
66
+ request[request_property] = {"username": "udata"}
67
67
  else:
68
68
  raise web.HTTPForbidden(reason="Invalid authentication token")
69
69
 
@@ -1,17 +0,0 @@
1
- import logging
2
- from typing import Optional
3
-
4
- import toml
5
-
6
- log = logging.getLogger("udata-hydra")
7
-
8
-
9
- def get_app_version() -> Optional[str]:
10
- """Get the app version from pyproject.toml"""
11
- try:
12
- pyproject: dict = toml.load("pyproject.toml")
13
- app_version: str = pyproject["tool"]["poetry"]["version"]
14
- return app_version
15
- except Exception as e:
16
- log.error(f"Error while getting app version: {e}")
17
- return None