udata-hydra 2.0.0.dev2064__tar.gz → 2.0.0.dev2124__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/PKG-INFO +2 -2
  2. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/pyproject.toml +2 -2
  3. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/config_default.toml +1 -0
  4. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/logger.py +15 -0
  5. udata_hydra-2.0.0.dev2124/udata_hydra/utils/app_version.py +17 -0
  6. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/README.md +0 -0
  7. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/__init__.py +0 -0
  8. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/analysis/__init__.py +0 -0
  9. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/analysis/csv.py +0 -0
  10. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/analysis/errors.py +0 -0
  11. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/analysis/helpers.py +0 -0
  12. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/analysis/resource.py +0 -0
  13. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/app.py +0 -0
  14. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/cli.py +0 -0
  15. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/context.py +0 -0
  16. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/crawl.py +0 -0
  17. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/migrations/__init__.py +0 -0
  18. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/migrations/csv/20221205_initial_up_rev1.sql +0 -0
  19. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/migrations/csv/20230130_drop_migrations.sql +0 -0
  20. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/migrations/csv/20230206_datetime_aware.sql +0 -0
  21. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/migrations/main/20221205_initial_up_rev1.sql +0 -0
  22. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/migrations/main/20221206_rev1_up_rev2.sql +0 -0
  23. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/migrations/main/20221206_rev2_up_rev3.sql +0 -0
  24. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/migrations/main/20221208_rev3_up_rev4.sql +0 -0
  25. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/migrations/main/20221208_rev4_up_rev5.sql +0 -0
  26. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/migrations/main/20230119_rev5_up_rev6.sql +0 -0
  27. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/migrations/main/20230121_rev6_up_rev7.sql +0 -0
  28. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/migrations/main/20230121_rev7_up_rev8.sql +0 -0
  29. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/migrations/main/20230130_drop_migrations.sql +0 -0
  30. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/migrations/main/20230206_datetime_aware.sql +0 -0
  31. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/migrations/main/20230515_rev8_up_rev9.sql +0 -0
  32. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/migrations/main/20230606_rev9_up_rev10.sql +0 -0
  33. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/migrations/main/20231102_drop_csv_analysis.sql +0 -0
  34. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/utils/__init__.py +0 -0
  35. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/utils/csv.py +0 -0
  36. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/utils/db.py +0 -0
  37. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/utils/file.py +0 -0
  38. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/utils/http.py +0 -0
  39. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/utils/json.py +0 -0
  40. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/utils/minio.py +0 -0
  41. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/utils/queue.py +0 -0
  42. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/utils/reader.py +0 -0
  43. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/utils/timer.py +0 -0
  44. {udata_hydra-2.0.0.dev2064 → udata_hydra-2.0.0.dev2124}/udata_hydra/worker.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: udata-hydra
3
- Version: 2.0.0.dev2064
3
+ Version: 2.0.0.dev2124
4
4
  Summary: Async crawler and parsing service for data.gouv.fr
5
5
  License: MIT
6
6
  Author: Opendata Team
@@ -27,7 +27,7 @@ Requires-Dist: python-dateutil (>=2.8.2,<3.0.0)
27
27
  Requires-Dist: python-magic (>=0.4.25,<0.5.0)
28
28
  Requires-Dist: redis (>=4.1.4,<5.0.0)
29
29
  Requires-Dist: rq (>=1.11.1,<2.0.0)
30
- Requires-Dist: sentry-sdk (>=1.11.1,<2.0.0)
30
+ Requires-Dist: sentry-sdk (>=2.10.0,<3.0.0)
31
31
  Requires-Dist: sqlalchemy (>=1.4.46,<2.0.0)
32
32
  Requires-Dist: str2bool (>=1.1,<2.0)
33
33
  Requires-Dist: str2float (>=0.0.9,<0.0.10)
@@ -9,7 +9,7 @@ readme = "README.md"
9
9
 
10
10
  [tool.poetry]
11
11
  name = "udata-hydra"
12
- version = "2.0.0.dev2064"
12
+ version = "2.0.0.dev2124"
13
13
  description = "Async crawler and parsing service for data.gouv.fr"
14
14
  authors = ["Opendata Team <opendatateam@data.gouv.fr>"]
15
15
  license = "MIT"
@@ -26,7 +26,7 @@ minicli = "^0.5.0"
26
26
  progressist = "^0.1.0"
27
27
  python-magic = "^0.4.25"
28
28
  redis = "^4.1.4"
29
- sentry-sdk = "^1.11.1"
29
+ sentry-sdk = "^2.10.0"
30
30
  aiocontextvars = "^0.2.2"
31
31
  coloredlogs = "^15.0.1"
32
32
  rq = "^1.11.1"
@@ -6,6 +6,7 @@ DATABASE_URL_CSV = "postgres://postgres:postgres@localhost:5434/postgres"
6
6
  DATABASE_SCHEMA = "public"
7
7
  REDIS_URL = "redis://localhost:6379/0"
8
8
  SENTRY_DSN = ""
9
+ SENTRY_SAMPLE_RATE = 1.0
9
10
  TESTING = false
10
11
  # max postgres pool size
11
12
  MAX_POOL_SIZE = 50
@@ -1,4 +1,6 @@
1
1
  import logging
2
+ import os
3
+ from typing import Union
2
4
 
3
5
  import coloredlogs
4
6
  import sentry_sdk
@@ -6,6 +8,7 @@ from sentry_sdk.integrations.aiohttp import AioHttpIntegration
6
8
  from sentry_sdk.integrations.rq import RqIntegration
7
9
 
8
10
  from udata_hydra import config
11
+ from udata_hydra.utils.app_version import get_app_version
9
12
 
10
13
  log = logging.getLogger("udata-hydra")
11
14
  context = {"inited": False}
@@ -14,6 +17,10 @@ context = {"inited": False}
14
17
  def setup_logging() -> logging.Logger:
15
18
  if context.get("inited"):
16
19
  return log
20
+ release = "hydra@unknown"
21
+ app_version: Union[str, None] = get_app_version()
22
+ if app_version:
23
+ release = f"hydra@{app_version}"
17
24
  if config.SENTRY_DSN:
18
25
  sentry_sdk.init(
19
26
  dsn=config.SENTRY_DSN,
@@ -21,7 +28,15 @@ def setup_logging() -> logging.Logger:
21
28
  AioHttpIntegration(),
22
29
  RqIntegration(),
23
30
  ],
31
+ release=release,
32
+ environment=os.getenv("HYDRA_ENV", "unknown"),
33
+ # Set traces_sample_rate to 1.0 to capture 100%
34
+ # of transactions for performance monitoring.
35
+ # Sentry recommends adjusting this value in production.
36
+ traces_sample_rate=config.SENTRY_SAMPLE_RATE,
37
+ profiles_sample_rate=config.SENTRY_SAMPLE_RATE,
24
38
  )
39
+
25
40
  coloredlogs.install(level=config.LOG_LEVEL)
26
41
  # silence urllib3 a bit
27
42
  logging.getLogger("urllib3").setLevel("INFO")
@@ -0,0 +1,17 @@
1
+ import logging
2
+ from typing import Union
3
+
4
+ import toml
5
+
6
+ log = logging.getLogger("udata-hydra")
7
+
8
+
9
+ def get_app_version() -> Union[str, None]:
10
+ """Get the app version from pyproject.toml"""
11
+ try:
12
+ pyproject: dict = toml.load("pyproject.toml")
13
+ app_version: str = pyproject["tool"]["poetry"]["version"]
14
+ return app_version
15
+ except Exception as e:
16
+ log.error(f"Error while getting app version: {e}")
17
+ return None