udata-hydra 2.4.1.dev9596__tar.gz → 2.4.2.dev9635__tar.gz

{udata_hydra-2.4.1.dev9596 → udata_hydra-2.4.2.dev9635}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: udata-hydra
-Version: 2.4.1.dev9596
+Version: 2.4.2.dev9635
 Summary: Async crawler and parsing service for data.gouv.fr
 License: MIT
 Author: Opendata Team

{udata_hydra-2.4.1.dev9596 → udata_hydra-2.4.2.dev9635}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "udata-hydra"
-version = "2.4.1.dev9596"
+version = "2.4.2.dev9635"
 description = "Async crawler and parsing service for data.gouv.fr"
 authors = [{ name = "Opendata Team", email = "opendatateam@data.gouv.fr" }]
 dependencies = [

{udata_hydra-2.4.1.dev9596 → udata_hydra-2.4.2.dev9635}/udata_hydra/__init__.py

@@ -49,5 +49,12 @@ class Configurator:
     def __dict__(self):
         return self.configuration
 
+    @property
+    def USER_AGENT_FULL(self) -> str:
+        """Build the complete user agent string with version"""
+        if self.USER_AGENT and self.APP_VERSION:
+            return f"{self.USER_AGENT}/{self.APP_VERSION}"
+        return "udata-hydra"
+
 
 config = Configurator()

{udata_hydra-2.4.1.dev9596 → udata_hydra-2.4.2.dev9635}/udata_hydra/config_default.toml

@@ -13,7 +13,7 @@ SENTRY_SAMPLE_RATE = 1.0
 TESTING = false
 # max postgres pool size
 MAX_POOL_SIZE = 50
-USER_AGENT = "udata-hydra/1.0"
+USER_AGENT = "udata-hydra" # without version - version is dynamically added
 NAMEDATALEN = 64  # should be set to the same value as in Postgres, but we can't query it
 
 API_KEY = "hydra_api_key_to_change"

{udata_hydra-2.4.1.dev9596 → udata_hydra-2.4.2.dev9635}/udata_hydra/crawl/check_resources.py

@@ -36,7 +36,8 @@ async def check_batch_resources(to_parse: list[Record]) -> None:
     context.monitor().set_status("Checking resources...")
     tasks: list = []
     async with aiohttp.ClientSession(
-        timeout=None, headers={"user-agent": config.USER_AGENT}
+        timeout=None,
+        headers={"user-agent": config.USER_AGENT_FULL},
     ) as session:
         for row in to_parse:
             tasks.append(

{udata_hydra-2.4.1.dev9596 → udata_hydra-2.4.2.dev9635}/udata_hydra/routes/checks.py

@@ -77,7 +77,8 @@ async def create_check(request: web.Request) -> web.Response:
     context.monitor().set_status(f'Crawling url "{url}"...')
 
     async with aiohttp.ClientSession(
-        timeout=None, headers={"user-agent": config.USER_AGENT}
+        timeout=None,
+        headers={"user-agent": config.USER_AGENT_FULL},
     ) as session:
         status: str = await check_resource(
             url=url,

{udata_hydra-2.4.1.dev9596 → udata_hydra-2.4.2.dev9635}/udata_hydra/utils/file.py

@@ -60,7 +60,7 @@ async def download_resource(
     too_large, download_error = False, None
     try:
         async with aiohttp.ClientSession(
-            headers={"user-agent": config.USER_AGENT} if config.USER_AGENT else None,
+            headers={"user-agent": config.USER_AGENT_FULL},
             raise_for_status=True,
         ) as session:
             async with session.get(url, allow_redirects=True) as response:

{udata_hydra-2.4.1.dev9596 → udata_hydra-2.4.2.dev9635}/udata_hydra/utils/http.py

@@ -79,6 +79,7 @@ async def send(dataset_id: str, resource_id: str, document: UdataPayload) -> Non
 
     uri = f"{config.UDATA_URI}/datasets/{dataset_id}/resources/{resource_id}/extras/"
     headers = {
+        "user-agent": config.USER_AGENT_FULL,
         "content-type": "application/json",
         "X-API-KEY": config.UDATA_URI_API_KEY,
     }