udata-hydra 2.0.0.dev1926__tar.gz → 2.0.0.dev2045__tar.gz

{udata_hydra-2.0.0.dev1926 → udata_hydra-2.0.0.dev2045}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata-hydra
-Version: 2.0.0.dev1926
+Version: 2.0.0.dev2045
 Summary: Async crawler and parsing service for data.gouv.fr
 License: MIT
 Author: Opendata Team
@@ -52,7 +52,7 @@ The architecture for the full workflow is the following:
 ![Full workflow architecture](docs/archi-idd-IDD.drawio.png)
 
 
-The hydra crawler is one of the components of the architecture. It will check if resource is available, analyze the type of file if the resource has been modified, and analyze the CSV content. It will also convert CSV resources to database tables and send the data to a udata instance. 
+The hydra crawler is one of the components of the architecture. It will check if resource is available, analyze the type of file if the resource has been modified, and analyze the CSV content. It will also convert CSV resources to database tables and send the data to a udata instance.
 
 ![Crawler architecture](docs/hydra.drawio.png)
 
@@ -97,6 +97,15 @@ Monitor worker status:
 
 Converted CSV tables will be stored in the database specified via `config.DATABASE_URL_CSV`. For tests it's same database as for the catalog. Locally, `docker compose` will launch two distinct database containers.
 
+## Tests
+
+To run the tests, you need to launch the database, the test database, and the Redis broker with `docker-compose -f docker-compose.yml -f docker-compose.test.yml up -f docker-compose.broker.yml up -d`.
+
+Then you can run the tests with `poetry run pytest`.
+
+If you would like to see print statements as they are executed, you can pass the -s flag to pytest (`poetry run pytest -s`). However, note that this can sometimes be difficult to parse.
+
+
 ## API
 
 ### Run

{udata_hydra-2.0.0.dev1926 → udata_hydra-2.0.0.dev2045}/README.md

@@ -16,7 +16,7 @@ The architecture for the full workflow is the following:
 ![Full workflow architecture](docs/archi-idd-IDD.drawio.png)
 
 
-The hydra crawler is one of the components of the architecture. It will check if resource is available, analyze the type of file if the resource has been modified, and analyze the CSV content. It will also convert CSV resources to database tables and send the data to a udata instance. 
+The hydra crawler is one of the components of the architecture. It will check if resource is available, analyze the type of file if the resource has been modified, and analyze the CSV content. It will also convert CSV resources to database tables and send the data to a udata instance.
 
 ![Crawler architecture](docs/hydra.drawio.png)
 
@@ -61,6 +61,15 @@ Monitor worker status:
 
 Converted CSV tables will be stored in the database specified via `config.DATABASE_URL_CSV`. For tests it's same database as for the catalog. Locally, `docker compose` will launch two distinct database containers.
 
+## Tests
+
+To run the tests, you need to launch the database, the test database, and the Redis broker with `docker-compose -f docker-compose.yml -f docker-compose.test.yml up -f docker-compose.broker.yml up -d`.
+
+Then you can run the tests with `poetry run pytest`.
+
+If you would like to see print statements as they are executed, you can pass the -s flag to pytest (`poetry run pytest -s`). However, note that this can sometimes be difficult to parse.
+
+
 ## API
 
 ### Run

{udata_hydra-2.0.0.dev1926 → udata_hydra-2.0.0.dev2045}/pyproject.toml

@@ -9,7 +9,7 @@ readme = "README.md"
 
 [tool.poetry]
 name = "udata-hydra"
-version = "2.0.0.dev1926"
+version = "2.0.0.dev2045"
 description = "Async crawler and parsing service for data.gouv.fr"
 authors = ["Opendata Team <opendatateam@data.gouv.fr>"]
 license = "MIT"
@@ -59,7 +59,6 @@ module = [
     "csv_detective.*",
     "humanfriendly.*",
     "minicli.*",
-    "pytz.*",
     "sqlalchemy.*",
     "str2bool.*",
     "str2float.*",

{udata_hydra-2.0.0.dev1926 → udata_hydra-2.0.0.dev2045}/udata_hydra/analysis/csv.py

@@ -4,10 +4,9 @@ import json
 import logging
 import os
 import sys
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Any, Union
 
-import pytz
 import sentry_sdk
 from csv_detective.detection import engine_to_file
 from csv_detective.explore_csv import routine as csv_detective_routine
@@ -132,7 +131,7 @@ async def analyse_csv(
 
     try:
         if check_id:
-            await update_check(check_id, {"parsing_started_at": datetime.now(pytz.UTC)})
+            await update_check(check_id, {"parsing_started_at": datetime.now(timezone.utc)})
         csv_inspection = await perform_csv_inspection(tmp_file.name)
         timer.mark("csv-inspection")
         await csv_to_db(tmp_file.name, csv_inspection, table_name, debug_insert=debug_insert)
@@ -142,7 +141,7 @@ async def analyse_csv(
                 check_id,
                 {
                     "parsing_table": table_name,
-                    "parsing_finished_at": datetime.now(pytz.UTC),
+                    "parsing_finished_at": datetime.now(timezone.utc),
                 },
             )
         await csv_to_db_index(table_name, csv_inspection, check)
@@ -287,7 +286,7 @@ async def handle_parse_exception(e: Exception, check_id: int, table_name: str) -
         err = f"{e.step}:sentry:{event_id}" if config.SENTRY_DSN else f"{e.step}:{str(e.__cause__)}"
         await update_check(
             check_id,
-            {"parsing_error": err, "parsing_finished_at": datetime.now(pytz.UTC)},
+            {"parsing_error": err, "parsing_finished_at": datetime.now(timezone.utc)},
         )
         log.error("Parsing error", exc_info=e)
     else:

{udata_hydra-2.0.0.dev1926 → udata_hydra-2.0.0.dev2045}/udata_hydra/analysis/resource.py

@@ -1,12 +1,11 @@
 import json
 import logging
 import os
-from datetime import datetime
+from datetime import datetime, timezone
 from enum import Enum
 from typing import Tuple, Union
 
 import magic
-import pytz
 from dateparser import parse as date_parser
 
 from udata_hydra import config, context
@@ -151,7 +150,7 @@ async def detect_resource_change_from_checksum(
         data = await connection.fetchrow(q, resource_id)
         if data and data["checksum"] != new_checksum:
             return Change.HAS_CHANGED, {
-                "analysis:last-modified-at": datetime.now(pytz.UTC).isoformat(),
+                "analysis:last-modified-at": datetime.now(timezone.utc).isoformat(),
                 "analysis:last-modified-detection": "computed-checksum",
             }
     return Change.NO_GUESS, None

{udata_hydra-2.0.0.dev1926 → udata_hydra-2.0.0.dev2045}/udata_hydra/app.py

@@ -1,8 +1,7 @@
 import json
 import os
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
-import pytz
 from aiohttp import web
 from humanfriendly import parse_timespan
 from marshmallow import Schema, ValidationError, fields
@@ -208,7 +207,7 @@ async def status_crawler(request: web.Request) -> web.Response:
     stats_catalog = await request.app["pool"].fetchrow(q)
 
     since = parse_timespan(config.SINCE)
-    since = datetime.now(pytz.UTC) - timedelta(seconds=since)
+    since = datetime.now(timezone.utc) - timedelta(seconds=since)
     q = f"""
         SELECT
             SUM(CASE WHEN checks.created_at <= $1 THEN 1 ELSE 0 END) AS count_outdated

{udata_hydra-2.0.0.dev1926 → udata_hydra-2.0.0.dev2045}/udata_hydra/crawl.py

@@ -7,7 +7,6 @@ from typing import Tuple, Union
 from urllib.parse import urlparse
 
 import aiohttp
-import pytz
 from humanfriendly import parse_timespan
 
 from udata_hydra import config, context
@@ -86,7 +85,7 @@ async def compute_check_has_changed(check_data, last_check) -> bool:
             "check:available": is_valid_status(check_data.get("status")),
             "check:status": check_data.get("status"),
             "check:timeout": check_data["timeout"],
-            "check:date": datetime.now(pytz.UTC).isoformat(),
+            "check:date": datetime.now(timezone.utc).isoformat(),
             "check:error": check_data.get("error"),
             "check:headers:content-type": await get_content_type_from_header(
                 check_data.get("headers", {})
@@ -413,7 +412,7 @@ async def crawl_batch():
         # if not enough for our batch size, handle outdated checks
         if len(to_check) < config.BATCH_SIZE:
             since = parse_timespan(config.SINCE)  # in seconds
-            since = datetime.now(pytz.UTC) - timedelta(seconds=since)
+            since = datetime.now(timezone.utc) - timedelta(seconds=since)
             limit = config.BATCH_SIZE - len(to_check)
             q = f"""
             SELECT * FROM (