udata-hydra 2.0.0.dev1953__tar.gz → 2.0.0.dev2045__tar.gz

{udata_hydra-2.0.0.dev1953 → udata_hydra-2.0.0.dev2045}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata-hydra
-Version: 2.0.0.dev1953
+Version: 2.0.0.dev2045
 Summary: Async crawler and parsing service for data.gouv.fr
 License: MIT
 Author: Opendata Team

{udata_hydra-2.0.0.dev1953 → udata_hydra-2.0.0.dev2045}/pyproject.toml

@@ -9,7 +9,7 @@ readme = "README.md"
 
 [tool.poetry]
 name = "udata-hydra"
-version = "2.0.0.dev1953"
+version = "2.0.0.dev2045"
 description = "Async crawler and parsing service for data.gouv.fr"
 authors = ["Opendata Team <opendatateam@data.gouv.fr>"]
 license = "MIT"
@@ -59,7 +59,6 @@ module = [
     "csv_detective.*",
     "humanfriendly.*",
     "minicli.*",
-    "pytz.*",
     "sqlalchemy.*",
     "str2bool.*",
     "str2float.*",

{udata_hydra-2.0.0.dev1953 → udata_hydra-2.0.0.dev2045}/udata_hydra/analysis/csv.py

@@ -4,10 +4,9 @@ import json
 import logging
 import os
 import sys
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Any, Union
 
-import pytz
 import sentry_sdk
 from csv_detective.detection import engine_to_file
 from csv_detective.explore_csv import routine as csv_detective_routine
@@ -132,7 +131,7 @@ async def analyse_csv(
 
     try:
         if check_id:
-            await update_check(check_id, {"parsing_started_at": datetime.now(pytz.UTC)})
+            await update_check(check_id, {"parsing_started_at": datetime.now(timezone.utc)})
         csv_inspection = await perform_csv_inspection(tmp_file.name)
         timer.mark("csv-inspection")
         await csv_to_db(tmp_file.name, csv_inspection, table_name, debug_insert=debug_insert)
@@ -142,7 +141,7 @@ async def analyse_csv(
                 check_id,
                 {
                     "parsing_table": table_name,
-                    "parsing_finished_at": datetime.now(pytz.UTC),
+                    "parsing_finished_at": datetime.now(timezone.utc),
                 },
             )
         await csv_to_db_index(table_name, csv_inspection, check)
@@ -287,7 +286,7 @@ async def handle_parse_exception(e: Exception, check_id: int, table_name: str) -
         err = f"{e.step}:sentry:{event_id}" if config.SENTRY_DSN else f"{e.step}:{str(e.__cause__)}"
         await update_check(
             check_id,
-            {"parsing_error": err, "parsing_finished_at": datetime.now(pytz.UTC)},
+            {"parsing_error": err, "parsing_finished_at": datetime.now(timezone.utc)},
         )
         log.error("Parsing error", exc_info=e)
     else:

{udata_hydra-2.0.0.dev1953 → udata_hydra-2.0.0.dev2045}/udata_hydra/analysis/resource.py

@@ -1,12 +1,11 @@
 import json
 import logging
 import os
-from datetime import datetime
+from datetime import datetime, timezone
 from enum import Enum
 from typing import Tuple, Union
 
 import magic
-import pytz
 from dateparser import parse as date_parser
 
 from udata_hydra import config, context
@@ -151,7 +150,7 @@ async def detect_resource_change_from_checksum(
         data = await connection.fetchrow(q, resource_id)
         if data and data["checksum"] != new_checksum:
             return Change.HAS_CHANGED, {
-                "analysis:last-modified-at": datetime.now(pytz.UTC).isoformat(),
+                "analysis:last-modified-at": datetime.now(timezone.utc).isoformat(),
                 "analysis:last-modified-detection": "computed-checksum",
             }
     return Change.NO_GUESS, None

{udata_hydra-2.0.0.dev1953 → udata_hydra-2.0.0.dev2045}/udata_hydra/app.py

@@ -1,8 +1,7 @@
 import json
 import os
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
-import pytz
 from aiohttp import web
 from humanfriendly import parse_timespan
 from marshmallow import Schema, ValidationError, fields
@@ -208,7 +207,7 @@ async def status_crawler(request: web.Request) -> web.Response:
     stats_catalog = await request.app["pool"].fetchrow(q)
 
     since = parse_timespan(config.SINCE)
-    since = datetime.now(pytz.UTC) - timedelta(seconds=since)
+    since = datetime.now(timezone.utc) - timedelta(seconds=since)
     q = f"""
         SELECT
             SUM(CASE WHEN checks.created_at <= $1 THEN 1 ELSE 0 END) AS count_outdated

{udata_hydra-2.0.0.dev1953 → udata_hydra-2.0.0.dev2045}/udata_hydra/crawl.py

@@ -7,7 +7,6 @@ from typing import Tuple, Union
 from urllib.parse import urlparse
 
 import aiohttp
-import pytz
 from humanfriendly import parse_timespan
 
 from udata_hydra import config, context
@@ -86,7 +85,7 @@ async def compute_check_has_changed(check_data, last_check) -> bool:
             "check:available": is_valid_status(check_data.get("status")),
             "check:status": check_data.get("status"),
             "check:timeout": check_data["timeout"],
-            "check:date": datetime.now(pytz.UTC).isoformat(),
+            "check:date": datetime.now(timezone.utc).isoformat(),
             "check:error": check_data.get("error"),
             "check:headers:content-type": await get_content_type_from_header(
                 check_data.get("headers", {})
@@ -413,7 +412,7 @@ async def crawl_batch():
         # if not enough for our batch size, handle outdated checks
         if len(to_check) < config.BATCH_SIZE:
             since = parse_timespan(config.SINCE)  # in seconds
-            since = datetime.now(pytz.UTC) - timedelta(seconds=since)
+            since = datetime.now(timezone.utc) - timedelta(seconds=since)
             limit = config.BATCH_SIZE - len(to_check)
             q = f"""
             SELECT * FROM (