PyPI - udata-hydra - Versions diffs - 2.0.0.dev2400__tar.gz → 2.0.0.dev2497__tar.gz - Mend

@@ -30,7 +30,7 @@ class Check:
         async with pool.acquire() as connection:
             q = f"""
             SELECT catalog.id as catalog_id, checks.id as check_id,
-                catalog.status as catalog_status, checks.status as check_status, *
+                catalog.status as catalog_status, checks.status as check_status, catalog.deleted as deleted, *
             FROM checks, catalog
             WHERE checks.id = catalog.last_check
             AND catalog.{column} = $1
@@ -44,7 +44,7 @@ class Check:
         async with pool.acquire() as connection:
             q = f"""
             SELECT catalog.id as catalog_id, checks.id as check_id,
-                catalog.status as catalog_status, checks.status as check_status, *
+                catalog.status as catalog_status, checks.status as check_status, catalog.deleted as deleted, *
             FROM checks, catalog
             WHERE catalog.{column} = $1
             AND catalog.url = checks.url
@@ -71,3 +71,10 @@ class Check:
     async def update(cls, check_id: int, data: dict) -> int:
         """Update a check in DB with new data and return the check id in DB"""
         return await update_table_record(table_name="checks", record_id=check_id, data=data)
+    @classmethod
+    async def delete(cls, check_id: int) -> int:
+        pool = await context.pool()
+        async with pool.acquire() as connection:
+            q = """DELETE FROM checks WHERE id = $1"""
+            return await connection.fetch(q, check_id)

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: udata-hydra
-Version: 2.0.0.dev2400
+Version: 2.0.0.dev2497
 Summary: Async crawler and parsing service for data.gouv.fr
 License: MIT
 Author: Opendata Team
@@ -374,7 +374,7 @@ Once this is done, code formatting and linting, as well as import sorting, will
 If you cannot use pre-commit, it is necessary to format, lint, and sort imports with [Ruff](https://docs.astral.sh/ruff/) before committing:
 ```bash
-ruff check --fix --select I .
+ruff check --fix .
 ruff format .
 ```

@@ -338,6 +338,6 @@ Once this is done, code formatting and linting, as well as import sorting, will
 If you cannot use pre-commit, it is necessary to format, lint, and sort imports with [Ruff](https://docs.astral.sh/ruff/) before committing:
 ```bash
-ruff check --fix --select I .
+ruff check --fix .
 ruff format .
 ```

@@ -9,7 +9,7 @@ readme = "README.md"
 [tool.poetry]
 name = "udata-hydra"
-version = "2.0.0.dev2400"
+version = "2.0.0.dev2497"
 description = "Async crawler and parsing service for data.gouv.fr"
 authors = ["Opendata Team <opendatateam@data.gouv.fr>"]
 license = "MIT"
@@ -76,7 +76,7 @@ markers = [
 addopts = "--cov=udata_hydra/ --cov-report term-missing"
 [tool.ruff]
-lint = { extend-select = ["I"] } # also sort imports with an isort rule
+lint = { extend-select = ["I"], ignore = ["F401"] } # ["I"] is to also sort imports with an isort rule # TODO: remove ignore F401 later
 line-length = 100
 [build-system]

@@ -132,6 +132,9 @@ async def process_check_data(check_data: dict) -> Tuple[int, bool]:
 async def is_backoff(domain: str) -> Tuple[bool, str]:
+    """Check if we should not crawl on this domain
+    Returns a tuple with if it should backoff or not (boolean) and the reason why (string)
+    """
     backoff = False, ""
     no_backoff = [f"'{d}'" for d in config.NO_BACKOFF_DOMAINS]
     no_backoff = f"({','.join(no_backoff)})"
@@ -201,7 +204,7 @@ def fix_surrogates(value):
     """FIX Unicode low surrogate must follow a high surrogate.
     eg in 'TREMI_2017-R\xe9sultats enqu\xeate bruts.csv'
     """
-    if not type(value) == str:
+    if type(value) is not str:
         value = str(value)
     return value.encode("utf-8", "surrogateescape").decode("utf-8", "replace")

@@ -1,6 +1,5 @@
 import json
-from aiohttp import web
 from marshmallow import Schema, fields

udata-hydra 2.0.0.dev2400__tar.gz → 2.0.0.dev2497__tar.gz

udata-hydra 2.0.0.dev2400tar.gz → 2.0.0.dev2497tar.gz