udata-hydra 2.2.2.dev7569__tar.gz → 2.2.2.dev7591__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/PKG-INFO +2 -3
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/README.md +1 -2
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/pyproject.toml +2 -2
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/routes/resources_exceptions.py +0 -1
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/schemas/__init__.py +8 -1
- udata_hydra-2.2.2.dev7591/udata_hydra/utils/__init__.py +26 -0
- udata_hydra-2.2.2.dev7569/udata_hydra/utils/__init__.py +0 -10
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/__init__.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/analysis/__init__.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/analysis/csv.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/analysis/geojson.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/analysis/helpers.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/analysis/resource.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/app.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/cli.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/config_default.toml +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/context.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/crawl/__init__.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/crawl/calculate_next_check.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/crawl/check_resources.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/crawl/helpers.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/crawl/preprocess_check_data.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/crawl/select_batch.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/db/__init__.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/db/check.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/db/resource.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/db/resource_exception.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/logger.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/__init__.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/csv/20221205_initial_up_rev1.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/csv/20230130_drop_migrations.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/csv/20230206_datetime_aware.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/csv/20240827_add_indexes_column_to_tables_index_table.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/csv/20250610_migrate_resources_exception.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/csv/20250626_delete_datetime_iso_references.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20221205_initial_up_rev1.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20221206_rev1_up_rev2.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20221206_rev2_up_rev3.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20221208_rev3_up_rev4.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20221208_rev4_up_rev5.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20230119_rev5_up_rev6.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20230121_rev6_up_rev7.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20230121_rev7_up_rev8.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20230130_drop_migrations.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20230206_datetime_aware.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20230515_rev8_up_rev9.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20230606_rev9_up_rev10.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20231102_drop_csv_analysis.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20240827_add_resources_exceptions_table.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20240926_add_indexes.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20241004_add_comment_column_to_resources_exceptions.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20241021_add_parquet_columns.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20241023_alter_foreign_key.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20241025_add_next_check_column.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20250108_add_indexes.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20250130_add_pmtiles_fields.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20250519_add_format_column_catalog.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20250610_migrate_resources_exception.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/migrations/main/20250611_add_status_since_catalog.sql +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/routes/__init__.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/routes/checks.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/routes/resources.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/routes/status.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/schemas/check.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/schemas/resource.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/schemas/resource_exception.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/utils/auth.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/utils/csv.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/utils/db.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/utils/errors.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/utils/file.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/utils/geojson.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/utils/http.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/utils/minio.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/utils/parquet.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/utils/queue.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/utils/reader.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/utils/timer.py +0 -0
- {udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/worker.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: udata-hydra
|
|
3
|
-
Version: 2.2.2.
|
|
3
|
+
Version: 2.2.2.dev7591
|
|
4
4
|
Summary: Async crawler and parsing service for data.gouv.fr
|
|
5
5
|
License: MIT
|
|
6
6
|
Author: Opendata Team
|
|
@@ -529,8 +529,7 @@ Once this is done, code formatting and linting, as well as import sorting, will
|
|
|
529
529
|
|
|
530
530
|
If you cannot use pre-commit, it is necessary to format, lint, and sort imports with [Ruff](https://docs.astral.sh/ruff/) before committing:
|
|
531
531
|
```bash
|
|
532
|
-
ruff check --fix .
|
|
533
|
-
ruff format .
|
|
532
|
+
poetry run ruff check --fix . && poetry run ruff format .
|
|
534
533
|
```
|
|
535
534
|
|
|
536
535
|
### Releases
|
|
@@ -479,8 +479,7 @@ Once this is done, code formatting and linting, as well as import sorting, will
|
|
|
479
479
|
|
|
480
480
|
If you cannot use pre-commit, it is necessary to format, lint, and sort imports with [Ruff](https://docs.astral.sh/ruff/) before committing:
|
|
481
481
|
```bash
|
|
482
|
-
ruff check --fix .
|
|
483
|
-
ruff format .
|
|
482
|
+
poetry run ruff check --fix . && poetry run ruff format .
|
|
484
483
|
```
|
|
485
484
|
|
|
486
485
|
### Releases
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "udata-hydra"
|
|
3
|
-
version = "2.2.2.
|
|
3
|
+
version = "2.2.2.dev7591"
|
|
4
4
|
description = "Async crawler and parsing service for data.gouv.fr"
|
|
5
5
|
authors = [{ name = "Opendata Team", email = "opendatateam@data.gouv.fr" }]
|
|
6
6
|
dependencies = [
|
|
@@ -79,7 +79,7 @@ markers = [
|
|
|
79
79
|
addopts = "--cov=udata_hydra/ --cov-report term-missing"
|
|
80
80
|
|
|
81
81
|
[tool.ruff]
|
|
82
|
-
lint = { extend-select = ["I"]
|
|
82
|
+
lint = { extend-select = ["I"] } # ["I"] is to also sort imports with an isort rule
|
|
83
83
|
line-length = 100
|
|
84
84
|
|
|
85
85
|
[build-system]
|
{udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/routes/resources_exceptions.py
RENAMED
|
@@ -5,7 +5,6 @@ from aiohttp import web
|
|
|
5
5
|
from asyncpg import Record
|
|
6
6
|
from asyncpg.exceptions import UniqueViolationError
|
|
7
7
|
|
|
8
|
-
from udata_hydra import config
|
|
9
8
|
from udata_hydra.db.resource import Resource
|
|
10
9
|
from udata_hydra.db.resource_exception import ResourceException
|
|
11
10
|
from udata_hydra.schemas import ResourceExceptionSchema
|
|
@@ -1,4 +1,11 @@
|
|
|
1
|
-
# ruff: noqa: F401
|
|
2
1
|
from .check import CheckGroupBy, CheckSchema
|
|
3
2
|
from .resource import ResourceDocumentSchema, ResourceSchema
|
|
4
3
|
from .resource_exception import ResourceExceptionSchema
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"CheckGroupBy",
|
|
7
|
+
"CheckSchema",
|
|
8
|
+
"ResourceDocumentSchema",
|
|
9
|
+
"ResourceSchema",
|
|
10
|
+
"ResourceExceptionSchema",
|
|
11
|
+
]
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from .auth import token_auth_middleware
|
|
2
|
+
from .csv import detect_tabular_from_headers
|
|
3
|
+
from .errors import IOException, ParseException, handle_parse_exception
|
|
4
|
+
from .file import compute_checksum_from_file, download_resource
|
|
5
|
+
from .geojson import detect_geojson_from_headers_or_catalog
|
|
6
|
+
from .http import UdataPayload, get_request_params, send
|
|
7
|
+
from .queue import enqueue
|
|
8
|
+
from .reader import Reader
|
|
9
|
+
from .timer import Timer
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"token_auth_middleware",
|
|
13
|
+
"detect_tabular_from_headers",
|
|
14
|
+
"IOException",
|
|
15
|
+
"ParseException",
|
|
16
|
+
"handle_parse_exception",
|
|
17
|
+
"compute_checksum_from_file",
|
|
18
|
+
"download_resource",
|
|
19
|
+
"detect_geojson_from_headers_or_catalog",
|
|
20
|
+
"UdataPayload",
|
|
21
|
+
"get_request_params",
|
|
22
|
+
"send",
|
|
23
|
+
"enqueue",
|
|
24
|
+
"Reader",
|
|
25
|
+
"Timer",
|
|
26
|
+
]
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
# ruff: noqa: F401
|
|
2
|
-
from .auth import token_auth_middleware
|
|
3
|
-
from .csv import detect_tabular_from_headers
|
|
4
|
-
from .errors import IOException, ParseException, handle_parse_exception
|
|
5
|
-
from .file import compute_checksum_from_file, download_resource, read_csv_gz
|
|
6
|
-
from .geojson import detect_geojson_from_headers_or_catalog
|
|
7
|
-
from .http import UdataPayload, get_request_params, is_valid_uri, send
|
|
8
|
-
from .queue import enqueue
|
|
9
|
-
from .reader import Reader, generate_dialect
|
|
10
|
-
from .timer import Timer
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/crawl/calculate_next_check.py
RENAMED
|
File without changes
|
{udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/crawl/check_resources.py
RENAMED
|
File without changes
|
|
File without changes
|
{udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/crawl/preprocess_check_data.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/db/resource_exception.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{udata_hydra-2.2.2.dev7569 → udata_hydra-2.2.2.dev7591}/udata_hydra/schemas/resource_exception.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|