udata-hydra 2.0.0.dev2547__tar.gz → 2.0.0.dev2576__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/PKG-INFO +8 -4
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/README.md +7 -3
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/pyproject.toml +1 -1
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/__init__.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/analysis/__init__.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/analysis/csv.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/analysis/errors.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/analysis/helpers.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/analysis/resource.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/app.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/cli.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/config_default.toml +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/context.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/crawl.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/db/__init__.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/db/check.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/db/resource.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/logger.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/migrations/__init__.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/migrations/csv/20221205_initial_up_rev1.sql +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/migrations/csv/20230130_drop_migrations.sql +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/migrations/csv/20230206_datetime_aware.sql +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/migrations/main/20221205_initial_up_rev1.sql +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/migrations/main/20221206_rev1_up_rev2.sql +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/migrations/main/20221206_rev2_up_rev3.sql +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/migrations/main/20221208_rev3_up_rev4.sql +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/migrations/main/20221208_rev4_up_rev5.sql +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/migrations/main/20230119_rev5_up_rev6.sql +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/migrations/main/20230121_rev6_up_rev7.sql +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/migrations/main/20230121_rev7_up_rev8.sql +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/migrations/main/20230130_drop_migrations.sql +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/migrations/main/20230206_datetime_aware.sql +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/migrations/main/20230515_rev8_up_rev9.sql +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/migrations/main/20230606_rev9_up_rev10.sql +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/migrations/main/20231102_drop_csv_analysis.sql +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/routes/__init__.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/routes/checks.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/routes/resources.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/routes/status.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/schemas/__init__.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/schemas/check.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/schemas/resource.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/utils/__init__.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/utils/app_version.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/utils/csv.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/utils/file.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/utils/http.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/utils/minio.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/utils/parquet.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/utils/queue.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/utils/reader.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/utils/timer.py +0 -0
- {udata_hydra-2.0.0.dev2547 → udata_hydra-2.0.0.dev2576}/udata_hydra/worker.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: udata-hydra
|
|
3
|
-
Version: 2.0.0.
|
|
3
|
+
Version: 2.0.0.dev2576
|
|
4
4
|
Summary: Async crawler and parsing service for data.gouv.fr
|
|
5
5
|
License: MIT
|
|
6
6
|
Author: Opendata Team
|
|
@@ -100,10 +100,14 @@ Converted CSV tables will be stored in the database specified via `config.DATABA
|
|
|
100
100
|
|
|
101
101
|
## Tests
|
|
102
102
|
|
|
103
|
-
To run the tests, you need to launch the database, the test database, and the Redis broker with `docker
|
|
103
|
+
To run the tests, you need to launch the database, the test database, and the Redis broker with `docker compose -f docker-compose.yml -f docker-compose.test.yml -f docker-compose.broker.yml up -d`.
|
|
104
104
|
|
|
105
105
|
Then you can run the tests with `poetry run pytest`.
|
|
106
106
|
|
|
107
|
+
To run a specific test file, you can pass the path to the file to pytest, like this: `poetry run pytest tests/test_app.py`.
|
|
108
|
+
|
|
109
|
+
To run a specific test function, you can pass the path to the file and the name of the function to pytest, like this: `poetry run pytest tests/test_app.py::test_get_latest_check`.
|
|
110
|
+
|
|
107
111
|
If you would like to see print statements as they are executed, you can pass the -s flag to pytest (`poetry run pytest -s`). However, note that this can sometimes be difficult to parse.
|
|
108
112
|
|
|
109
113
|
### Tests coverage
|
|
@@ -335,14 +339,14 @@ The payload should look something like:
|
|
|
335
339
|
|
|
336
340
|
## Development
|
|
337
341
|
|
|
338
|
-
### docker
|
|
342
|
+
### docker compose
|
|
339
343
|
|
|
340
344
|
Multiple docker-compose files are provided:
|
|
341
345
|
- a minimal `docker-compose.yml` with two PostgreSQL containers (one for catalog and metadata, the other for converted CSV to database)
|
|
342
346
|
- `docker-compose.broker.yml` adds a Redis broker
|
|
343
347
|
- `docker-compose.test.yml` launches a test DB, needed to run tests
|
|
344
348
|
|
|
345
|
-
NB: you can launch compose from multiple files like this: `docker
|
|
349
|
+
NB: you can launch compose from multiple files like this: `docker compose -f docker-compose.yml -f docker-compose.test.yml up`
|
|
346
350
|
|
|
347
351
|
### Logging & Debugging
|
|
348
352
|
|
|
@@ -63,10 +63,14 @@ Converted CSV tables will be stored in the database specified via `config.DATABA
|
|
|
63
63
|
|
|
64
64
|
## Tests
|
|
65
65
|
|
|
66
|
-
To run the tests, you need to launch the database, the test database, and the Redis broker with `docker
|
|
66
|
+
To run the tests, you need to launch the database, the test database, and the Redis broker with `docker compose -f docker-compose.yml -f docker-compose.test.yml -f docker-compose.broker.yml up -d`.
|
|
67
67
|
|
|
68
68
|
Then you can run the tests with `poetry run pytest`.
|
|
69
69
|
|
|
70
|
+
To run a specific test file, you can pass the path to the file to pytest, like this: `poetry run pytest tests/test_app.py`.
|
|
71
|
+
|
|
72
|
+
To run a specific test function, you can pass the path to the file and the name of the function to pytest, like this: `poetry run pytest tests/test_app.py::test_get_latest_check`.
|
|
73
|
+
|
|
70
74
|
If you would like to see print statements as they are executed, you can pass the -s flag to pytest (`poetry run pytest -s`). However, note that this can sometimes be difficult to parse.
|
|
71
75
|
|
|
72
76
|
### Tests coverage
|
|
@@ -298,14 +302,14 @@ The payload should look something like:
|
|
|
298
302
|
|
|
299
303
|
## Development
|
|
300
304
|
|
|
301
|
-
### docker
|
|
305
|
+
### docker compose
|
|
302
306
|
|
|
303
307
|
Multiple docker-compose files are provided:
|
|
304
308
|
- a minimal `docker-compose.yml` with two PostgreSQL containers (one for catalog and metadata, the other for converted CSV to database)
|
|
305
309
|
- `docker-compose.broker.yml` adds a Redis broker
|
|
306
310
|
- `docker-compose.test.yml` launches a test DB, needed to run tests
|
|
307
311
|
|
|
308
|
-
NB: you can launch compose from multiple files like this: `docker
|
|
312
|
+
NB: you can launch compose from multiple files like this: `docker compose -f docker-compose.yml -f docker-compose.test.yml up`
|
|
309
313
|
|
|
310
314
|
### Logging & Debugging
|
|
311
315
|
|
|
@@ -9,7 +9,7 @@ readme = "README.md"
|
|
|
9
9
|
|
|
10
10
|
[tool.poetry]
|
|
11
11
|
name = "udata-hydra"
|
|
12
|
-
version = "2.0.0.
|
|
12
|
+
version = "2.0.0.dev2576"
|
|
13
13
|
description = "Async crawler and parsing service for data.gouv.fr"
|
|
14
14
|
authors = ["Opendata Team <opendatateam@data.gouv.fr>"]
|
|
15
15
|
license = "MIT"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|