udata-hydra 2.2.1.dev7367__tar.gz → 2.2.1.dev7385__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/PKG-INFO +1 -1
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/pyproject.toml +1 -1
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/cli.py +9 -9
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/README.md +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/__init__.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/analysis/__init__.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/analysis/csv.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/analysis/geojson.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/analysis/helpers.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/analysis/resource.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/app.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/config_default.toml +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/context.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/crawl/__init__.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/crawl/calculate_next_check.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/crawl/check_resources.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/crawl/helpers.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/crawl/preprocess_check_data.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/crawl/select_batch.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/db/__init__.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/db/check.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/db/resource.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/db/resource_exception.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/logger.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/__init__.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/csv/20221205_initial_up_rev1.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/csv/20230130_drop_migrations.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/csv/20230206_datetime_aware.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/csv/20240827_add_indexes_column_to_tables_index_table.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20221205_initial_up_rev1.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20221206_rev1_up_rev2.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20221206_rev2_up_rev3.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20221208_rev3_up_rev4.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20221208_rev4_up_rev5.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20230119_rev5_up_rev6.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20230121_rev6_up_rev7.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20230121_rev7_up_rev8.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20230130_drop_migrations.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20230206_datetime_aware.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20230515_rev8_up_rev9.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20230606_rev9_up_rev10.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20231102_drop_csv_analysis.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20240827_add_resources_exceptions_table.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20240926_add_indexes.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20241004_add_comment_column_to_resources_exceptions.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20241021_add_parquet_columns.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20241023_alter_foreign_key.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20241025_add_next_check_column.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20250108_add_indexes.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/migrations/main/20250130_add_pmtiles_fields.sql +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/routes/__init__.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/routes/checks.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/routes/resources.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/routes/resources_exceptions.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/routes/status.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/schemas/__init__.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/schemas/check.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/schemas/resource.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/schemas/resource_exception.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/utils/__init__.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/utils/auth.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/utils/csv.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/utils/db.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/utils/errors.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/utils/file.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/utils/geojson.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/utils/http.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/utils/minio.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/utils/parquet.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/utils/queue.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/utils/reader.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/utils/timer.py +0 -0
- {udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/worker.py +0 -0
|
@@ -445,26 +445,26 @@ async def insert_resource_into_catalog(resource_id: str):
|
|
|
445
445
|
|
|
446
446
|
@cli
|
|
447
447
|
async def purge_selected_csv_tables(
|
|
448
|
-
|
|
449
|
-
|
|
448
|
+
retention_days: int | None = None,
|
|
449
|
+
retention_tables: int | None = None,
|
|
450
450
|
quiet: bool = False,
|
|
451
451
|
) -> None:
|
|
452
452
|
"""Delete converted CSV tables either:
|
|
453
|
-
- if they're more than
|
|
454
|
-
- if they're not in the top
|
|
453
|
+
- if they're more than retention_days days old
|
|
454
|
+
- if they're not in the top retention_tables most recent
|
|
455
455
|
"""
|
|
456
456
|
if quiet:
|
|
457
457
|
log.setLevel(logging.ERROR)
|
|
458
458
|
|
|
459
|
-
assert
|
|
459
|
+
assert retention_days is not None or retention_tables is not None
|
|
460
460
|
conn_csv = await connection(db_name="csv")
|
|
461
|
-
if
|
|
462
|
-
threshold = datetime.now(timezone.utc) - timedelta(days=int(
|
|
461
|
+
if retention_days is not None:
|
|
462
|
+
threshold = datetime.now(timezone.utc) - timedelta(days=int(retention_days))
|
|
463
463
|
q = """SELECT parsing_table FROM tables_index WHERE created_at <= $1"""
|
|
464
464
|
res: list[Record] = await conn_csv.fetch(q, threshold)
|
|
465
|
-
elif
|
|
465
|
+
elif retention_tables is not None:
|
|
466
466
|
q = """SELECT parsing_table FROM tables_index ORDER BY created_at DESC OFFSET $1"""
|
|
467
|
-
res: list[Record] = await conn_csv.fetch(q, int(
|
|
467
|
+
res: list[Record] = await conn_csv.fetch(q, int(retention_tables))
|
|
468
468
|
|
|
469
469
|
tables_to_delete: list[str] = [r["parsing_table"] for r in res]
|
|
470
470
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/crawl/calculate_next_check.py
RENAMED
|
File without changes
|
{udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/crawl/check_resources.py
RENAMED
|
File without changes
|
|
File without changes
|
{udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/crawl/preprocess_check_data.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/db/resource_exception.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/routes/resources_exceptions.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{udata_hydra-2.2.1.dev7367 → udata_hydra-2.2.1.dev7385}/udata_hydra/schemas/resource_exception.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|