eolas-data 1.3.5__tar.gz → 1.3.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {eolas_data-1.3.5 → eolas_data-1.3.13}/.github/workflows/smoke.yml +2 -2
- {eolas_data-1.3.5 → eolas_data-1.3.13}/PKG-INFO +1 -1
- {eolas_data-1.3.5 → eolas_data-1.3.13}/eolas_data/__init__.py +1 -1
- {eolas_data-1.3.5 → eolas_data-1.3.13}/eolas_data/client.py +58 -6
- {eolas_data-1.3.5 → eolas_data-1.3.13}/eolas_data/library.py +93 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/eolas_data/meta.py +59 -5
- {eolas_data-1.3.5 → eolas_data-1.3.13}/pyproject.toml +1 -1
- {eolas_data-1.3.5 → eolas_data-1.3.13}/tests/test_meta.py +26 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/tests/test_progress.py +38 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/tests/test_smoke_live.py +32 -3
- {eolas_data-1.3.5 → eolas_data-1.3.13}/tests/test_sync_bulk.py +59 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/.github/workflows/catalog-drift.yml +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/.github/workflows/publish.yml +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/.github/workflows/test.yml +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/.gitignore +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/README.md +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/eolas_data/_dataset_names.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/eolas_data/_regen_names.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/eolas_data/cdc.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/eolas_data/cli.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/eolas_data/console.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/eolas_data/dataset.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/eolas_data/exceptions.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/eolas_data/rows.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/eolas_data/schedule.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/eolas_data/search.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/scripts/preflight.sh +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/tests/test_as_arrow.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/tests/test_cdc_roundtrip.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/tests/test_cli.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/tests/test_client.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/tests/test_keyring.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/tests/test_library.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/tests/test_rows.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/tests/test_schedule.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/tests/test_search.py +0 -0
- {eolas_data-1.3.5 → eolas_data-1.3.13}/tests/test_sync_changes.py +0 -0
|
@@ -266,6 +266,36 @@ class Client:
|
|
|
266
266
|
self._meta_cache[key] = self.info(name)
|
|
267
267
|
return self._meta_cache[key]
|
|
268
268
|
|
|
269
|
+
def _apply_force(self, name: Union[str, "DatasetName"], force: bool) -> None:
|
|
270
|
+
if force:
|
|
271
|
+
self._meta_cache.pop(str(name), None)
|
|
272
|
+
|
|
273
|
+
def cache_clear(
|
|
274
|
+
self,
|
|
275
|
+
name: Optional[Union[str, "DatasetName"]] = None,
|
|
276
|
+
*,
|
|
277
|
+
cache_dir: Optional[Union[str, "pathlib.Path"]] = None,
|
|
278
|
+
format: Optional[str] = None,
|
|
279
|
+
files: bool = True,
|
|
280
|
+
meta: bool = True,
|
|
281
|
+
) -> dict:
|
|
282
|
+
"""Clear client-side cache (library files and/or session metadata).
|
|
283
|
+
|
|
284
|
+
See :func:`eolas_data.library.cache_clear` for details. Pass
|
|
285
|
+
``force=True`` to :meth:`get_local` / :meth:`sync_bulk` to clear
|
|
286
|
+
metadata and re-download in one step.
|
|
287
|
+
"""
|
|
288
|
+
from .library import cache_clear as _cache_clear
|
|
289
|
+
|
|
290
|
+
return _cache_clear(
|
|
291
|
+
None if name is None else str(name),
|
|
292
|
+
cache_dir=cache_dir,
|
|
293
|
+
format=format,
|
|
294
|
+
files=files,
|
|
295
|
+
meta=meta,
|
|
296
|
+
meta_cache=self._meta_cache if meta else None,
|
|
297
|
+
)
|
|
298
|
+
|
|
269
299
|
def _attach_dataset_meta(
|
|
270
300
|
self,
|
|
271
301
|
result: "pd.DataFrame",
|
|
@@ -486,6 +516,7 @@ class Client:
|
|
|
486
516
|
format: str = "parquet",
|
|
487
517
|
freshness: str = "auto",
|
|
488
518
|
progress: ProgressControl = None,
|
|
519
|
+
force: bool = False,
|
|
489
520
|
) -> SyncResult:
|
|
490
521
|
"""Incrementally sync a bulk dataset file — only re-download when the snapshot changes.
|
|
491
522
|
|
|
@@ -518,6 +549,8 @@ class Client:
|
|
|
518
549
|
See :meth:`get_local` for the full selector vocabulary.
|
|
519
550
|
When ``status="unchanged"`` no download bar is shown; an
|
|
520
551
|
informative cached-file message is printed instead.
|
|
552
|
+
force: When ``True``, skip the sidecar unchanged fast path and
|
|
553
|
+
re-download even when the local snapshot id matches the server.
|
|
521
554
|
|
|
522
555
|
Returns:
|
|
523
556
|
A :class:`SyncResult` dataclass with ``status``,
|
|
@@ -565,6 +598,8 @@ class Client:
|
|
|
565
598
|
f"Unknown freshness {freshness!r}. Expected 'auto', 'monthly', or 'current'."
|
|
566
599
|
)
|
|
567
600
|
|
|
601
|
+
self._apply_force(name, force)
|
|
602
|
+
|
|
568
603
|
out = pathlib.Path(path).expanduser().resolve()
|
|
569
604
|
sidecar = pathlib.Path(str(out) + ".eolas-meta.json")
|
|
570
605
|
|
|
@@ -601,7 +636,8 @@ class Client:
|
|
|
601
636
|
|
|
602
637
|
# No-op fast path: snapshot hasn't changed AND file exists on disk.
|
|
603
638
|
if (
|
|
604
|
-
|
|
639
|
+
not force
|
|
640
|
+
and prev is not None
|
|
605
641
|
and prev.get("snapshot_id") == current_sid
|
|
606
642
|
and out.exists()
|
|
607
643
|
):
|
|
@@ -677,6 +713,7 @@ class Client:
|
|
|
677
713
|
*,
|
|
678
714
|
format: str = "parquet",
|
|
679
715
|
progress: Optional[bool] = None,
|
|
716
|
+
force: bool = False,
|
|
680
717
|
) -> SyncResult:
|
|
681
718
|
"""Incrementally sync a changelog-tier dataset via the /changes feed.
|
|
682
719
|
|
|
@@ -710,6 +747,7 @@ class Client:
|
|
|
710
747
|
changelog sync; other formats raise ``ValueError``.
|
|
711
748
|
progress: Forwarded to :meth:`sync_bulk` for the baseline download
|
|
712
749
|
progress bar (``None`` auto-detects TTY).
|
|
750
|
+
force: When ``True``, re-baseline from a full bulk snapshot.
|
|
713
751
|
|
|
714
752
|
Returns:
|
|
715
753
|
A :class:`SyncResult` with ``sync_mode='changelog'``,
|
|
@@ -748,6 +786,7 @@ class Client:
|
|
|
748
786
|
|
|
749
787
|
out = pathlib.Path(path).expanduser().resolve()
|
|
750
788
|
sidecar_path = pathlib.Path(str(out) + ".eolas-meta.json")
|
|
789
|
+
self._apply_force(name, force)
|
|
751
790
|
|
|
752
791
|
# Read sidecar.
|
|
753
792
|
sidecar: Optional[dict] = None
|
|
@@ -758,7 +797,7 @@ class Client:
|
|
|
758
797
|
sidecar = None
|
|
759
798
|
|
|
760
799
|
# Determine if we need a cold-start baseline.
|
|
761
|
-
needs_baseline = (
|
|
800
|
+
needs_baseline = force or (
|
|
762
801
|
sidecar is None
|
|
763
802
|
or sidecar.get("sync_mode") != "changelog"
|
|
764
803
|
or sidecar.get("watermark_seq") is None
|
|
@@ -782,6 +821,7 @@ class Client:
|
|
|
782
821
|
format=fmt,
|
|
783
822
|
freshness="current",
|
|
784
823
|
progress=progress,
|
|
824
|
+
force=force,
|
|
785
825
|
)
|
|
786
826
|
baseline_snapshot_id = bulk_result.current_snapshot_id
|
|
787
827
|
|
|
@@ -844,6 +884,7 @@ class Client:
|
|
|
844
884
|
format=fmt,
|
|
845
885
|
freshness="current",
|
|
846
886
|
progress=progress,
|
|
887
|
+
force=force,
|
|
847
888
|
)
|
|
848
889
|
baseline_snapshot_id = bulk_result.current_snapshot_id
|
|
849
890
|
high_seq = self._fetch_changes_seq_high(changes_url, since_seq=2**62)
|
|
@@ -948,6 +989,7 @@ class Client:
|
|
|
948
989
|
format: str = "parquet",
|
|
949
990
|
freshness: str = "auto",
|
|
950
991
|
progress: Optional[bool] = None,
|
|
992
|
+
force: bool = False,
|
|
951
993
|
) -> SyncResult:
|
|
952
994
|
"""Unified sync dispatcher — keeps a local file current automatically.
|
|
953
995
|
|
|
@@ -973,6 +1015,7 @@ class Client:
|
|
|
973
1015
|
freshness: ``"auto"`` (default), ``"monthly"``, or ``"current"``. Only
|
|
974
1016
|
used for the snapshot path (passed to :meth:`sync_bulk`).
|
|
975
1017
|
progress: Control the download progress bar.
|
|
1018
|
+
force: Bypass local unchanged cache and re-sync from the server.
|
|
976
1019
|
|
|
977
1020
|
Returns:
|
|
978
1021
|
A :class:`SyncResult`. The ``sync_mode`` field is ``"snapshot"`` or
|
|
@@ -996,7 +1039,7 @@ class Client:
|
|
|
996
1039
|
tier = meta.get("cdc_serving_tier", "snapshot") or "snapshot"
|
|
997
1040
|
|
|
998
1041
|
if tier == "changelog":
|
|
999
|
-
return self.sync_changes(name, path, format=format, progress=progress)
|
|
1042
|
+
return self.sync_changes(name, path, format=format, progress=progress, force=force)
|
|
1000
1043
|
else:
|
|
1001
1044
|
result = self.sync_bulk(
|
|
1002
1045
|
name,
|
|
@@ -1004,6 +1047,7 @@ class Client:
|
|
|
1004
1047
|
format=format,
|
|
1005
1048
|
freshness=freshness,
|
|
1006
1049
|
progress=progress,
|
|
1050
|
+
force=force,
|
|
1007
1051
|
)
|
|
1008
1052
|
# Tag the result with sync_mode so callers don't need to inspect tier.
|
|
1009
1053
|
result.sync_mode = "snapshot"
|
|
@@ -1842,6 +1886,7 @@ class Client:
|
|
|
1842
1886
|
as_arrow: bool = False,
|
|
1843
1887
|
meta: bool = True,
|
|
1844
1888
|
progress: ProgressControl = None,
|
|
1889
|
+
force: bool = False,
|
|
1845
1890
|
) -> "pd.DataFrame":
|
|
1846
1891
|
"""Download (or serve from cache) a whole dataset as a local DataFrame.
|
|
1847
1892
|
|
|
@@ -1886,6 +1931,8 @@ class Client:
|
|
|
1886
1931
|
or ``"none"`` for one phase only. Suppressed by
|
|
1887
1932
|
``EOLAS_NO_PROGRESS=1``. Cached snapshots skip the download bar
|
|
1888
1933
|
and print an informative message instead.
|
|
1934
|
+
force: Re-download the library file even when the sidecar says the
|
|
1935
|
+
snapshot is current. See :meth:`cache_clear`.
|
|
1889
1936
|
|
|
1890
1937
|
Returns:
|
|
1891
1938
|
``pd.DataFrame`` (tabular) or ``geopandas.GeoDataFrame`` (geo +
|
|
@@ -1908,6 +1955,7 @@ class Client:
|
|
|
1908
1955
|
)
|
|
1909
1956
|
# Resolve as_geo: None → True (auto) unless as_arrow overrides.
|
|
1910
1957
|
resolved_as_geo = as_geo if as_geo is not None else (not as_arrow)
|
|
1958
|
+
self._apply_force(name, force)
|
|
1911
1959
|
|
|
1912
1960
|
# ---- resolve cache_dir -----------------------------------------------
|
|
1913
1961
|
if cache_dir is not None:
|
|
@@ -1946,7 +1994,10 @@ class Client:
|
|
|
1946
1994
|
file_path = cache_path / f"{name}{ext}"
|
|
1947
1995
|
|
|
1948
1996
|
# ---- sync (download if needed, HEAD check if cached) -----------------
|
|
1949
|
-
self.sync_bulk(
|
|
1997
|
+
self.sync_bulk(
|
|
1998
|
+
name, path=file_path, format=fmt, freshness=freshness,
|
|
1999
|
+
progress=progress, force=force,
|
|
2000
|
+
)
|
|
1950
2001
|
|
|
1951
2002
|
show_read = self._resolve_show_progress(progress, "read")
|
|
1952
2003
|
read_label = file_path.name
|
|
@@ -1988,7 +2039,7 @@ class Client:
|
|
|
1988
2039
|
csv_path = cache_path / f"{name}{self._BULK_EXTENSIONS['csv_gz']}"
|
|
1989
2040
|
self.sync_bulk(
|
|
1990
2041
|
name, path=csv_path, format="csv_gz",
|
|
1991
|
-
freshness=freshness, progress=progress,
|
|
2042
|
+
freshness=freshness, progress=progress, force=force,
|
|
1992
2043
|
)
|
|
1993
2044
|
result = _read_bulk_file(csv_path, "csv_gz")
|
|
1994
2045
|
else:
|
|
@@ -2011,6 +2062,7 @@ class Client:
|
|
|
2011
2062
|
as_arrow: bool = False,
|
|
2012
2063
|
meta: bool = True,
|
|
2013
2064
|
envelope: bool = False,
|
|
2065
|
+
force: bool = False,
|
|
2014
2066
|
) -> Dataset:
|
|
2015
2067
|
"""Fetch dataset rows as a pandas (or polars / geopandas) DataFrame.
|
|
2016
2068
|
|
|
@@ -2096,7 +2148,7 @@ class Client:
|
|
|
2096
2148
|
info_meta = self._info_cached(name)
|
|
2097
2149
|
if self._bulk_export_allowed(info_meta) and self._live_pull_blocked(info_meta):
|
|
2098
2150
|
result = self.get_local(
|
|
2099
|
-
name, as_geo=as_geo, as_arrow=False, meta=meta,
|
|
2151
|
+
name, as_geo=as_geo, as_arrow=False, meta=meta, force=force,
|
|
2100
2152
|
)
|
|
2101
2153
|
if engine == "polars":
|
|
2102
2154
|
try:
|
|
@@ -39,6 +39,13 @@ _CONFIG_FILE = _CONFIG_DIR / "config.json"
|
|
|
39
39
|
# Fallback cache directory (Step 5)
|
|
40
40
|
_FALLBACK_DIR = pathlib.Path.home() / ".cache" / "eolas"
|
|
41
41
|
|
|
42
|
+
# Bulk file extensions (mirrors Client._BULK_EXTENSIONS).
|
|
43
|
+
_BULK_EXTENSIONS = {
|
|
44
|
+
"parquet": ".parquet",
|
|
45
|
+
"csv_gz": ".csv.gz",
|
|
46
|
+
"geoparquet": ".geo.parquet",
|
|
47
|
+
}
|
|
48
|
+
|
|
42
49
|
|
|
43
50
|
# ---------------------------------------------------------------------------
|
|
44
51
|
# Public API
|
|
@@ -94,6 +101,92 @@ def library_clear() -> None:
|
|
|
94
101
|
_remove_config_key("library_dir")
|
|
95
102
|
|
|
96
103
|
|
|
104
|
+
def cache_clear(
|
|
105
|
+
name: Optional[str] = None,
|
|
106
|
+
*,
|
|
107
|
+
cache_dir: Optional[str | pathlib.Path] = None,
|
|
108
|
+
format: Optional[str] = None,
|
|
109
|
+
files: bool = True,
|
|
110
|
+
meta: bool = True,
|
|
111
|
+
meta_cache: Optional[dict] = None,
|
|
112
|
+
) -> dict:
|
|
113
|
+
"""Clear client-side cache for a dataset (or the whole library).
|
|
114
|
+
|
|
115
|
+
eolas-data caches at two client-only levels:
|
|
116
|
+
|
|
117
|
+
* **On-disk bulk files** in the library directory (Parquet/GeoParquet +
|
|
118
|
+
``.eolas-meta.json`` sidecars) — cleared when ``files=True``.
|
|
119
|
+
* **Session metadata** (``Client._meta_cache`` / ``info()`` per dataset) —
|
|
120
|
+
cleared when ``meta=True`` and ``meta_cache`` is the client's dict.
|
|
121
|
+
|
|
122
|
+
Does not contact the API. Use :meth:`Client.get_local` with ``force=True``
|
|
123
|
+
to clear and re-download in one step.
|
|
124
|
+
|
|
125
|
+
Args:
|
|
126
|
+
name: Dataset identifier. ``None`` sweeps the whole library (``files``)
|
|
127
|
+
and/or all session metadata entries (``meta``).
|
|
128
|
+
cache_dir: Library directory. ``None`` uses :func:`resolve_library_dir`.
|
|
129
|
+
format: ``"parquet"``, ``"csv_gz"``, or ``"geoparquet"``. ``None``
|
|
130
|
+
deletes all bulk variants for ``name`` (ignored when ``name`` is
|
|
131
|
+
``None``).
|
|
132
|
+
files: Delete on-disk bulk data files and sidecars.
|
|
133
|
+
meta: Drop session-cached ``info()`` entries from ``meta_cache``.
|
|
134
|
+
meta_cache: The client's ``_meta_cache`` dict (required for ``meta``).
|
|
135
|
+
|
|
136
|
+
Returns:
|
|
137
|
+
``{"files": [deleted paths], "meta_cleared": int}``
|
|
138
|
+
"""
|
|
139
|
+
deleted: list[str] = []
|
|
140
|
+
meta_n = 0
|
|
141
|
+
|
|
142
|
+
if meta and meta_cache is not None:
|
|
143
|
+
if name is None:
|
|
144
|
+
meta_n = len(meta_cache)
|
|
145
|
+
meta_cache.clear()
|
|
146
|
+
else:
|
|
147
|
+
key = str(name)
|
|
148
|
+
if key in meta_cache:
|
|
149
|
+
del meta_cache[key]
|
|
150
|
+
meta_n = 1
|
|
151
|
+
|
|
152
|
+
if files:
|
|
153
|
+
root = (
|
|
154
|
+
pathlib.Path(cache_dir).expanduser().resolve()
|
|
155
|
+
if cache_dir is not None
|
|
156
|
+
else resolve_library_dir(interactive=False)
|
|
157
|
+
)
|
|
158
|
+
if name is None:
|
|
159
|
+
if root.is_dir():
|
|
160
|
+
for p in root.iterdir():
|
|
161
|
+
if p.suffix in {".parquet", ".gz"} or p.name.endswith(".geo.parquet"):
|
|
162
|
+
p.unlink(missing_ok=True)
|
|
163
|
+
deleted.append(str(p))
|
|
164
|
+
elif p.name.endswith(".eolas-meta.json"):
|
|
165
|
+
p.unlink(missing_ok=True)
|
|
166
|
+
deleted.append(str(p))
|
|
167
|
+
elif format is None:
|
|
168
|
+
for ext in _BULK_EXTENSIONS.values():
|
|
169
|
+
p = root / f"{name}{ext}"
|
|
170
|
+
for candidate in (p, pathlib.Path(str(p) + ".eolas-meta.json")):
|
|
171
|
+
if candidate.exists():
|
|
172
|
+
candidate.unlink()
|
|
173
|
+
deleted.append(str(candidate))
|
|
174
|
+
else:
|
|
175
|
+
fmt = format.lower()
|
|
176
|
+
if fmt not in _BULK_EXTENSIONS:
|
|
177
|
+
raise ValueError(
|
|
178
|
+
f"Unknown format {format!r}. Expected one of: "
|
|
179
|
+
+ ", ".join(_BULK_EXTENSIONS)
|
|
180
|
+
)
|
|
181
|
+
p = root / f"{name}{_BULK_EXTENSIONS[fmt]}"
|
|
182
|
+
for candidate in (p, pathlib.Path(str(p) + ".eolas-meta.json")):
|
|
183
|
+
if candidate.exists():
|
|
184
|
+
candidate.unlink()
|
|
185
|
+
deleted.append(str(candidate))
|
|
186
|
+
|
|
187
|
+
return {"files": deleted, "meta_cleared": meta_n}
|
|
188
|
+
|
|
189
|
+
|
|
97
190
|
def library_status() -> dict:
|
|
98
191
|
"""Return a dict describing how the library directory is currently resolved.
|
|
99
192
|
|
|
@@ -68,6 +68,40 @@ def split_meta(info: dict) -> tuple[dict, Optional[pd.DataFrame]]:
|
|
|
68
68
|
return table, columns
|
|
69
69
|
|
|
70
70
|
|
|
71
|
+
def _column_meta_records(
|
|
72
|
+
column_meta: Optional[pd.DataFrame | list[dict[str, Any]]],
|
|
73
|
+
) -> Optional[list[dict[str, Any]]]:
|
|
74
|
+
if column_meta is None:
|
|
75
|
+
return None
|
|
76
|
+
if isinstance(column_meta, pd.DataFrame):
|
|
77
|
+
if column_meta.empty:
|
|
78
|
+
return None
|
|
79
|
+
return column_meta.to_dict("records")
|
|
80
|
+
if isinstance(column_meta, list):
|
|
81
|
+
return column_meta or None
|
|
82
|
+
return None
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _column_meta_dataframe(
|
|
86
|
+
column_meta: Optional[pd.DataFrame | list[dict[str, Any]]],
|
|
87
|
+
) -> Optional[pd.DataFrame]:
|
|
88
|
+
if column_meta is None:
|
|
89
|
+
return None
|
|
90
|
+
if isinstance(column_meta, pd.DataFrame):
|
|
91
|
+
return column_meta if not column_meta.empty else None
|
|
92
|
+
if isinstance(column_meta, list) and column_meta:
|
|
93
|
+
return pd.DataFrame(column_meta)
|
|
94
|
+
return None
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _is_geodataframe(df: pd.DataFrame) -> bool:
|
|
98
|
+
try:
|
|
99
|
+
import geopandas as gpd
|
|
100
|
+
except ImportError:
|
|
101
|
+
return False
|
|
102
|
+
return isinstance(df, gpd.GeoDataFrame)
|
|
103
|
+
|
|
104
|
+
|
|
71
105
|
def attach_meta(
|
|
72
106
|
df: pd.DataFrame,
|
|
73
107
|
*,
|
|
@@ -77,16 +111,32 @@ def attach_meta(
|
|
|
77
111
|
column_meta: Optional[pd.DataFrame] = None,
|
|
78
112
|
) -> pd.DataFrame:
|
|
79
113
|
"""Attach eolas metadata attrs to a DataFrame / Dataset / GeoDataFrame."""
|
|
80
|
-
|
|
114
|
+
col_records = _column_meta_records(column_meta)
|
|
115
|
+
col_df = _column_meta_dataframe(column_meta)
|
|
116
|
+
# attrs must be JSON-serialisable — never store a DataFrame there (breaks
|
|
117
|
+
# pandas repr/head on GeoDataFrames when attrs are compared with ==).
|
|
118
|
+
attrs_payload = {
|
|
81
119
|
"eolas_name": name,
|
|
82
120
|
"eolas_source": source,
|
|
83
121
|
"eolas_meta": table_meta or {},
|
|
84
|
-
"eolas_columns":
|
|
122
|
+
"eolas_columns": col_records,
|
|
85
123
|
}
|
|
86
124
|
attrs = getattr(df, "attrs", None)
|
|
87
125
|
if isinstance(attrs, dict):
|
|
88
|
-
attrs.update(
|
|
89
|
-
|
|
126
|
+
attrs.update(attrs_payload)
|
|
127
|
+
|
|
128
|
+
if _is_geodataframe(df):
|
|
129
|
+
# GeoDataFrame: attrs only — setattr triggers geopandas/pandas warnings
|
|
130
|
+
# and can break tabular repr.
|
|
131
|
+
return df
|
|
132
|
+
|
|
133
|
+
object_payload = {
|
|
134
|
+
"eolas_name": name,
|
|
135
|
+
"eolas_source": source,
|
|
136
|
+
"eolas_meta": table_meta or {},
|
|
137
|
+
"eolas_columns": col_df,
|
|
138
|
+
}
|
|
139
|
+
for key, val in object_payload.items():
|
|
90
140
|
try:
|
|
91
141
|
setattr(df, key, val)
|
|
92
142
|
except (AttributeError, TypeError):
|
|
@@ -108,7 +158,11 @@ def meta_subtitle(table_meta: Optional[dict]) -> str:
|
|
|
108
158
|
return " · ".join(parts)
|
|
109
159
|
|
|
110
160
|
|
|
111
|
-
def column_label(
|
|
161
|
+
def column_label(
|
|
162
|
+
column_meta: Optional[pd.DataFrame | list[dict[str, Any]]],
|
|
163
|
+
column: str,
|
|
164
|
+
) -> Optional[str]:
|
|
165
|
+
column_meta = _column_meta_dataframe(column_meta)
|
|
112
166
|
if column_meta is None or column_meta.empty or "name" not in column_meta.columns:
|
|
113
167
|
return None
|
|
114
168
|
rows = column_meta.loc[column_meta["name"] == column, "description"]
|
|
@@ -62,6 +62,32 @@ def test_meta_false_skips_info_fetch(client):
|
|
|
62
62
|
assert df.eolas_columns is None
|
|
63
63
|
|
|
64
64
|
|
|
65
|
+
def test_attach_meta_geodataframe_head_does_not_break_repr():
|
|
66
|
+
gpd = pytest.importorskip("geopandas")
|
|
67
|
+
from shapely.geometry import Point
|
|
68
|
+
|
|
69
|
+
from eolas_data.meta import attach_meta
|
|
70
|
+
|
|
71
|
+
gdf = gpd.GeoDataFrame(
|
|
72
|
+
{"id": [1]},
|
|
73
|
+
geometry=[Point(0, 0)],
|
|
74
|
+
crs="EPSG:4326",
|
|
75
|
+
)
|
|
76
|
+
cols = pd.DataFrame([
|
|
77
|
+
{"name": "id", "type": "int", "description": "Identifier"},
|
|
78
|
+
])
|
|
79
|
+
attach_meta(
|
|
80
|
+
gdf,
|
|
81
|
+
name="nz_addresses",
|
|
82
|
+
source="LINZ",
|
|
83
|
+
table_meta={"title": "NZ Addresses"},
|
|
84
|
+
column_meta=cols,
|
|
85
|
+
)
|
|
86
|
+
gdf.head() # must not raise (pandas attrs must not hold DataFrames)
|
|
87
|
+
assert gdf.attrs["eolas_name"] == "nz_addresses"
|
|
88
|
+
assert gdf.attrs["eolas_columns"][0]["name"] == "id"
|
|
89
|
+
|
|
90
|
+
|
|
65
91
|
@resp_lib.activate
|
|
66
92
|
def test_info_cached_on_second_get(client):
|
|
67
93
|
resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_cpi/data", json={"data": RECORDS})
|
|
@@ -203,6 +203,44 @@ def test_progress_kwarg_forces_hide(client, tmp_path):
|
|
|
203
203
|
assert all(disabled_values), "tqdm must be disabled when progress=False"
|
|
204
204
|
|
|
205
205
|
|
|
206
|
+
# ---------------------------------------------------------------------------
|
|
207
|
+
# test_download_bulk_missing_content_length
|
|
208
|
+
# ---------------------------------------------------------------------------
|
|
209
|
+
|
|
210
|
+
@resp_lib.activate
|
|
211
|
+
def test_download_bulk_missing_content_length(client, tmp_path):
|
|
212
|
+
"""CDN may omit Content-Length — download must succeed with indeterminate bar."""
|
|
213
|
+
resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_cpi",
|
|
214
|
+
json=BULK_DATASET_META, status=200)
|
|
215
|
+
resp_lib.add(resp_lib.GET, f"{BASE}/v1/bulk/statsnz/nz_cpi",
|
|
216
|
+
body=FAKE_PARQUET,
|
|
217
|
+
content_type="application/octet-stream",
|
|
218
|
+
status=200)
|
|
219
|
+
# Deliberately no Content-Length header.
|
|
220
|
+
|
|
221
|
+
dest = tmp_path / "nz_cpi.parquet"
|
|
222
|
+
totals_seen = []
|
|
223
|
+
|
|
224
|
+
class FakeTqdm:
|
|
225
|
+
def __init__(self, **kwargs):
|
|
226
|
+
totals_seen.append(kwargs.get("total"))
|
|
227
|
+
def __enter__(self):
|
|
228
|
+
return self
|
|
229
|
+
def __exit__(self, *a):
|
|
230
|
+
pass
|
|
231
|
+
def update(self, n):
|
|
232
|
+
pass
|
|
233
|
+
|
|
234
|
+
with patch("sys.stdout") as mock_stdout, \
|
|
235
|
+
patch("tqdm.auto.tqdm", FakeTqdm):
|
|
236
|
+
mock_stdout.isatty.return_value = True
|
|
237
|
+
out = client.download_bulk("nz_cpi", path=dest, progress=True)
|
|
238
|
+
|
|
239
|
+
assert out == dest
|
|
240
|
+
assert dest.read_bytes() == FAKE_PARQUET
|
|
241
|
+
assert totals_seen == [None]
|
|
242
|
+
|
|
243
|
+
|
|
206
244
|
# ---------------------------------------------------------------------------
|
|
207
245
|
# test_sync_bulk_unchanged_no_bar
|
|
208
246
|
# ---------------------------------------------------------------------------
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
"""Live API smoke tests — run against api.eolas.fyi with a real key.
|
|
2
2
|
|
|
3
|
+
Mirrors eolas-r/tests/smoke-live.R and eolas/docs/client-contract.md.
|
|
4
|
+
|
|
3
5
|
Skipped in the default unit CI job (``pytest -m 'not integration'``).
|
|
4
6
|
Enable locally or in the weekly smoke workflow::
|
|
5
7
|
|
|
@@ -8,6 +10,7 @@ Enable locally or in the weekly smoke workflow::
|
|
|
8
10
|
from __future__ import annotations
|
|
9
11
|
|
|
10
12
|
import os
|
|
13
|
+
from pathlib import Path
|
|
11
14
|
|
|
12
15
|
import pandas as pd
|
|
13
16
|
import pytest
|
|
@@ -19,8 +22,17 @@ pytestmark = pytest.mark.integration
|
|
|
19
22
|
_SKIP = not os.environ.get("EOLAS_API_KEY")
|
|
20
23
|
|
|
21
24
|
|
|
22
|
-
@pytest.fixture(
|
|
23
|
-
def
|
|
25
|
+
@pytest.fixture()
|
|
26
|
+
def smoke_library(tmp_path, monkeypatch):
|
|
27
|
+
"""Isolated on-disk library — same idea as EOLAS_LIBRARY in R smoke."""
|
|
28
|
+
lib = tmp_path / "eolas-smoke-lib"
|
|
29
|
+
lib.mkdir()
|
|
30
|
+
monkeypatch.setenv("EOLAS_LIBRARY", str(lib))
|
|
31
|
+
return lib
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@pytest.fixture()
|
|
35
|
+
def live_client(smoke_library):
|
|
24
36
|
if _SKIP:
|
|
25
37
|
pytest.skip("EOLAS_API_KEY not set — live smoke tests skipped")
|
|
26
38
|
return Client()
|
|
@@ -53,4 +65,21 @@ def test_get_nz_cpi_slice(live_client):
|
|
|
53
65
|
def test_info_nz_cpi(live_client):
|
|
54
66
|
meta = live_client.info("nz_cpi")
|
|
55
67
|
assert meta["name"] == "nz_cpi"
|
|
56
|
-
assert "source" in meta
|
|
68
|
+
assert "source" in meta
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def test_client_exports_cache_clear(live_client):
|
|
72
|
+
assert hasattr(live_client, "cache_clear")
|
|
73
|
+
assert callable(live_client.cache_clear)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def test_linz_nz_addresses_bulk_route_and_head(live_client, smoke_library):
|
|
77
|
+
"""User path: client.linz() — not get_local(). head() must not crash."""
|
|
78
|
+
gdf = live_client.linz("nz_addresses", progress=False)
|
|
79
|
+
assert len(gdf) > 100_000
|
|
80
|
+
# GeoDataFrame or Dataset — repr/head must work (pandas attrs regression)
|
|
81
|
+
gdf.head()
|
|
82
|
+
# Bulk file landed in isolated library
|
|
83
|
+
assert any(smoke_library.iterdir())
|
|
84
|
+
geo_files = list(smoke_library.glob("nz_addresses*.geo.parquet"))
|
|
85
|
+
assert geo_files, f"expected bulk geo file under {smoke_library}"
|
|
@@ -438,3 +438,62 @@ def test_cli_sync_unknown_freshness_exits_usage():
|
|
|
438
438
|
def test_cli_sync_invalid_watch_exits_usage():
|
|
439
439
|
result = runner.invoke(app, ["sync", "nz_cpi", "--watch", "forever", "--api-key", "k"])
|
|
440
440
|
assert result.exit_code == cli_module.EXIT_USAGE
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
# ---------------------------------------------------------------------------
|
|
444
|
+
# cache_clear / force
|
|
445
|
+
# ---------------------------------------------------------------------------
|
|
446
|
+
|
|
447
|
+
@resp_lib.activate
|
|
448
|
+
def test_sync_bulk_force_redownloads_when_unchanged(client, tmp_path):
|
|
449
|
+
"""force=True skips the sidecar unchanged fast path."""
|
|
450
|
+
dest = tmp_path / "nz_cpi.parquet"
|
|
451
|
+
dest.write_bytes(FAKE_PARQUET)
|
|
452
|
+
_write_sidecar(dest, SNAPSHOT_V1)
|
|
453
|
+
|
|
454
|
+
resp_lib.add(resp_lib.GET, f"{BASE}/v1/datasets/nz_cpi",
|
|
455
|
+
json=BULK_DATASET_META, status=200)
|
|
456
|
+
resp_lib.add(resp_lib.HEAD, f"{BASE}/v1/bulk/statsnz/nz_cpi",
|
|
457
|
+
body=b"", status=200,
|
|
458
|
+
headers={"X-Snapshot-Version": SNAPSHOT_V1})
|
|
459
|
+
resp_lib.add(resp_lib.GET, f"{BASE}/v1/bulk/statsnz/nz_cpi",
|
|
460
|
+
body=FAKE_PARQUET_V2,
|
|
461
|
+
content_type="application/octet-stream",
|
|
462
|
+
status=200)
|
|
463
|
+
|
|
464
|
+
result = client.sync_bulk("nz_cpi", path=dest, force=True)
|
|
465
|
+
|
|
466
|
+
assert result.status == "updated"
|
|
467
|
+
assert dest.read_bytes() == FAKE_PARQUET_V2
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def test_cache_clear_removes_files_and_session_meta(client, tmp_path, monkeypatch):
|
|
471
|
+
"""cache_clear deletes on-disk bulk files and drops session info cache."""
|
|
472
|
+
monkeypatch.setenv("EOLAS_LIBRARY", str(tmp_path))
|
|
473
|
+
|
|
474
|
+
dest = tmp_path / "nz_parcels.geo.parquet"
|
|
475
|
+
dest.write_bytes(FAKE_PARQUET)
|
|
476
|
+
_write_sidecar(dest, SNAPSHOT_V1)
|
|
477
|
+
|
|
478
|
+
client._meta_cache["nz_parcels"] = {"name": "nz_parcels"}
|
|
479
|
+
|
|
480
|
+
cleared = client.cache_clear("nz_parcels")
|
|
481
|
+
|
|
482
|
+
assert not dest.exists()
|
|
483
|
+
assert not pathlib.Path(str(dest) + ".eolas-meta.json").exists()
|
|
484
|
+
assert "nz_parcels" not in client._meta_cache
|
|
485
|
+
assert len(cleared["files"]) >= 2
|
|
486
|
+
assert cleared["meta_cleared"] == 1
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def test_cache_clear_meta_only(client):
|
|
490
|
+
"""files=False clears session metadata without deleting library files."""
|
|
491
|
+
client._meta_cache["nz_cpi"] = {"name": "nz_cpi"}
|
|
492
|
+
client._meta_cache["nz_parcels"] = {"name": "nz_parcels"}
|
|
493
|
+
|
|
494
|
+
cleared = client.cache_clear("nz_cpi", files=False)
|
|
495
|
+
|
|
496
|
+
assert "nz_cpi" not in client._meta_cache
|
|
497
|
+
assert "nz_parcels" in client._meta_cache
|
|
498
|
+
assert cleared["files"] == []
|
|
499
|
+
assert cleared["meta_cleared"] == 1
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|