epidatpy 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- epidatpy/__init__.py +17 -0
- epidatpy/_auth.py +15 -0
- epidatpy/_call.py +380 -0
- epidatpy/_constants.py +7 -0
- epidatpy/_covidcast.py +397 -0
- epidatpy/_endpoints.py +1731 -0
- epidatpy/_model.py +189 -0
- epidatpy/_parse.py +133 -0
- epidatpy/request.py +54 -0
- epidatpy-0.5.0.dist-info/METADATA +184 -0
- epidatpy-0.5.0.dist-info/RECORD +14 -0
- epidatpy-0.5.0.dist-info/WHEEL +5 -0
- epidatpy-0.5.0.dist-info/licenses/LICENSE +21 -0
- epidatpy-0.5.0.dist-info/top_level.txt +1 -0
epidatpy/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Fetch data from Delphi's API."""
|
|
2
|
+
|
|
3
|
+
# Make the linter happy about the unused variables
|
|
4
|
+
__all__ = [
|
|
5
|
+
"__version__",
|
|
6
|
+
"available_endpoints",
|
|
7
|
+
"EpiDataContext",
|
|
8
|
+
"CovidcastEpidata",
|
|
9
|
+
"EpiRange",
|
|
10
|
+
"InvalidArgumentException",
|
|
11
|
+
]
|
|
12
|
+
__author__ = "Delphi Research Group"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
from ._constants import __version__
|
|
16
|
+
from ._model import EpiRange, InvalidArgumentException
|
|
17
|
+
from .request import CovidcastEpidata, EpiDataContext, available_endpoints
|
epidatpy/_auth.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import warnings
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def _get_api_key() -> str:
|
|
6
|
+
key = os.environ.get("DELPHI_EPIDATA_KEY", "")
|
|
7
|
+
|
|
8
|
+
if not key:
|
|
9
|
+
warnings.warn(
|
|
10
|
+
"DELPHI_EPIDATA_KEY environment variable not set. "
|
|
11
|
+
"Please set it to your Epidata API key to avoid rate limits. "
|
|
12
|
+
"You can get a free key at: https://api.delphi.cmu.edu/epidata/admin/registration_form"
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
return key
|
epidatpy/_call.py
ADDED
|
@@ -0,0 +1,380 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import warnings
|
|
4
|
+
from collections.abc import Mapping, Sequence
|
|
5
|
+
from datetime import date
|
|
6
|
+
from io import StringIO
|
|
7
|
+
from os import environ
|
|
8
|
+
from typing import (
|
|
9
|
+
Any,
|
|
10
|
+
Final,
|
|
11
|
+
cast,
|
|
12
|
+
)
|
|
13
|
+
from urllib.parse import urlencode
|
|
14
|
+
|
|
15
|
+
from appdirs import user_cache_dir
|
|
16
|
+
from diskcache import Cache
|
|
17
|
+
from pandas import CategoricalDtype, DataFrame, Series, read_csv, to_datetime
|
|
18
|
+
from requests import Response, Session
|
|
19
|
+
from requests.auth import HTTPBasicAuth
|
|
20
|
+
from tenacity import retry, stop_after_attempt
|
|
21
|
+
|
|
22
|
+
from ._auth import _get_api_key
|
|
23
|
+
from ._constants import HTTP_HEADERS
|
|
24
|
+
from ._model import (
|
|
25
|
+
ApiVersion,
|
|
26
|
+
CastPostFilter,
|
|
27
|
+
EpidataFieldInfo,
|
|
28
|
+
EpidataFieldType,
|
|
29
|
+
EpiDataResponse,
|
|
30
|
+
EpiRangeParam,
|
|
31
|
+
OnlySupportsClassicFormatException,
|
|
32
|
+
add_endpoint_to_url,
|
|
33
|
+
cast_filter,
|
|
34
|
+
format_list,
|
|
35
|
+
)
|
|
36
|
+
from ._parse import (
|
|
37
|
+
fields_to_predicate,
|
|
38
|
+
parse_api_date,
|
|
39
|
+
parse_api_date_or_week,
|
|
40
|
+
parse_api_week,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
CACHE_DIRECTORY = user_cache_dir(appname="epidatpy", appauthor="delphi")
|
|
44
|
+
|
|
45
|
+
if environ.get("USE_EPIDATPY_CACHE", None):
|
|
46
|
+
warnings.warn(
|
|
47
|
+
f"diskcache is being used (unset USE_EPIDATPY_CACHE if not intended). "
|
|
48
|
+
f"The cache directory is {CACHE_DIRECTORY}. "
|
|
49
|
+
f"The TTL is set to {environ.get('EPIDATPY_CACHE_MAX_AGE_DAYS', '7')} days."
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@retry(reraise=True, stop=stop_after_attempt(2))
|
|
54
|
+
def _request_with_retry(
|
|
55
|
+
url: str,
|
|
56
|
+
params: Mapping[str, str],
|
|
57
|
+
session: Session | None = None,
|
|
58
|
+
stream: bool = False,
|
|
59
|
+
api_version: ApiVersion = "classic",
|
|
60
|
+
) -> Response:
|
|
61
|
+
"""Make request with a retry if an exception is thrown."""
|
|
62
|
+
key = _get_api_key()
|
|
63
|
+
if api_version == "cast":
|
|
64
|
+
# CAST API uses a token header instead of HTTP basic auth.
|
|
65
|
+
headers = {**HTTP_HEADERS, "token": key} if key else HTTP_HEADERS
|
|
66
|
+
auth = None
|
|
67
|
+
else:
|
|
68
|
+
headers = HTTP_HEADERS
|
|
69
|
+
auth = HTTPBasicAuth("epidata", key)
|
|
70
|
+
|
|
71
|
+
def call_impl(s: Session) -> Response:
|
|
72
|
+
res = s.get(url, params=params, headers=headers, stream=stream, auth=auth)
|
|
73
|
+
if res.status_code == 414:
|
|
74
|
+
return s.post(url, params=params, headers=headers, stream=stream, auth=auth)
|
|
75
|
+
return res
|
|
76
|
+
|
|
77
|
+
if session:
|
|
78
|
+
return call_impl(session)
|
|
79
|
+
|
|
80
|
+
with Session() as s:
|
|
81
|
+
return call_impl(s)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class EpiDataCall:
|
|
85
|
+
"""epidata call representation"""
|
|
86
|
+
|
|
87
|
+
_base_url: Final[str]
|
|
88
|
+
_endpoint: Final[str]
|
|
89
|
+
_params: Final[Mapping[str, EpiRangeParam | None]]
|
|
90
|
+
_api_version: Final[ApiVersion]
|
|
91
|
+
_post_filter: Final[CastPostFilter | None]
|
|
92
|
+
_session: Final[Session | None]
|
|
93
|
+
meta: Final[Sequence[EpidataFieldInfo]]
|
|
94
|
+
meta_by_name: Final[Mapping[str, EpidataFieldInfo]]
|
|
95
|
+
only_supports_classic: Final[bool]
|
|
96
|
+
use_cache: Final[bool]
|
|
97
|
+
|
|
98
|
+
def __init__(
|
|
99
|
+
self,
|
|
100
|
+
base_url: str,
|
|
101
|
+
session: Session | None,
|
|
102
|
+
endpoint: str,
|
|
103
|
+
params: Mapping[str, EpiRangeParam | None],
|
|
104
|
+
meta: Sequence[EpidataFieldInfo] | None = None,
|
|
105
|
+
only_supports_classic: bool = False,
|
|
106
|
+
use_cache: bool | None = None,
|
|
107
|
+
cache_max_age_days: int | None = None,
|
|
108
|
+
api_version: ApiVersion = "classic",
|
|
109
|
+
post_filter: CastPostFilter | None = None,
|
|
110
|
+
) -> None:
|
|
111
|
+
self._base_url = base_url
|
|
112
|
+
self._endpoint = endpoint
|
|
113
|
+
self._params = params
|
|
114
|
+
self._api_version = api_version
|
|
115
|
+
self._post_filter = post_filter
|
|
116
|
+
self._session = session
|
|
117
|
+
self.only_supports_classic = only_supports_classic
|
|
118
|
+
self.meta = meta or []
|
|
119
|
+
self.meta_by_name = {k.name: k for k in self.meta}
|
|
120
|
+
# Set use_cache from the constructor if present; otherwise check
|
|
121
|
+
# USE_EPIDATPY_CACHE, accepting various "truthy" values.
|
|
122
|
+
self.use_cache = (
|
|
123
|
+
use_cache
|
|
124
|
+
if use_cache is not None
|
|
125
|
+
else (environ.get("USE_EPIDATPY_CACHE", "").lower() in ["true", "t", "1"])
|
|
126
|
+
)
|
|
127
|
+
if cache_max_age_days is not None:
|
|
128
|
+
self.cache_max_age_days = cache_max_age_days
|
|
129
|
+
else:
|
|
130
|
+
env_days = environ.get("EPIDATPY_CACHE_MAX_AGE_DAYS", "7")
|
|
131
|
+
self.cache_max_age_days = int(env_days) if env_days.isdigit() else 7
|
|
132
|
+
|
|
133
|
+
def _formatted_parameters(
|
|
134
|
+
self,
|
|
135
|
+
fields: Sequence[str] | None = None,
|
|
136
|
+
) -> Mapping[str, str]:
|
|
137
|
+
all_params = dict(self._params)
|
|
138
|
+
if fields:
|
|
139
|
+
all_params["fields"] = fields
|
|
140
|
+
return {k: format_list(v) for k, v in all_params.items() if v is not None}
|
|
141
|
+
|
|
142
|
+
def request_arguments(
|
|
143
|
+
self,
|
|
144
|
+
fields: Sequence[str] | None = None,
|
|
145
|
+
) -> tuple[str, Mapping[str, str]]:
|
|
146
|
+
"""Format this call into a (URL, params) tuple."""
|
|
147
|
+
formatted_params = self._formatted_parameters(fields)
|
|
148
|
+
full_url = add_endpoint_to_url(self._base_url, self._endpoint)
|
|
149
|
+
return full_url, formatted_params
|
|
150
|
+
|
|
151
|
+
def request_url(
|
|
152
|
+
self,
|
|
153
|
+
fields: Sequence[str] | None = None,
|
|
154
|
+
) -> str:
|
|
155
|
+
"""Format this call into a full HTTP request url with encoded parameters."""
|
|
156
|
+
u, p = self.request_arguments(fields)
|
|
157
|
+
query = urlencode(p)
|
|
158
|
+
if query:
|
|
159
|
+
return f"{u}?{query}"
|
|
160
|
+
return u
|
|
161
|
+
|
|
162
|
+
def __repr__(self) -> str:
|
|
163
|
+
return str(self)
|
|
164
|
+
|
|
165
|
+
def __str__(self) -> str:
|
|
166
|
+
return f"EpiDataCall(endpoint={self._endpoint}, params={self._formatted_parameters()})"
|
|
167
|
+
|
|
168
|
+
def _parse_value(
|
|
169
|
+
self,
|
|
170
|
+
key: str,
|
|
171
|
+
value: str | float | int | None,
|
|
172
|
+
disable_date_parsing: bool | None = False,
|
|
173
|
+
) -> str | float | int | date | None:
|
|
174
|
+
meta = self.meta_by_name.get(key)
|
|
175
|
+
if not meta or value is None:
|
|
176
|
+
return value
|
|
177
|
+
if meta.type == EpidataFieldType.date_or_epiweek and not disable_date_parsing:
|
|
178
|
+
return parse_api_date_or_week(value)
|
|
179
|
+
if meta.type == EpidataFieldType.date and not disable_date_parsing:
|
|
180
|
+
return parse_api_date(value)
|
|
181
|
+
if meta.type == EpidataFieldType.epiweek and not disable_date_parsing:
|
|
182
|
+
return parse_api_week(value)
|
|
183
|
+
if meta.type == EpidataFieldType.bool:
|
|
184
|
+
return bool(value)
|
|
185
|
+
return value
|
|
186
|
+
|
|
187
|
+
def _parse_row(
|
|
188
|
+
self,
|
|
189
|
+
row: Mapping[str, str | float | int | None],
|
|
190
|
+
disable_date_parsing: bool | None = False,
|
|
191
|
+
) -> Mapping[str, str | float | int | date | None]:
|
|
192
|
+
if not self.meta:
|
|
193
|
+
return row
|
|
194
|
+
return {k: self._parse_value(k, v, disable_date_parsing) for k, v in row.items()}
|
|
195
|
+
|
|
196
|
+
def with_base_url(self, base_url: str) -> EpiDataCall:
|
|
197
|
+
return EpiDataCall(
|
|
198
|
+
base_url,
|
|
199
|
+
self._session,
|
|
200
|
+
self._endpoint,
|
|
201
|
+
self._params,
|
|
202
|
+
meta=self.meta,
|
|
203
|
+
only_supports_classic=self.only_supports_classic,
|
|
204
|
+
use_cache=self.use_cache,
|
|
205
|
+
cache_max_age_days=self.cache_max_age_days,
|
|
206
|
+
api_version=self._api_version,
|
|
207
|
+
post_filter=self._post_filter,
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
def with_session(self, session: Session) -> EpiDataCall:
|
|
211
|
+
return EpiDataCall(
|
|
212
|
+
self._base_url,
|
|
213
|
+
session,
|
|
214
|
+
self._endpoint,
|
|
215
|
+
self._params,
|
|
216
|
+
meta=self.meta,
|
|
217
|
+
only_supports_classic=self.only_supports_classic,
|
|
218
|
+
use_cache=self.use_cache,
|
|
219
|
+
cache_max_age_days=self.cache_max_age_days,
|
|
220
|
+
api_version=self._api_version,
|
|
221
|
+
post_filter=self._post_filter,
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
def _call(
|
|
225
|
+
self,
|
|
226
|
+
fields: Sequence[str] | None = None,
|
|
227
|
+
stream: bool = False,
|
|
228
|
+
extra_params: Mapping[str, str] | None = None,
|
|
229
|
+
) -> Response:
|
|
230
|
+
url, params = self.request_arguments(fields)
|
|
231
|
+
if extra_params:
|
|
232
|
+
params = {**params, **extra_params}
|
|
233
|
+
return _request_with_retry(url, params, self._session, stream, api_version=self._api_version)
|
|
234
|
+
|
|
235
|
+
def _get_cache_key(self, method: str) -> str:
|
|
236
|
+
cache_key = f"{self._endpoint} | {self._api_version} | {method}"
|
|
237
|
+
if self._params:
|
|
238
|
+
cache_key += f" | {str(dict(sorted(self._params.items())))}"
|
|
239
|
+
return cache_key
|
|
240
|
+
|
|
241
|
+
def classic(
|
|
242
|
+
self,
|
|
243
|
+
fields: Sequence[str] | None = None,
|
|
244
|
+
disable_date_parsing: bool | None = False,
|
|
245
|
+
disable_type_parsing: bool | None = False,
|
|
246
|
+
) -> EpiDataResponse:
|
|
247
|
+
"""Request and parse epidata in CLASSIC message format."""
|
|
248
|
+
try:
|
|
249
|
+
if self.use_cache:
|
|
250
|
+
with Cache(CACHE_DIRECTORY) as cache:
|
|
251
|
+
cache_key = self._get_cache_key("classic")
|
|
252
|
+
if cache_key in cache:
|
|
253
|
+
return cast(EpiDataResponse, cache[cache_key])
|
|
254
|
+
response = self._call(fields)
|
|
255
|
+
r = cast(EpiDataResponse, response.json())
|
|
256
|
+
if disable_type_parsing:
|
|
257
|
+
return r
|
|
258
|
+
epidata = r.get("epidata")
|
|
259
|
+
if epidata and isinstance(epidata, list) and len(epidata) > 0 and isinstance(epidata[0], dict):
|
|
260
|
+
r["epidata"] = [self._parse_row(row, disable_date_parsing=disable_date_parsing) for row in epidata]
|
|
261
|
+
if self.use_cache:
|
|
262
|
+
with Cache(CACHE_DIRECTORY) as cache:
|
|
263
|
+
cache_key = self._get_cache_key("classic")
|
|
264
|
+
cache.set(cache_key, r, expire=self.cache_max_age_days * 24 * 60 * 60)
|
|
265
|
+
return r
|
|
266
|
+
except Exception as e:
|
|
267
|
+
return {"result": 0, "message": f"error: {e}", "epidata": []}
|
|
268
|
+
|
|
269
|
+
def __call__(
|
|
270
|
+
self,
|
|
271
|
+
fields: Sequence[str] | None = None,
|
|
272
|
+
disable_date_parsing: bool | None = False,
|
|
273
|
+
) -> EpiDataResponse | DataFrame:
|
|
274
|
+
"""Request and parse epidata in df message format."""
|
|
275
|
+
if self.only_supports_classic:
|
|
276
|
+
return self.classic(
|
|
277
|
+
fields,
|
|
278
|
+
disable_date_parsing=disable_date_parsing,
|
|
279
|
+
disable_type_parsing=False,
|
|
280
|
+
)
|
|
281
|
+
return self.df(fields, disable_date_parsing=disable_date_parsing)
|
|
282
|
+
|
|
283
|
+
def df(
|
|
284
|
+
self,
|
|
285
|
+
fields: Sequence[str] | None = None,
|
|
286
|
+
disable_date_parsing: bool | None = False,
|
|
287
|
+
) -> DataFrame:
|
|
288
|
+
"""Request and parse epidata as a pandas data frame"""
|
|
289
|
+
if self.only_supports_classic:
|
|
290
|
+
raise OnlySupportsClassicFormatException()
|
|
291
|
+
|
|
292
|
+
if self.use_cache:
|
|
293
|
+
with Cache(CACHE_DIRECTORY) as cache:
|
|
294
|
+
cache_key = self._get_cache_key("df")
|
|
295
|
+
if cache_key in cache:
|
|
296
|
+
return cast(DataFrame, cache[cache_key])
|
|
297
|
+
|
|
298
|
+
pred = fields_to_predicate(fields)
|
|
299
|
+
columns: list[str] = [info.name for info in self.meta if pred(info.name)]
|
|
300
|
+
if self._api_version == "cast":
|
|
301
|
+
# CAST endpoints only speak CSV.
|
|
302
|
+
response = self._call(fields, extra_params={"format": "csv"})
|
|
303
|
+
response.raise_for_status()
|
|
304
|
+
body = response.text
|
|
305
|
+
if body.strip():
|
|
306
|
+
df = read_csv(StringIO(body), dtype=str)
|
|
307
|
+
# Keep only fields in meta that exist in the response, in meta order.
|
|
308
|
+
df_cols = set(df.columns)
|
|
309
|
+
cols_in_df = [c for c in columns if c in df_cols]
|
|
310
|
+
df = df[cols_in_df] if cols_in_df else df
|
|
311
|
+
else:
|
|
312
|
+
df = DataFrame(columns=columns or None)
|
|
313
|
+
else:
|
|
314
|
+
json = self.classic(fields, disable_type_parsing=True)
|
|
315
|
+
rows = json.get("epidata", [])
|
|
316
|
+
df = DataFrame(rows, columns=columns or None)
|
|
317
|
+
|
|
318
|
+
data_types: dict[str, Any] = {}
|
|
319
|
+
time_fields: list[EpidataFieldInfo] = []
|
|
320
|
+
for info in self.meta:
|
|
321
|
+
if not pred(info.name):
|
|
322
|
+
continue
|
|
323
|
+
if info.name not in df.columns:
|
|
324
|
+
# CAST responses may omit source-specific columns; skip them.
|
|
325
|
+
continue
|
|
326
|
+
if info.type == EpidataFieldType.bool:
|
|
327
|
+
data_types[info.name] = bool
|
|
328
|
+
elif info.type == EpidataFieldType.categorical:
|
|
329
|
+
data_types[info.name] = CategoricalDtype(
|
|
330
|
+
categories=Series(info.categories) if info.categories else None,
|
|
331
|
+
ordered=True,
|
|
332
|
+
)
|
|
333
|
+
elif info.type == EpidataFieldType.int:
|
|
334
|
+
data_types[info.name] = "Int64"
|
|
335
|
+
elif info.type in (
|
|
336
|
+
EpidataFieldType.date,
|
|
337
|
+
EpidataFieldType.epiweek,
|
|
338
|
+
EpidataFieldType.date_or_epiweek,
|
|
339
|
+
):
|
|
340
|
+
data_types[info.name] = "string"
|
|
341
|
+
time_fields.append(info)
|
|
342
|
+
elif info.type == EpidataFieldType.float:
|
|
343
|
+
data_types[info.name] = "Float64"
|
|
344
|
+
else:
|
|
345
|
+
data_types[info.name] = "string"
|
|
346
|
+
if data_types:
|
|
347
|
+
df = df.astype(data_types)
|
|
348
|
+
if not disable_date_parsing:
|
|
349
|
+
for info in time_fields:
|
|
350
|
+
if info.type == EpidataFieldType.epiweek:
|
|
351
|
+
continue
|
|
352
|
+
# Try known date formats in priority order; keep as string if all
|
|
353
|
+
# fail. The try/except is needed because the time field might be
|
|
354
|
+
# date_or_epiweek, and the CAST CSV path returns timestamps
|
|
355
|
+
# (e.g. "2024-04-18 00:00:00").
|
|
356
|
+
parsed = False
|
|
357
|
+
for fmt in ("%Y-%m-%d", "%Y%m%d", "%Y-%m-%d %H:%M:%S"):
|
|
358
|
+
try:
|
|
359
|
+
df[info.name] = to_datetime(df[info.name], format=fmt)
|
|
360
|
+
parsed = True
|
|
361
|
+
break
|
|
362
|
+
except ValueError:
|
|
363
|
+
continue
|
|
364
|
+
if not parsed:
|
|
365
|
+
# Last resort: let pandas infer (slower but flexible).
|
|
366
|
+
try:
|
|
367
|
+
df[info.name] = to_datetime(df[info.name])
|
|
368
|
+
except (ValueError, TypeError):
|
|
369
|
+
pass
|
|
370
|
+
|
|
371
|
+
if self.use_cache:
|
|
372
|
+
with Cache(CACHE_DIRECTORY) as cache:
|
|
373
|
+
cache_key = self._get_cache_key("df")
|
|
374
|
+
cache.set(cache_key, df, expire=self.cache_max_age_days * 24 * 60 * 60)
|
|
375
|
+
|
|
376
|
+
if self._post_filter is not None:
|
|
377
|
+
geo_values, reference_time, report_time = self._post_filter
|
|
378
|
+
df = cast_filter(df, geo_values=geo_values, reference_time=reference_time, report_time=report_time)
|
|
379
|
+
|
|
380
|
+
return df
|
epidatpy/_constants.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
from importlib.metadata import version
|
|
2
|
+
from typing import Final
|
|
3
|
+
|
|
4
|
+
__version__: Final = version("epidatpy")
|
|
5
|
+
HTTP_HEADERS: Final = {"User-Agent": f"epidatpy/{__version__}"}
|
|
6
|
+
BASE_URL: Final = "https://api.delphi.cmu.edu/epidata/"
|
|
7
|
+
CAST_BASE_URL: Final = "https://delphi.cmu.edu/epidata/v5/"
|