epidatpy 0.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
epidatpy/__init__.py ADDED
@@ -0,0 +1,17 @@
1
+ """Fetch data from Delphi's API."""
2
+
3
+ # Make the linter happy about the unused variables
4
+ __all__ = [
5
+ "__version__",
6
+ "available_endpoints",
7
+ "EpiDataContext",
8
+ "CovidcastEpidata",
9
+ "EpiRange",
10
+ "InvalidArgumentException",
11
+ ]
12
+ __author__ = "Delphi Research Group"
13
+
14
+
15
+ from ._constants import __version__
16
+ from ._model import EpiRange, InvalidArgumentException
17
+ from .request import CovidcastEpidata, EpiDataContext, available_endpoints
epidatpy/_auth.py ADDED
@@ -0,0 +1,15 @@
1
+ import os
2
+ import warnings
3
+
4
+
5
+ def _get_api_key() -> str:
6
+ key = os.environ.get("DELPHI_EPIDATA_KEY", "")
7
+
8
+ if not key:
9
+ warnings.warn(
10
+ "DELPHI_EPIDATA_KEY environment variable not set. "
11
+ "Please set it to your Epidata API key to avoid rate limits. "
12
+ "You can get a free key at: https://api.delphi.cmu.edu/epidata/admin/registration_form"
13
+ )
14
+
15
+ return key
epidatpy/_call.py ADDED
@@ -0,0 +1,380 @@
1
+ from __future__ import annotations
2
+
3
+ import warnings
4
+ from collections.abc import Mapping, Sequence
5
+ from datetime import date
6
+ from io import StringIO
7
+ from os import environ
8
+ from typing import (
9
+ Any,
10
+ Final,
11
+ cast,
12
+ )
13
+ from urllib.parse import urlencode
14
+
15
+ from appdirs import user_cache_dir
16
+ from diskcache import Cache
17
+ from pandas import CategoricalDtype, DataFrame, Series, read_csv, to_datetime
18
+ from requests import Response, Session
19
+ from requests.auth import HTTPBasicAuth
20
+ from tenacity import retry, stop_after_attempt
21
+
22
+ from ._auth import _get_api_key
23
+ from ._constants import HTTP_HEADERS
24
+ from ._model import (
25
+ ApiVersion,
26
+ CastPostFilter,
27
+ EpidataFieldInfo,
28
+ EpidataFieldType,
29
+ EpiDataResponse,
30
+ EpiRangeParam,
31
+ OnlySupportsClassicFormatException,
32
+ add_endpoint_to_url,
33
+ cast_filter,
34
+ format_list,
35
+ )
36
+ from ._parse import (
37
+ fields_to_predicate,
38
+ parse_api_date,
39
+ parse_api_date_or_week,
40
+ parse_api_week,
41
+ )
42
+
43
+ CACHE_DIRECTORY = user_cache_dir(appname="epidatpy", appauthor="delphi")
44
+
45
+ if environ.get("USE_EPIDATPY_CACHE", None):
46
+ warnings.warn(
47
+ f"diskcache is being used (unset USE_EPIDATPY_CACHE if not intended). "
48
+ f"The cache directory is {CACHE_DIRECTORY}. "
49
+ f"The TTL is set to {environ.get('EPIDATPY_CACHE_MAX_AGE_DAYS', '7')} days."
50
+ )
51
+
52
+
53
+ @retry(reraise=True, stop=stop_after_attempt(2))
54
+ def _request_with_retry(
55
+ url: str,
56
+ params: Mapping[str, str],
57
+ session: Session | None = None,
58
+ stream: bool = False,
59
+ api_version: ApiVersion = "classic",
60
+ ) -> Response:
61
+ """Make request with a retry if an exception is thrown."""
62
+ key = _get_api_key()
63
+ if api_version == "cast":
64
+ # CAST API uses a token header instead of HTTP basic auth.
65
+ headers = {**HTTP_HEADERS, "token": key} if key else HTTP_HEADERS
66
+ auth = None
67
+ else:
68
+ headers = HTTP_HEADERS
69
+ auth = HTTPBasicAuth("epidata", key)
70
+
71
+ def call_impl(s: Session) -> Response:
72
+ res = s.get(url, params=params, headers=headers, stream=stream, auth=auth)
73
+ if res.status_code == 414:
74
+ return s.post(url, params=params, headers=headers, stream=stream, auth=auth)
75
+ return res
76
+
77
+ if session:
78
+ return call_impl(session)
79
+
80
+ with Session() as s:
81
+ return call_impl(s)
82
+
83
+
84
+ class EpiDataCall:
85
+ """epidata call representation"""
86
+
87
+ _base_url: Final[str]
88
+ _endpoint: Final[str]
89
+ _params: Final[Mapping[str, EpiRangeParam | None]]
90
+ _api_version: Final[ApiVersion]
91
+ _post_filter: Final[CastPostFilter | None]
92
+ _session: Final[Session | None]
93
+ meta: Final[Sequence[EpidataFieldInfo]]
94
+ meta_by_name: Final[Mapping[str, EpidataFieldInfo]]
95
+ only_supports_classic: Final[bool]
96
+ use_cache: Final[bool]
97
+
98
+ def __init__(
99
+ self,
100
+ base_url: str,
101
+ session: Session | None,
102
+ endpoint: str,
103
+ params: Mapping[str, EpiRangeParam | None],
104
+ meta: Sequence[EpidataFieldInfo] | None = None,
105
+ only_supports_classic: bool = False,
106
+ use_cache: bool | None = None,
107
+ cache_max_age_days: int | None = None,
108
+ api_version: ApiVersion = "classic",
109
+ post_filter: CastPostFilter | None = None,
110
+ ) -> None:
111
+ self._base_url = base_url
112
+ self._endpoint = endpoint
113
+ self._params = params
114
+ self._api_version = api_version
115
+ self._post_filter = post_filter
116
+ self._session = session
117
+ self.only_supports_classic = only_supports_classic
118
+ self.meta = meta or []
119
+ self.meta_by_name = {k.name: k for k in self.meta}
120
+ # Set use_cache from the constructor if present; otherwise check
121
+ # USE_EPIDATPY_CACHE, accepting various "truthy" values.
122
+ self.use_cache = (
123
+ use_cache
124
+ if use_cache is not None
125
+ else (environ.get("USE_EPIDATPY_CACHE", "").lower() in ["true", "t", "1"])
126
+ )
127
+ if cache_max_age_days is not None:
128
+ self.cache_max_age_days = cache_max_age_days
129
+ else:
130
+ env_days = environ.get("EPIDATPY_CACHE_MAX_AGE_DAYS", "7")
131
+ self.cache_max_age_days = int(env_days) if env_days.isdigit() else 7
132
+
133
+ def _formatted_parameters(
134
+ self,
135
+ fields: Sequence[str] | None = None,
136
+ ) -> Mapping[str, str]:
137
+ all_params = dict(self._params)
138
+ if fields:
139
+ all_params["fields"] = fields
140
+ return {k: format_list(v) for k, v in all_params.items() if v is not None}
141
+
142
+ def request_arguments(
143
+ self,
144
+ fields: Sequence[str] | None = None,
145
+ ) -> tuple[str, Mapping[str, str]]:
146
+ """Format this call into a (URL, params) tuple."""
147
+ formatted_params = self._formatted_parameters(fields)
148
+ full_url = add_endpoint_to_url(self._base_url, self._endpoint)
149
+ return full_url, formatted_params
150
+
151
+ def request_url(
152
+ self,
153
+ fields: Sequence[str] | None = None,
154
+ ) -> str:
155
+ """Format this call into a full HTTP request url with encoded parameters."""
156
+ u, p = self.request_arguments(fields)
157
+ query = urlencode(p)
158
+ if query:
159
+ return f"{u}?{query}"
160
+ return u
161
+
162
+ def __repr__(self) -> str:
163
+ return str(self)
164
+
165
+ def __str__(self) -> str:
166
+ return f"EpiDataCall(endpoint={self._endpoint}, params={self._formatted_parameters()})"
167
+
168
+ def _parse_value(
169
+ self,
170
+ key: str,
171
+ value: str | float | int | None,
172
+ disable_date_parsing: bool | None = False,
173
+ ) -> str | float | int | date | None:
174
+ meta = self.meta_by_name.get(key)
175
+ if not meta or value is None:
176
+ return value
177
+ if meta.type == EpidataFieldType.date_or_epiweek and not disable_date_parsing:
178
+ return parse_api_date_or_week(value)
179
+ if meta.type == EpidataFieldType.date and not disable_date_parsing:
180
+ return parse_api_date(value)
181
+ if meta.type == EpidataFieldType.epiweek and not disable_date_parsing:
182
+ return parse_api_week(value)
183
+ if meta.type == EpidataFieldType.bool:
184
+ return bool(value)
185
+ return value
186
+
187
+ def _parse_row(
188
+ self,
189
+ row: Mapping[str, str | float | int | None],
190
+ disable_date_parsing: bool | None = False,
191
+ ) -> Mapping[str, str | float | int | date | None]:
192
+ if not self.meta:
193
+ return row
194
+ return {k: self._parse_value(k, v, disable_date_parsing) for k, v in row.items()}
195
+
196
+ def with_base_url(self, base_url: str) -> EpiDataCall:
197
+ return EpiDataCall(
198
+ base_url,
199
+ self._session,
200
+ self._endpoint,
201
+ self._params,
202
+ meta=self.meta,
203
+ only_supports_classic=self.only_supports_classic,
204
+ use_cache=self.use_cache,
205
+ cache_max_age_days=self.cache_max_age_days,
206
+ api_version=self._api_version,
207
+ post_filter=self._post_filter,
208
+ )
209
+
210
+ def with_session(self, session: Session) -> EpiDataCall:
211
+ return EpiDataCall(
212
+ self._base_url,
213
+ session,
214
+ self._endpoint,
215
+ self._params,
216
+ meta=self.meta,
217
+ only_supports_classic=self.only_supports_classic,
218
+ use_cache=self.use_cache,
219
+ cache_max_age_days=self.cache_max_age_days,
220
+ api_version=self._api_version,
221
+ post_filter=self._post_filter,
222
+ )
223
+
224
+ def _call(
225
+ self,
226
+ fields: Sequence[str] | None = None,
227
+ stream: bool = False,
228
+ extra_params: Mapping[str, str] | None = None,
229
+ ) -> Response:
230
+ url, params = self.request_arguments(fields)
231
+ if extra_params:
232
+ params = {**params, **extra_params}
233
+ return _request_with_retry(url, params, self._session, stream, api_version=self._api_version)
234
+
235
+ def _get_cache_key(self, method: str) -> str:
236
+ cache_key = f"{self._endpoint} | {self._api_version} | {method}"
237
+ if self._params:
238
+ cache_key += f" | {str(dict(sorted(self._params.items())))}"
239
+ return cache_key
240
+
241
+ def classic(
242
+ self,
243
+ fields: Sequence[str] | None = None,
244
+ disable_date_parsing: bool | None = False,
245
+ disable_type_parsing: bool | None = False,
246
+ ) -> EpiDataResponse:
247
+ """Request and parse epidata in CLASSIC message format."""
248
+ try:
249
+ if self.use_cache:
250
+ with Cache(CACHE_DIRECTORY) as cache:
251
+ cache_key = self._get_cache_key("classic")
252
+ if cache_key in cache:
253
+ return cast(EpiDataResponse, cache[cache_key])
254
+ response = self._call(fields)
255
+ r = cast(EpiDataResponse, response.json())
256
+ if disable_type_parsing:
257
+ return r
258
+ epidata = r.get("epidata")
259
+ if epidata and isinstance(epidata, list) and len(epidata) > 0 and isinstance(epidata[0], dict):
260
+ r["epidata"] = [self._parse_row(row, disable_date_parsing=disable_date_parsing) for row in epidata]
261
+ if self.use_cache:
262
+ with Cache(CACHE_DIRECTORY) as cache:
263
+ cache_key = self._get_cache_key("classic")
264
+ cache.set(cache_key, r, expire=self.cache_max_age_days * 24 * 60 * 60)
265
+ return r
266
+ except Exception as e:
267
+ return {"result": 0, "message": f"error: {e}", "epidata": []}
268
+
269
+ def __call__(
270
+ self,
271
+ fields: Sequence[str] | None = None,
272
+ disable_date_parsing: bool | None = False,
273
+ ) -> EpiDataResponse | DataFrame:
274
+ """Request and parse epidata in df message format."""
275
+ if self.only_supports_classic:
276
+ return self.classic(
277
+ fields,
278
+ disable_date_parsing=disable_date_parsing,
279
+ disable_type_parsing=False,
280
+ )
281
+ return self.df(fields, disable_date_parsing=disable_date_parsing)
282
+
283
+ def df(
284
+ self,
285
+ fields: Sequence[str] | None = None,
286
+ disable_date_parsing: bool | None = False,
287
+ ) -> DataFrame:
288
+ """Request and parse epidata as a pandas data frame"""
289
+ if self.only_supports_classic:
290
+ raise OnlySupportsClassicFormatException()
291
+
292
+ if self.use_cache:
293
+ with Cache(CACHE_DIRECTORY) as cache:
294
+ cache_key = self._get_cache_key("df")
295
+ if cache_key in cache:
296
+ return cast(DataFrame, cache[cache_key])
297
+
298
+ pred = fields_to_predicate(fields)
299
+ columns: list[str] = [info.name for info in self.meta if pred(info.name)]
300
+ if self._api_version == "cast":
301
+ # CAST endpoints only speak CSV.
302
+ response = self._call(fields, extra_params={"format": "csv"})
303
+ response.raise_for_status()
304
+ body = response.text
305
+ if body.strip():
306
+ df = read_csv(StringIO(body), dtype=str)
307
+ # Keep only fields in meta that exist in the response, in meta order.
308
+ df_cols = set(df.columns)
309
+ cols_in_df = [c for c in columns if c in df_cols]
310
+ df = df[cols_in_df] if cols_in_df else df
311
+ else:
312
+ df = DataFrame(columns=columns or None)
313
+ else:
314
+ json = self.classic(fields, disable_type_parsing=True)
315
+ rows = json.get("epidata", [])
316
+ df = DataFrame(rows, columns=columns or None)
317
+
318
+ data_types: dict[str, Any] = {}
319
+ time_fields: list[EpidataFieldInfo] = []
320
+ for info in self.meta:
321
+ if not pred(info.name):
322
+ continue
323
+ if info.name not in df.columns:
324
+ # CAST responses may omit source-specific columns; skip them.
325
+ continue
326
+ if info.type == EpidataFieldType.bool:
327
+ data_types[info.name] = bool
328
+ elif info.type == EpidataFieldType.categorical:
329
+ data_types[info.name] = CategoricalDtype(
330
+ categories=Series(info.categories) if info.categories else None,
331
+ ordered=True,
332
+ )
333
+ elif info.type == EpidataFieldType.int:
334
+ data_types[info.name] = "Int64"
335
+ elif info.type in (
336
+ EpidataFieldType.date,
337
+ EpidataFieldType.epiweek,
338
+ EpidataFieldType.date_or_epiweek,
339
+ ):
340
+ data_types[info.name] = "string"
341
+ time_fields.append(info)
342
+ elif info.type == EpidataFieldType.float:
343
+ data_types[info.name] = "Float64"
344
+ else:
345
+ data_types[info.name] = "string"
346
+ if data_types:
347
+ df = df.astype(data_types)
348
+ if not disable_date_parsing:
349
+ for info in time_fields:
350
+ if info.type == EpidataFieldType.epiweek:
351
+ continue
352
+ # Try known date formats in priority order; keep as string if all
353
+ # fail. The try/except is needed because the time field might be
354
+ # date_or_epiweek, and the CAST CSV path returns timestamps
355
+ # (e.g. "2024-04-18 00:00:00").
356
+ parsed = False
357
+ for fmt in ("%Y-%m-%d", "%Y%m%d", "%Y-%m-%d %H:%M:%S"):
358
+ try:
359
+ df[info.name] = to_datetime(df[info.name], format=fmt)
360
+ parsed = True
361
+ break
362
+ except ValueError:
363
+ continue
364
+ if not parsed:
365
+ # Last resort: let pandas infer (slower but flexible).
366
+ try:
367
+ df[info.name] = to_datetime(df[info.name])
368
+ except (ValueError, TypeError):
369
+ pass
370
+
371
+ if self.use_cache:
372
+ with Cache(CACHE_DIRECTORY) as cache:
373
+ cache_key = self._get_cache_key("df")
374
+ cache.set(cache_key, df, expire=self.cache_max_age_days * 24 * 60 * 60)
375
+
376
+ if self._post_filter is not None:
377
+ geo_values, reference_time, report_time = self._post_filter
378
+ df = cast_filter(df, geo_values=geo_values, reference_time=reference_time, report_time=report_time)
379
+
380
+ return df
epidatpy/_constants.py ADDED
@@ -0,0 +1,7 @@
1
+ from importlib.metadata import version
2
+ from typing import Final
3
+
4
+ __version__: Final = version("epidatpy")
5
+ HTTP_HEADERS: Final = {"User-Agent": f"epidatpy/{__version__}"}
6
+ BASE_URL: Final = "https://api.delphi.cmu.edu/epidata/"
7
+ CAST_BASE_URL: Final = "https://delphi.cmu.edu/epidata/v5/"