aadtkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aadtkit/__init__.py +134 -0
- aadtkit/__main__.py +7 -0
- aadtkit/_version.py +1 -0
- aadtkit/aadt.py +282 -0
- aadtkit/classification.py +105 -0
- aadtkit/cli.py +307 -0
- aadtkit/errors.py +17 -0
- aadtkit/expand.py +148 -0
- aadtkit/factors.py +372 -0
- aadtkit/io.py +312 -0
- aadtkit/peak.py +154 -0
- aadtkit/py.typed +0 -0
- aadtkit/quality.py +231 -0
- aadtkit-0.1.0.dist-info/METADATA +225 -0
- aadtkit-0.1.0.dist-info/RECORD +19 -0
- aadtkit-0.1.0.dist-info/WHEEL +5 -0
- aadtkit-0.1.0.dist-info/entry_points.txt +2 -0
- aadtkit-0.1.0.dist-info/licenses/LICENSE +21 -0
- aadtkit-0.1.0.dist-info/top_level.txt +1 -0
aadtkit/factors.py
ADDED
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
"""Expansion factors, factor groups and axle-correction factors.
|
|
2
|
+
|
|
3
|
+
Definitions follow the FHWA *Traffic Monitoring Guide* (TMG 2022, Section
|
|
4
|
+
3.1.4.7, Steps 4-5):
|
|
5
|
+
|
|
6
|
+
* monthly factor of a station: ``M_m = AADT / MADT_m``;
|
|
7
|
+
* day-of-week factor: ``D_j = AADT / ADT_j`` with ``ADT_j`` the annual mean of
|
|
8
|
+
the monthly average day-of-week volumes (MADW) of day *j*;
|
|
9
|
+
* month x day-of-week factor: ``F_mj = AADT / MADW_mj`` (the most specific
|
|
10
|
+
form; it equals ``M_m * D_j`` when seasonal and weekly patterns are
|
|
11
|
+
multiplicatively separable);
|
|
12
|
+
* group factors are the arithmetic mean of the factors of the continuous
|
|
13
|
+
count stations in the group (TMG 2022, 3.1.4.7, Step 5), and their
|
|
14
|
+
precision is the 95 % confidence half-width ``t * s / sqrt(n)``
|
|
15
|
+
(TMG 2022, 3.1.4.5 and 3.2.6.2).
|
|
16
|
+
|
|
17
|
+
Factor groups can be formed by clustering stations on their factor patterns
|
|
18
|
+
(:func:`cluster_stations`, k-means), which is one of the approaches discussed
|
|
19
|
+
in TMG 2022 Section 3.1.4 and NCHRP Research Report 1081 (2024).
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import datetime as dt
|
|
25
|
+
import math
|
|
26
|
+
import random
|
|
27
|
+
from dataclasses import dataclass, field
|
|
28
|
+
from typing import Dict, List, Mapping, Optional, Sequence, Tuple
|
|
29
|
+
|
|
30
|
+
from .aadt import AADTResult
|
|
31
|
+
from .errors import InsufficientDataError
|
|
32
|
+
|
|
33
|
+
__all__ = [
|
|
34
|
+
"DEFAULT_AXLES_PER_CLASS",
|
|
35
|
+
"ClusterResult",
|
|
36
|
+
"FactorGroup",
|
|
37
|
+
"FactorSet",
|
|
38
|
+
"FactorStats",
|
|
39
|
+
"axle_correction_factor",
|
|
40
|
+
"axle_correction_from_classes",
|
|
41
|
+
"cluster_stations",
|
|
42
|
+
"factors_from_result",
|
|
43
|
+
"group_factors",
|
|
44
|
+
"t_critical_95",
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass
|
|
49
|
+
class FactorSet:
|
|
50
|
+
"""Multiplicative expansion factors of one station or factor group.
|
|
51
|
+
|
|
52
|
+
Attributes:
|
|
53
|
+
monthly: ``{month (1..12): AADT / MADT_m}``.
|
|
54
|
+
dow: ``{day of week (0=Mon..6=Sun): AADT / ADT_j}``.
|
|
55
|
+
month_dow: optional ``{(month, dow): AADT / MADW_mj}``.
|
|
56
|
+
name: station id or group label.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
monthly: Dict[int, float]
|
|
60
|
+
dow: Dict[int, float]
|
|
61
|
+
month_dow: Dict[Tuple[int, int], float] = field(default_factory=dict)
|
|
62
|
+
name: str = ""
|
|
63
|
+
|
|
64
|
+
def factor(self, day: dt.date, *, use_month_dow: bool = True) -> float:
|
|
65
|
+
"""Combined factor that converts a 24-h volume on ``day`` to AADT.
|
|
66
|
+
|
|
67
|
+
Uses the month x day-of-week factor when available (and requested),
|
|
68
|
+
otherwise the product of the monthly and day-of-week factors, which is
|
|
69
|
+
how TMG 2022 Section 3.2.8 applies them ("first factored for
|
|
70
|
+
seasonality and then for the day of week").
|
|
71
|
+
"""
|
|
72
|
+
m, j = day.month, day.weekday()
|
|
73
|
+
if use_month_dow and (m, j) in self.month_dow:
|
|
74
|
+
return self.month_dow[(m, j)]
|
|
75
|
+
try:
|
|
76
|
+
return self.monthly[m] * self.dow[j]
|
|
77
|
+
except KeyError as exc:
|
|
78
|
+
raise InsufficientDataError(
|
|
79
|
+
f"factor set {self.name!r} has no factor for month {m} / dow {j}"
|
|
80
|
+
) from exc
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass
|
|
84
|
+
class FactorStats:
|
|
85
|
+
"""Dispersion of one factor across the stations of a group."""
|
|
86
|
+
|
|
87
|
+
mean: float
|
|
88
|
+
sd: float
|
|
89
|
+
n: int
|
|
90
|
+
cv: float
|
|
91
|
+
ci_half_width: float
|
|
92
|
+
relative_precision: float
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@dataclass
|
|
96
|
+
class FactorGroup(FactorSet):
|
|
97
|
+
"""Group-average factors plus per-factor statistics and member list."""
|
|
98
|
+
|
|
99
|
+
members: List[str] = field(default_factory=list)
|
|
100
|
+
monthly_stats: Dict[int, FactorStats] = field(default_factory=dict)
|
|
101
|
+
dow_stats: Dict[int, FactorStats] = field(default_factory=dict)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def factors_from_result(result: AADTResult, name: str = "") -> FactorSet:
|
|
105
|
+
"""Derive monthly, day-of-week and month x day-of-week factors from a CCS result."""
|
|
106
|
+
if not result.madt or not result.madw:
|
|
107
|
+
raise InsufficientDataError("result carries no monthly/day-of-week averages")
|
|
108
|
+
a = result.aadt
|
|
109
|
+
monthly = {m: a / v for m, v in result.madt.items() if v > 0}
|
|
110
|
+
dow = {j: a / v for j, v in result.dow_average().items() if v > 0}
|
|
111
|
+
month_dow = {k: a / v for k, v in result.madw.items() if v > 0}
|
|
112
|
+
return FactorSet(monthly=monthly, dow=dow, month_dow=month_dow, name=name)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
# Two-sided 95 % Student t critical values t_{0.975, df}, df = 1..30.
|
|
116
|
+
_T975 = (
|
|
117
|
+
12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228,
|
|
118
|
+
2.201, 2.179, 2.160, 2.145, 2.131, 2.120, 2.110, 2.101, 2.093, 2.086,
|
|
119
|
+
2.080, 2.074, 2.069, 2.064, 2.060, 2.056, 2.052, 2.048, 2.045, 2.042,
|
|
120
|
+
) # fmt: skip
|
|
121
|
+
_T975_LARGE = ((40, 2.021), (60, 2.000), (120, 1.980))
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def t_critical_95(df: int) -> float:
|
|
125
|
+
"""Two-sided 95 % Student-t critical value for ``df`` degrees of freedom.
|
|
126
|
+
|
|
127
|
+
Tabulated for df <= 30, 40, 60, 120; interpolated linearly in ``1/df``
|
|
128
|
+
in between and tending to 1.960 for df -> infinity.
|
|
129
|
+
"""
|
|
130
|
+
if df < 1:
|
|
131
|
+
raise ValueError("df must be >= 1")
|
|
132
|
+
if df <= 30:
|
|
133
|
+
return _T975[df - 1]
|
|
134
|
+
pts = [(30, _T975[-1]), *_T975_LARGE, (math.inf, 1.960)]
|
|
135
|
+
for (d0, t0), (d1, t1) in zip(pts, pts[1:]):
|
|
136
|
+
if d0 <= df <= d1:
|
|
137
|
+
x0, x1, x = 1.0 / d0, (0.0 if math.isinf(d1) else 1.0 / d1), 1.0 / df
|
|
138
|
+
return t0 + (t1 - t0) * (x0 - x) / (x0 - x1)
|
|
139
|
+
return 1.960 # pragma: no cover
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _stats(values: Sequence[float]) -> FactorStats:
|
|
143
|
+
n = len(values)
|
|
144
|
+
mean = sum(values) / n
|
|
145
|
+
sd = math.sqrt(sum((v - mean) ** 2 for v in values) / (n - 1)) if n > 1 else 0.0
|
|
146
|
+
half = t_critical_95(n - 1) * sd / math.sqrt(n) if n > 1 else math.nan
|
|
147
|
+
return FactorStats(
|
|
148
|
+
mean=mean,
|
|
149
|
+
sd=sd,
|
|
150
|
+
n=n,
|
|
151
|
+
cv=sd / mean if mean else math.nan,
|
|
152
|
+
ci_half_width=half,
|
|
153
|
+
relative_precision=half / mean if mean else math.nan,
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def group_factors(sets: Sequence[FactorSet], name: str = "group") -> FactorGroup:
|
|
158
|
+
"""Average the factors of several stations into a group (TMG 2022, 3.1.4.7 Step 5).
|
|
159
|
+
|
|
160
|
+
Each factor is averaged over the stations that have it.
|
|
161
|
+
"""
|
|
162
|
+
if not sets:
|
|
163
|
+
raise InsufficientDataError("no stations to group")
|
|
164
|
+
|
|
165
|
+
def avg(dicts: Sequence[Mapping]) -> Tuple[Dict, Dict]:
|
|
166
|
+
acc: Dict = {}
|
|
167
|
+
for d in dicts:
|
|
168
|
+
for k, v in d.items():
|
|
169
|
+
acc.setdefault(k, []).append(v)
|
|
170
|
+
return ({k: sum(v) / len(v) for k, v in sorted(acc.items())}, acc)
|
|
171
|
+
|
|
172
|
+
monthly, m_acc = avg([s.monthly for s in sets])
|
|
173
|
+
dow, d_acc = avg([s.dow for s in sets])
|
|
174
|
+
month_dow, _ = avg([s.month_dow for s in sets])
|
|
175
|
+
return FactorGroup(
|
|
176
|
+
monthly=monthly,
|
|
177
|
+
dow=dow,
|
|
178
|
+
month_dow=month_dow,
|
|
179
|
+
name=name,
|
|
180
|
+
members=[s.name for s in sets],
|
|
181
|
+
monthly_stats={k: _stats(v) for k, v in sorted(m_acc.items())},
|
|
182
|
+
dow_stats={k: _stats(v) for k, v in sorted(d_acc.items())},
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
@dataclass
|
|
187
|
+
class ClusterResult:
|
|
188
|
+
"""Outcome of :func:`cluster_stations`."""
|
|
189
|
+
|
|
190
|
+
labels: Dict[str, int]
|
|
191
|
+
centroids: List[List[float]]
|
|
192
|
+
inertia: float
|
|
193
|
+
features: str
|
|
194
|
+
|
|
195
|
+
def groups(self) -> Dict[int, List[str]]:
|
|
196
|
+
"""Station names per cluster label."""
|
|
197
|
+
out: Dict[int, List[str]] = {}
|
|
198
|
+
for name, lab in self.labels.items():
|
|
199
|
+
out.setdefault(lab, []).append(name)
|
|
200
|
+
return dict(sorted(out.items()))
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _feature_vector(fs: FactorSet, features: str) -> List[float]:
|
|
204
|
+
try:
|
|
205
|
+
if features == "monthly":
|
|
206
|
+
return [fs.monthly[m] for m in range(1, 13)]
|
|
207
|
+
if features == "dow":
|
|
208
|
+
return [fs.dow[j] for j in range(7)]
|
|
209
|
+
if features == "both":
|
|
210
|
+
return [fs.monthly[m] for m in range(1, 13)] + [fs.dow[j] for j in range(7)]
|
|
211
|
+
except KeyError as exc:
|
|
212
|
+
raise InsufficientDataError(f"station {fs.name!r} lacks factor {exc}") from exc
|
|
213
|
+
raise ValueError("features must be 'monthly', 'dow' or 'both'")
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _sqdist(a: Sequence[float], b: Sequence[float]) -> float:
|
|
217
|
+
return sum((x - y) ** 2 for x, y in zip(a, b))
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _nearest(p: Sequence[float], centroids: Sequence[Sequence[float]]) -> int:
|
|
221
|
+
best, best_d = 0, math.inf
|
|
222
|
+
for c, cen in enumerate(centroids):
|
|
223
|
+
d = _sqdist(p, cen)
|
|
224
|
+
if d < best_d:
|
|
225
|
+
best, best_d = c, d
|
|
226
|
+
return best
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _kmeans_once(
|
|
230
|
+
pts: List[List[float]], k: int, rng: random.Random, max_iter: int
|
|
231
|
+
) -> Tuple[List[int], List[List[float]], float]:
|
|
232
|
+
# k-means++ seeding (Arthur & Vassilvitskii, 2007).
|
|
233
|
+
centroids = [list(pts[rng.randrange(len(pts))])]
|
|
234
|
+
while len(centroids) < k:
|
|
235
|
+
d2 = [min(_sqdist(p, c) for c in centroids) for p in pts]
|
|
236
|
+
total = sum(d2)
|
|
237
|
+
if total == 0:
|
|
238
|
+
centroids.append(list(pts[rng.randrange(len(pts))]))
|
|
239
|
+
continue
|
|
240
|
+
r, cum = rng.random() * total, 0.0
|
|
241
|
+
for p, w in zip(pts, d2):
|
|
242
|
+
cum += w
|
|
243
|
+
if cum >= r:
|
|
244
|
+
centroids.append(list(p))
|
|
245
|
+
break
|
|
246
|
+
else: # pragma: no cover - floating point guard
|
|
247
|
+
centroids.append(list(pts[-1]))
|
|
248
|
+
labels = [0] * len(pts)
|
|
249
|
+
for it in range(max_iter):
|
|
250
|
+
new = [_nearest(p, centroids) for p in pts]
|
|
251
|
+
for c in range(k):
|
|
252
|
+
members = [p for p, lab in zip(pts, new) if lab == c]
|
|
253
|
+
if members:
|
|
254
|
+
centroids[c] = [sum(col) / len(members) for col in zip(*members)]
|
|
255
|
+
if new == labels and it > 0:
|
|
256
|
+
break
|
|
257
|
+
labels = new
|
|
258
|
+
inertia = sum(_sqdist(p, centroids[lab]) for p, lab in zip(pts, labels))
|
|
259
|
+
return labels, centroids, inertia
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def cluster_stations(
|
|
263
|
+
sets: Sequence[FactorSet],
|
|
264
|
+
k: int,
|
|
265
|
+
*,
|
|
266
|
+
features: str = "monthly",
|
|
267
|
+
seed: int = 0,
|
|
268
|
+
n_init: int = 10,
|
|
269
|
+
max_iter: int = 100,
|
|
270
|
+
) -> ClusterResult:
|
|
271
|
+
"""Group stations with similar factor patterns by k-means.
|
|
272
|
+
|
|
273
|
+
Args:
|
|
274
|
+
sets: one factor set per continuous count station (``name`` is used as id).
|
|
275
|
+
k: number of factor groups.
|
|
276
|
+
features: ``"monthly"`` (12 monthly factors, the classic seasonal
|
|
277
|
+
grouping), ``"dow"`` (7 day-of-week factors) or ``"both"``.
|
|
278
|
+
seed: random seed; results are deterministic for a given seed.
|
|
279
|
+
n_init: number of k-means++ restarts; the lowest inertia is kept.
|
|
280
|
+
|
|
281
|
+
Labels are renumbered so that cluster 0 holds the first station.
|
|
282
|
+
"""
|
|
283
|
+
if not 1 <= k <= len(sets):
|
|
284
|
+
raise ValueError("k must be between 1 and the number of stations")
|
|
285
|
+
pts = [_feature_vector(s, features) for s in sets]
|
|
286
|
+
rng = random.Random(seed)
|
|
287
|
+
best: Optional[Tuple[List[int], List[List[float]], float]] = None
|
|
288
|
+
for _ in range(max(1, n_init)):
|
|
289
|
+
res = _kmeans_once(pts, k, rng, max_iter)
|
|
290
|
+
if best is None or res[2] < best[2] - 1e-12:
|
|
291
|
+
best = res
|
|
292
|
+
assert best is not None
|
|
293
|
+
labels, centroids, inertia = best
|
|
294
|
+
remap: Dict[int, int] = {}
|
|
295
|
+
for lab in labels:
|
|
296
|
+
remap.setdefault(lab, len(remap))
|
|
297
|
+
for lab in range(k):
|
|
298
|
+
remap.setdefault(lab, len(remap))
|
|
299
|
+
new_centroids = [centroids[0]] * k
|
|
300
|
+
for old, new in remap.items():
|
|
301
|
+
new_centroids[new] = centroids[old]
|
|
302
|
+
return ClusterResult(
|
|
303
|
+
labels={s.name or str(i): remap[lab] for i, (s, lab) in enumerate(zip(sets, labels))},
|
|
304
|
+
centroids=new_centroids,
|
|
305
|
+
inertia=inertia,
|
|
306
|
+
features=features,
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def axle_correction_factor(vehicles: float, axles: float, *, axles_per_count: float = 2.0) -> float:
|
|
311
|
+
"""Axle-correction factor from a classification count.
|
|
312
|
+
|
|
313
|
+
TMG defines the factor as total vehicles divided by total axles of those
|
|
314
|
+
vehicles (TMG 2022 glossary). Most single-sensor (road tube) counters
|
|
315
|
+
record one "count" per two axle hits, so the factor is usually applied to
|
|
316
|
+
*axle pairs*; ``axles_per_count`` sets that divisor::
|
|
317
|
+
|
|
318
|
+
ACF = vehicles / (axles / axles_per_count)
|
|
319
|
+
|
|
320
|
+
With ``axles_per_count=1`` the literal vehicles-per-axle ratio is returned.
|
|
321
|
+
A traffic stream of two-axle vehicles gives ``ACF = 1`` with the default.
|
|
322
|
+
"""
|
|
323
|
+
if vehicles < 0 or axles <= 0 or axles_per_count <= 0:
|
|
324
|
+
raise ValueError("vehicles must be >= 0 and axles, axles_per_count > 0")
|
|
325
|
+
return vehicles / (axles / axles_per_count)
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
#: Nominal axles per FHWA vehicle class, used only when per-vehicle axle counts
|
|
329
|
+
#: are unavailable. APPROXIMATE: classes 3, 4, 7, 8, 10, 11 and 13 cover a
|
|
330
|
+
#: range of axle numbers (e.g. class 8 = 3 or 4 axles); the values below are the
|
|
331
|
+
#: minimum or the typical configuration and should be replaced by measured
|
|
332
|
+
#: means from the agency's own classification data.
|
|
333
|
+
DEFAULT_AXLES_PER_CLASS: Dict[int, float] = {
|
|
334
|
+
1: 2.0,
|
|
335
|
+
2: 2.0,
|
|
336
|
+
3: 2.0,
|
|
337
|
+
4: 2.0,
|
|
338
|
+
5: 2.0,
|
|
339
|
+
6: 3.0,
|
|
340
|
+
7: 4.0,
|
|
341
|
+
8: 4.0,
|
|
342
|
+
9: 5.0,
|
|
343
|
+
10: 6.0,
|
|
344
|
+
11: 5.0,
|
|
345
|
+
12: 6.0,
|
|
346
|
+
13: 7.0,
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def axle_correction_from_classes(
|
|
351
|
+
class_counts: Mapping[int, float],
|
|
352
|
+
axles_per_class: Optional[Mapping[int, float]] = None,
|
|
353
|
+
*,
|
|
354
|
+
axles_per_count: float = 2.0,
|
|
355
|
+
) -> float:
|
|
356
|
+
"""Axle-correction factor from vehicle counts by FHWA class.
|
|
357
|
+
|
|
358
|
+
Total axles are estimated as ``sum(count_c * axles_c)``; supply
|
|
359
|
+
``axles_per_class`` with measured averages for best results (defaults to
|
|
360
|
+
:data:`DEFAULT_AXLES_PER_CLASS`).
|
|
361
|
+
"""
|
|
362
|
+
table = dict(DEFAULT_AXLES_PER_CLASS)
|
|
363
|
+
if axles_per_class:
|
|
364
|
+
table.update(axles_per_class)
|
|
365
|
+
vehicles = 0.0
|
|
366
|
+
axles = 0.0
|
|
367
|
+
for c, n in class_counts.items():
|
|
368
|
+
if c not in table:
|
|
369
|
+
raise ValueError(f"no axle number for class {c}")
|
|
370
|
+
vehicles += n
|
|
371
|
+
axles += n * table[c]
|
|
372
|
+
return axle_correction_factor(vehicles, axles, axles_per_count=axles_per_count)
|
aadtkit/io.py
ADDED
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
"""CSV input/output for hourly counts and factor tables.
|
|
2
|
+
|
|
3
|
+
Hourly count CSV (header names are case-insensitive)::
|
|
4
|
+
|
|
5
|
+
station,datetime,volume[,direction][,vehicle_class]
|
|
6
|
+
S1,2024-01-01 00:00,312,NB,2
|
|
7
|
+
|
|
8
|
+
Instead of ``datetime`` a ``date`` column plus an ``hour`` column (0-23 for
|
|
9
|
+
hour beginning, 1-24 for hour ending) may be given. Aliases: ``timestamp`` /
|
|
10
|
+
``time`` for ``datetime``, ``count`` for ``volume``, ``class`` for
|
|
11
|
+
``vehicle_class``, ``dir`` for ``direction``, ``site`` / ``station_id`` for
|
|
12
|
+
``station``. Rows with an empty or ``NA`` volume are treated as missing hours.
|
|
13
|
+
|
|
14
|
+
Factor CSV::
|
|
15
|
+
|
|
16
|
+
name,type,month,dow,factor
|
|
17
|
+
G1,monthly,1,,1.12
|
|
18
|
+
G1,dow,,0,1.03
|
|
19
|
+
G1,month_dow,1,0,1.15
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import csv
|
|
25
|
+
import datetime as dt
|
|
26
|
+
import os
|
|
27
|
+
from collections import Counter
|
|
28
|
+
from dataclasses import dataclass
|
|
29
|
+
from typing import IO, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple, Union
|
|
30
|
+
|
|
31
|
+
from .errors import CSVFormatError
|
|
32
|
+
from .factors import FactorSet
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"HourlyRecord",
|
|
36
|
+
"by_direction",
|
|
37
|
+
"by_station",
|
|
38
|
+
"class_totals",
|
|
39
|
+
"duplicate_keys",
|
|
40
|
+
"read_factors_csv",
|
|
41
|
+
"read_hourly_csv",
|
|
42
|
+
"to_series",
|
|
43
|
+
"write_factors_csv",
|
|
44
|
+
"write_hourly_csv",
|
|
45
|
+
"write_rows_csv",
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
PathOrFile = Union[str, "os.PathLike[str]", IO[str]]
|
|
49
|
+
|
|
50
|
+
_ALIASES = {
|
|
51
|
+
"station": ("station", "station_id", "site", "site_id"),
|
|
52
|
+
"datetime": ("datetime", "timestamp", "time", "date_time"),
|
|
53
|
+
"date": ("date",),
|
|
54
|
+
"hour": ("hour",),
|
|
55
|
+
"volume": ("volume", "count", "vol"),
|
|
56
|
+
"direction": ("direction", "dir"),
|
|
57
|
+
"vehicle_class": ("vehicle_class", "class", "fhwa_class"),
|
|
58
|
+
}
|
|
59
|
+
_NA = {"", "na", "nan", "null", "none", "-"}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class HourlyRecord:
|
|
64
|
+
"""One hourly volume (hour-beginning timestamp)."""
|
|
65
|
+
|
|
66
|
+
station: str
|
|
67
|
+
time: dt.datetime
|
|
68
|
+
volume: float
|
|
69
|
+
direction: str = ""
|
|
70
|
+
vehicle_class: Optional[int] = None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _open(src: PathOrFile, mode: str):
|
|
74
|
+
if isinstance(src, (str, os.PathLike)):
|
|
75
|
+
return open(src, mode, encoding="utf-8-sig" if "r" in mode else "utf-8", newline="")
|
|
76
|
+
return _Borrowed(src)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class _Borrowed:
|
|
80
|
+
"""Context manager that does not close a caller-owned stream."""
|
|
81
|
+
|
|
82
|
+
def __init__(self, f: IO[str]) -> None:
|
|
83
|
+
self.f = f
|
|
84
|
+
|
|
85
|
+
def __enter__(self) -> IO[str]:
|
|
86
|
+
return self.f
|
|
87
|
+
|
|
88
|
+
def __exit__(self, *exc: object) -> None:
|
|
89
|
+
return None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _parse_dt(text: str) -> dt.datetime:
|
|
93
|
+
t = text.strip().replace("T", " ")
|
|
94
|
+
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d %H", "%Y/%m/%d %H:%M"):
|
|
95
|
+
try:
|
|
96
|
+
return dt.datetime.strptime(t, fmt)
|
|
97
|
+
except ValueError:
|
|
98
|
+
continue
|
|
99
|
+
try:
|
|
100
|
+
return dt.datetime.fromisoformat(t)
|
|
101
|
+
except ValueError as exc:
|
|
102
|
+
raise CSVFormatError(f"cannot parse datetime {text!r}") from exc
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def read_hourly_csv(
|
|
106
|
+
src: PathOrFile,
|
|
107
|
+
*,
|
|
108
|
+
hour_convention: str = "beginning",
|
|
109
|
+
delimiter: str = ",",
|
|
110
|
+
default_station: str = "station",
|
|
111
|
+
) -> List[HourlyRecord]:
|
|
112
|
+
"""Read hourly counts from a CSV file or text stream.
|
|
113
|
+
|
|
114
|
+
Args:
|
|
115
|
+
src: path or open text stream.
|
|
116
|
+
hour_convention: ``"beginning"`` (timestamps / hours 0-23 mark the
|
|
117
|
+
start of the hour) or ``"ending"`` (they mark the end, hours 1-24);
|
|
118
|
+
records are always returned with hour-beginning stamps.
|
|
119
|
+
delimiter: field separator.
|
|
120
|
+
default_station: station id when the file has no station column.
|
|
121
|
+
"""
|
|
122
|
+
if hour_convention not in ("beginning", "ending"):
|
|
123
|
+
raise ValueError("hour_convention must be 'beginning' or 'ending'")
|
|
124
|
+
shift = dt.timedelta(hours=1) if hour_convention == "ending" else dt.timedelta(0)
|
|
125
|
+
out: List[HourlyRecord] = []
|
|
126
|
+
with _open(src, "r") as f:
|
|
127
|
+
reader = csv.DictReader(f, delimiter=delimiter)
|
|
128
|
+
if not reader.fieldnames:
|
|
129
|
+
raise CSVFormatError("empty CSV")
|
|
130
|
+
lower = {name.strip().lower(): name for name in reader.fieldnames}
|
|
131
|
+
cols: Dict[str, str] = {}
|
|
132
|
+
for key, aliases in _ALIASES.items():
|
|
133
|
+
for a in aliases:
|
|
134
|
+
if a in lower:
|
|
135
|
+
cols[key] = lower[a]
|
|
136
|
+
break
|
|
137
|
+
if "volume" not in cols:
|
|
138
|
+
raise CSVFormatError("CSV needs a 'volume' (or 'count') column")
|
|
139
|
+
if "datetime" not in cols and not ("date" in cols and "hour" in cols):
|
|
140
|
+
raise CSVFormatError("CSV needs 'datetime' or 'date' + 'hour' columns")
|
|
141
|
+
for lineno, row in enumerate(reader, start=2):
|
|
142
|
+
raw_v = (row.get(cols["volume"]) or "").strip()
|
|
143
|
+
if raw_v.lower() in _NA:
|
|
144
|
+
continue
|
|
145
|
+
try:
|
|
146
|
+
vol = float(raw_v)
|
|
147
|
+
except ValueError as exc:
|
|
148
|
+
raise CSVFormatError(f"line {lineno}: bad volume {raw_v!r}") from exc
|
|
149
|
+
if "datetime" in cols:
|
|
150
|
+
ts = _parse_dt(row[cols["datetime"]])
|
|
151
|
+
else:
|
|
152
|
+
try:
|
|
153
|
+
day = dt.date.fromisoformat(row[cols["date"]].strip())
|
|
154
|
+
hour = int(float(row[cols["hour"]]))
|
|
155
|
+
except ValueError as exc:
|
|
156
|
+
raise CSVFormatError(f"line {lineno}: bad date/hour") from exc
|
|
157
|
+
ts = dt.datetime.combine(day, dt.time()) + dt.timedelta(hours=hour)
|
|
158
|
+
ts = ts - shift
|
|
159
|
+
vc: Optional[int] = None
|
|
160
|
+
if "vehicle_class" in cols:
|
|
161
|
+
raw_c = (row.get(cols["vehicle_class"]) or "").strip()
|
|
162
|
+
if raw_c.lower() not in _NA:
|
|
163
|
+
try:
|
|
164
|
+
vc = int(float(raw_c))
|
|
165
|
+
except ValueError as exc:
|
|
166
|
+
raise CSVFormatError(f"line {lineno}: bad class {raw_c!r}") from exc
|
|
167
|
+
out.append(
|
|
168
|
+
HourlyRecord(
|
|
169
|
+
station=(row.get(cols.get("station", ""), "") or default_station).strip()
|
|
170
|
+
if "station" in cols
|
|
171
|
+
else default_station,
|
|
172
|
+
time=ts,
|
|
173
|
+
volume=vol,
|
|
174
|
+
direction=(row.get(cols["direction"]) or "").strip()
|
|
175
|
+
if "direction" in cols
|
|
176
|
+
else "",
|
|
177
|
+
vehicle_class=vc,
|
|
178
|
+
)
|
|
179
|
+
)
|
|
180
|
+
return out
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def to_series(
|
|
184
|
+
records: Iterable[HourlyRecord],
|
|
185
|
+
*,
|
|
186
|
+
station: Optional[str] = None,
|
|
187
|
+
direction: Optional[str] = None,
|
|
188
|
+
vehicle_class: Optional[int] = None,
|
|
189
|
+
) -> Dict[dt.datetime, float]:
|
|
190
|
+
"""Sum records (optionally filtered) into ``{hour: volume}``."""
|
|
191
|
+
out: Dict[dt.datetime, float] = {}
|
|
192
|
+
for r in records:
|
|
193
|
+
if station is not None and r.station != station:
|
|
194
|
+
continue
|
|
195
|
+
if direction is not None and r.direction != direction:
|
|
196
|
+
continue
|
|
197
|
+
if vehicle_class is not None and r.vehicle_class != vehicle_class:
|
|
198
|
+
continue
|
|
199
|
+
h = r.time.replace(minute=0, second=0, microsecond=0)
|
|
200
|
+
out[h] = out.get(h, 0.0) + r.volume
|
|
201
|
+
return dict(sorted(out.items()))
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def by_station(records: Iterable[HourlyRecord]) -> Dict[str, Dict[dt.datetime, float]]:
|
|
205
|
+
"""``{station: {hour: total volume}}`` (all directions and classes summed)."""
|
|
206
|
+
recs = list(records)
|
|
207
|
+
names = sorted({r.station for r in recs})
|
|
208
|
+
return {s: to_series(recs, station=s) for s in names}
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def by_direction(
|
|
212
|
+
records: Iterable[HourlyRecord], station: Optional[str] = None
|
|
213
|
+
) -> Dict[str, Dict[dt.datetime, float]]:
|
|
214
|
+
"""``{direction: {hour: volume}}`` for one station (or all records)."""
|
|
215
|
+
recs = [r for r in records if station is None or r.station == station]
|
|
216
|
+
dirs = sorted({r.direction for r in recs})
|
|
217
|
+
return {d: to_series(recs, direction=d) for d in dirs}
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def class_totals(
|
|
221
|
+
records: Iterable[HourlyRecord], station: Optional[str] = None
|
|
222
|
+
) -> Dict[int, float]:
|
|
223
|
+
"""Total volume per FHWA class (records without a class are ignored)."""
|
|
224
|
+
out: Dict[int, float] = {}
|
|
225
|
+
for r in records:
|
|
226
|
+
if r.vehicle_class is None or (station is not None and r.station != station):
|
|
227
|
+
continue
|
|
228
|
+
out[r.vehicle_class] = out.get(r.vehicle_class, 0.0) + r.volume
|
|
229
|
+
return dict(sorted(out.items()))
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def duplicate_keys(
|
|
233
|
+
records: Iterable[HourlyRecord],
|
|
234
|
+
) -> List[Tuple[str, dt.datetime, str, Optional[int]]]:
|
|
235
|
+
"""Keys ``(station, hour, direction, class)`` that occur more than once."""
|
|
236
|
+
c = Counter((r.station, r.time, r.direction, r.vehicle_class) for r in records)
|
|
237
|
+
return sorted((k for k, n in c.items() if n > 1), key=lambda k: (k[0], k[1], k[2], k[3] or 0))
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def write_rows_csv(
|
|
241
|
+
dst: PathOrFile, rows: Sequence[Mapping[str, object]], fieldnames: Sequence[str]
|
|
242
|
+
) -> None:
|
|
243
|
+
"""Write dictionaries as CSV rows."""
|
|
244
|
+
with _open(dst, "w") as f:
|
|
245
|
+
w = csv.DictWriter(
|
|
246
|
+
f, fieldnames=list(fieldnames), lineterminator="\n", extrasaction="ignore"
|
|
247
|
+
)
|
|
248
|
+
w.writeheader()
|
|
249
|
+
for row in rows:
|
|
250
|
+
w.writerow(row)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def write_hourly_csv(dst: PathOrFile, records: Iterable[HourlyRecord]) -> None:
|
|
254
|
+
"""Write records in the canonical hourly CSV layout."""
|
|
255
|
+
rows = [
|
|
256
|
+
{
|
|
257
|
+
"station": r.station,
|
|
258
|
+
"datetime": r.time.strftime("%Y-%m-%d %H:%M"),
|
|
259
|
+
"volume": f"{r.volume:g}",
|
|
260
|
+
"direction": r.direction,
|
|
261
|
+
"vehicle_class": "" if r.vehicle_class is None else r.vehicle_class,
|
|
262
|
+
}
|
|
263
|
+
for r in records
|
|
264
|
+
]
|
|
265
|
+
write_rows_csv(dst, rows, ["station", "datetime", "volume", "direction", "vehicle_class"])
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def write_factors_csv(dst: PathOrFile, sets: Iterable[FactorSet]) -> None:
|
|
269
|
+
"""Write factor sets as ``name,type,month,dow,factor`` rows."""
|
|
270
|
+
rows: List[Dict[str, object]] = []
|
|
271
|
+
for fs in sets:
|
|
272
|
+
for m, v in sorted(fs.monthly.items()):
|
|
273
|
+
rows.append(
|
|
274
|
+
{"name": fs.name, "type": "monthly", "month": m, "dow": "", "factor": f"{v:.6f}"}
|
|
275
|
+
)
|
|
276
|
+
for j, v in sorted(fs.dow.items()):
|
|
277
|
+
rows.append(
|
|
278
|
+
{"name": fs.name, "type": "dow", "month": "", "dow": j, "factor": f"{v:.6f}"}
|
|
279
|
+
)
|
|
280
|
+
for (m, j), v in sorted(fs.month_dow.items()):
|
|
281
|
+
rows.append(
|
|
282
|
+
{"name": fs.name, "type": "month_dow", "month": m, "dow": j, "factor": f"{v:.6f}"}
|
|
283
|
+
)
|
|
284
|
+
write_rows_csv(dst, rows, ["name", "type", "month", "dow", "factor"])
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def read_factors_csv(src: PathOrFile) -> Dict[str, FactorSet]:
|
|
288
|
+
"""Read factor sets written by :func:`write_factors_csv` (or by hand)."""
|
|
289
|
+
sets: Dict[str, FactorSet] = {}
|
|
290
|
+
with _open(src, "r") as f:
|
|
291
|
+
reader = csv.DictReader(f)
|
|
292
|
+
need = {"type", "factor"}
|
|
293
|
+
if not reader.fieldnames or not need <= {n.strip().lower() for n in reader.fieldnames}:
|
|
294
|
+
raise CSVFormatError("factor CSV needs at least 'type' and 'factor' columns")
|
|
295
|
+
for lineno, raw in enumerate(reader, start=2):
|
|
296
|
+
row = {(k or "").strip().lower(): (v or "").strip() for k, v in raw.items()}
|
|
297
|
+
name = row.get("name", "") or "default"
|
|
298
|
+
fs = sets.setdefault(name, FactorSet(monthly={}, dow={}, name=name))
|
|
299
|
+
kind = row.get("type", "").lower()
|
|
300
|
+
if kind not in ("monthly", "dow", "month_dow"):
|
|
301
|
+
raise CSVFormatError(f"line {lineno}: unknown factor type {kind!r}")
|
|
302
|
+
try:
|
|
303
|
+
val = float(row["factor"])
|
|
304
|
+
if kind == "monthly":
|
|
305
|
+
fs.monthly[int(row["month"])] = val
|
|
306
|
+
elif kind == "dow":
|
|
307
|
+
fs.dow[int(row["dow"])] = val
|
|
308
|
+
else:
|
|
309
|
+
fs.month_dow[(int(row["month"]), int(row["dow"]))] = val
|
|
310
|
+
except (KeyError, ValueError) as exc:
|
|
311
|
+
raise CSVFormatError(f"line {lineno}: {exc}") from exc
|
|
312
|
+
return sets
|