cloudsealed-jit 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloudsealed_jit/__init__.py +24 -0
- cloudsealed_jit/analysis.py +423 -0
- cloudsealed_jit/api.py +165 -0
- cloudsealed_jit/cli.py +57 -0
- cloudsealed_jit/kernels.py +140 -0
- cloudsealed_jit/parsing.py +321 -0
- cloudsealed_jit-0.2.0.dist-info/METADATA +222 -0
- cloudsealed_jit-0.2.0.dist-info/RECORD +12 -0
- cloudsealed_jit-0.2.0.dist-info/WHEEL +5 -0
- cloudsealed_jit-0.2.0.dist-info/entry_points.txt +2 -0
- cloudsealed_jit-0.2.0.dist-info/licenses/LICENSE +21 -0
- cloudsealed_jit-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Numerical kernels for baseline estimation.
|
|
3
|
+
|
|
4
|
+
Cloud spend series contain the very spikes we are trying to detect. Estimating
|
|
5
|
+
a baseline with a mean and flagging outliers with a standard-deviation z-score
|
|
6
|
+
fails on this data: a handful of large spikes inflate the standard deviation
|
|
7
|
+
enough to hide themselves and everything near them. This is the classic
|
|
8
|
+
masking effect.
|
|
9
|
+
|
|
10
|
+
We therefore use robust statistics throughout:
|
|
11
|
+
|
|
12
|
+
baseline = centred rolling median
|
|
13
|
+
scale = median absolute deviation (MAD)
|
|
14
|
+
z = 0.6745 * (x - median) / MAD [Iglewicz & Hoaglin, 1993]
|
|
15
|
+
|
|
16
|
+
The 0.6745 constant makes the MAD a consistent estimator of the standard
|
|
17
|
+
deviation for normally distributed data, so the resulting score keeps the
|
|
18
|
+
familiar "number of deviations" interpretation while staying resistant to
|
|
19
|
+
roughly half the sample being contaminated.
|
|
20
|
+
|
|
21
|
+
A centred rolling median over a window is O(n * w log w) in the naive form.
|
|
22
|
+
For the daily series produced by :mod:`cloudsealed_jit.parsing` that is
|
|
23
|
+
irrelevant, but the same kernels run over per-service series and over hourly
|
|
24
|
+
exports spanning years, where it is not. Numba compiles them to native code
|
|
25
|
+
via LLVM; the pure NumPy path is kept as a fallback so the package installs
|
|
26
|
+
and runs without a compiler toolchain.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import numpy as np
|
|
32
|
+
|
|
33
|
+
__all__ = ["rolling_median", "mad", "modified_zscores", "JIT_ENABLED"]
|
|
34
|
+
|
|
35
|
+
_MAD_TO_SIGMA = 0.6745
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
try: # pragma: no cover - depends on the runtime environment
|
|
39
|
+
from numba import njit
|
|
40
|
+
|
|
41
|
+
JIT_ENABLED = True
|
|
42
|
+
except ImportError: # pragma: no cover
|
|
43
|
+
JIT_ENABLED = False
|
|
44
|
+
|
|
45
|
+
def njit(*args, **kwargs): # type: ignore[misc]
|
|
46
|
+
"""No-op stand-in used when numba is unavailable."""
|
|
47
|
+
def wrap(func):
|
|
48
|
+
return func
|
|
49
|
+
if args and callable(args[0]):
|
|
50
|
+
return args[0]
|
|
51
|
+
return wrap
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@njit(cache=True)
|
|
55
|
+
def _rolling_median_kernel(values: np.ndarray, window: int) -> np.ndarray:
|
|
56
|
+
"""Centred rolling median with shrinking windows at the edges."""
|
|
57
|
+
n = values.shape[0]
|
|
58
|
+
out = np.empty(n, dtype=np.float64)
|
|
59
|
+
half = window // 2
|
|
60
|
+
buffer = np.empty(window, dtype=np.float64)
|
|
61
|
+
|
|
62
|
+
for i in range(n):
|
|
63
|
+
lo = i - half
|
|
64
|
+
hi = i + half + 1
|
|
65
|
+
if lo < 0:
|
|
66
|
+
lo = 0
|
|
67
|
+
if hi > n:
|
|
68
|
+
hi = n
|
|
69
|
+
|
|
70
|
+
size = hi - lo
|
|
71
|
+
for k in range(size):
|
|
72
|
+
buffer[k] = values[lo + k]
|
|
73
|
+
|
|
74
|
+
# Insertion sort: window is small and the data is near-sorted.
|
|
75
|
+
for a in range(1, size):
|
|
76
|
+
key = buffer[a]
|
|
77
|
+
b = a - 1
|
|
78
|
+
while b >= 0 and buffer[b] > key:
|
|
79
|
+
buffer[b + 1] = buffer[b]
|
|
80
|
+
b -= 1
|
|
81
|
+
buffer[b + 1] = key
|
|
82
|
+
|
|
83
|
+
if size % 2 == 1:
|
|
84
|
+
out[i] = buffer[size // 2]
|
|
85
|
+
else:
|
|
86
|
+
out[i] = 0.5 * (buffer[size // 2 - 1] + buffer[size // 2])
|
|
87
|
+
|
|
88
|
+
return out
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def rolling_median(values: np.ndarray, window: int = 7) -> np.ndarray:
|
|
92
|
+
"""Centred rolling median.
|
|
93
|
+
|
|
94
|
+
Args:
|
|
95
|
+
values: 1-D series.
|
|
96
|
+
window: window length in samples. Forced odd and clamped to the series
|
|
97
|
+
length so short inputs degrade to a global median rather than fail.
|
|
98
|
+
"""
|
|
99
|
+
series = np.asarray(values, dtype=np.float64)
|
|
100
|
+
if series.size == 0:
|
|
101
|
+
return series.copy()
|
|
102
|
+
|
|
103
|
+
window = max(1, min(int(window), series.size))
|
|
104
|
+
if window % 2 == 0:
|
|
105
|
+
window -= 1
|
|
106
|
+
if window < 1:
|
|
107
|
+
window = 1
|
|
108
|
+
return _rolling_median_kernel(series, window)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def mad(values: np.ndarray, centre: np.ndarray | float | None = None) -> float:
|
|
112
|
+
"""Median absolute deviation about ``centre`` (default: the median)."""
|
|
113
|
+
series = np.asarray(values, dtype=np.float64)
|
|
114
|
+
if series.size == 0:
|
|
115
|
+
return 0.0
|
|
116
|
+
if centre is None:
|
|
117
|
+
centre = float(np.median(series))
|
|
118
|
+
return float(np.median(np.abs(series - centre)))
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def modified_zscores(values: np.ndarray, baseline: np.ndarray) -> np.ndarray:
|
|
122
|
+
"""Robust z-scores of ``values`` against ``baseline``.
|
|
123
|
+
|
|
124
|
+
Falls back to a standard-deviation scale when the MAD is zero, which
|
|
125
|
+
happens on series that are constant apart from a few spikes. If both
|
|
126
|
+
scales are zero the series is flat and every score is zero.
|
|
127
|
+
"""
|
|
128
|
+
series = np.asarray(values, dtype=np.float64)
|
|
129
|
+
base = np.asarray(baseline, dtype=np.float64)
|
|
130
|
+
residuals = series - base
|
|
131
|
+
|
|
132
|
+
scale = float(np.median(np.abs(residuals)))
|
|
133
|
+
if scale > 0.0:
|
|
134
|
+
return _MAD_TO_SIGMA * residuals / scale
|
|
135
|
+
|
|
136
|
+
fallback = float(np.std(residuals))
|
|
137
|
+
if fallback > 0.0:
|
|
138
|
+
return residuals / fallback
|
|
139
|
+
|
|
140
|
+
return np.zeros_like(series)
|
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Billing export parsing.
|
|
3
|
+
|
|
4
|
+
Cloud providers export cost data with different schemas and different
|
|
5
|
+
granularities. A single day of spend is spread across many line items -- one
|
|
6
|
+
per resource, SKU or usage type. Any analysis that treats a CSV row as an
|
|
7
|
+
observation is measuring line-item size, not daily spend.
|
|
8
|
+
|
|
9
|
+
This module normalises the common export formats into a single daily series:
|
|
10
|
+
|
|
11
|
+
date (UTC calendar day) -> total cost, and cost per service
|
|
12
|
+
|
|
13
|
+
Supported layouts, detected from the header row:
|
|
14
|
+
|
|
15
|
+
AWS Cost and Usage Report lineItem/UsageStartDate, lineItem/UnblendedCost
|
|
16
|
+
GCP billing export usage_start_time, cost, service.description
|
|
17
|
+
Azure cost export Date / UsageDateTime, Cost / CostInBillingCurrency
|
|
18
|
+
Generic heuristic match on date / cost / service columns
|
|
19
|
+
|
|
20
|
+
Rows that cannot be parsed are counted and reported rather than silently
|
|
21
|
+
dropped, so callers can tell a partially-understood file from a clean one.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import csv
|
|
27
|
+
import io
|
|
28
|
+
from collections import defaultdict
|
|
29
|
+
from dataclasses import dataclass, field
|
|
30
|
+
from datetime import date, datetime
|
|
31
|
+
from typing import Iterable, Sequence
|
|
32
|
+
|
|
33
|
+
__all__ = ["BillingSeries", "ParseError", "parse_billing_csv"]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class ParseError(ValueError):
|
|
37
|
+
"""Raised when a billing export cannot be interpreted at all."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# --------------------------------------------------------------------------
|
|
41
|
+
# Column detection
|
|
42
|
+
# --------------------------------------------------------------------------
|
|
43
|
+
|
|
44
|
+
# Explicit, provider-specific column names take priority over the heuristics.
|
|
45
|
+
# Order matters: the first match wins.
|
|
46
|
+
_DATE_COLUMNS: Sequence[str] = (
|
|
47
|
+
"lineitem/usagestartdate", # AWS CUR
|
|
48
|
+
"lineitem/usagestarttime",
|
|
49
|
+
"bill/billingperiodstartdate",
|
|
50
|
+
"usage_start_time", # GCP
|
|
51
|
+
"usage_date",
|
|
52
|
+
"usagedatetime", # Azure
|
|
53
|
+
"date",
|
|
54
|
+
"billingdate",
|
|
55
|
+
"usagestart",
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
_COST_COLUMNS: Sequence[str] = (
|
|
59
|
+
"lineitem/unblendedcost", # AWS CUR
|
|
60
|
+
"lineitem/blendedcost",
|
|
61
|
+
"lineitem/netunblendedcost",
|
|
62
|
+
"cost", # GCP / generic
|
|
63
|
+
"costinbillingcurrency", # Azure
|
|
64
|
+
"pretaxcost",
|
|
65
|
+
"amortizedcost",
|
|
66
|
+
"effectivecost",
|
|
67
|
+
"amount",
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
_SERVICE_COLUMNS: Sequence[str] = (
|
|
71
|
+
"product/productname", # AWS CUR
|
|
72
|
+
"lineitem/productcode",
|
|
73
|
+
"service.description", # GCP
|
|
74
|
+
"service_description",
|
|
75
|
+
"service",
|
|
76
|
+
"servicename", # Azure
|
|
77
|
+
"metercategory",
|
|
78
|
+
"product",
|
|
79
|
+
"resourcegroup",
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
# Substrings used when no explicit column name matches. Deliberately narrow:
|
|
83
|
+
# matching "cost" alone would also match "costcenter".
|
|
84
|
+
_DATE_HINTS = ("date", "day", "time", "period")
|
|
85
|
+
_COST_HINTS = ("cost", "amount", "charge", "spend")
|
|
86
|
+
_SERVICE_HINTS = ("service", "product", "category", "resource")
|
|
87
|
+
|
|
88
|
+
_COST_EXCLUDE = ("center", "code", "currency", "type", "category")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _normalise(name: str) -> str:
|
|
92
|
+
return name.strip().lower().replace(" ", "").replace("_", "")
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _pick(headers: Sequence[str], explicit: Sequence[str], hints: Sequence[str],
|
|
96
|
+
exclude: Sequence[str] = ()) -> str | None:
|
|
97
|
+
lookup = {_normalise(h): h for h in headers}
|
|
98
|
+
|
|
99
|
+
for candidate in explicit:
|
|
100
|
+
key = _normalise(candidate)
|
|
101
|
+
if key in lookup:
|
|
102
|
+
return lookup[key]
|
|
103
|
+
|
|
104
|
+
for key, original in lookup.items():
|
|
105
|
+
if any(bad in key for bad in exclude):
|
|
106
|
+
continue
|
|
107
|
+
if any(hint in key for hint in hints):
|
|
108
|
+
return original
|
|
109
|
+
return None
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# --------------------------------------------------------------------------
|
|
113
|
+
# Value coercion
|
|
114
|
+
# --------------------------------------------------------------------------
|
|
115
|
+
|
|
116
|
+
_DATE_FORMATS = (
|
|
117
|
+
"%Y-%m-%d",
|
|
118
|
+
"%Y/%m/%d",
|
|
119
|
+
"%d/%m/%Y",
|
|
120
|
+
"%m/%d/%Y",
|
|
121
|
+
"%Y-%m-%dT%H:%M:%S",
|
|
122
|
+
"%Y-%m-%dT%H:%M:%SZ",
|
|
123
|
+
"%Y-%m-%d %H:%M:%S",
|
|
124
|
+
"%Y-%m-%d %H:%M:%S %Z",
|
|
125
|
+
"%Y-%m-%dT%H:%M:%S%z",
|
|
126
|
+
"%m/%d/%Y %H:%M:%S",
|
|
127
|
+
"%d-%m-%Y",
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _parse_date(raw: str) -> date | None:
|
|
132
|
+
text = (raw or "").strip()
|
|
133
|
+
if not text:
|
|
134
|
+
return None
|
|
135
|
+
|
|
136
|
+
# ISO-8601 with timezone offsets and fractional seconds.
|
|
137
|
+
try:
|
|
138
|
+
return datetime.fromisoformat(text.replace("Z", "+00:00")).date()
|
|
139
|
+
except ValueError:
|
|
140
|
+
pass
|
|
141
|
+
|
|
142
|
+
for fmt in _DATE_FORMATS:
|
|
143
|
+
try:
|
|
144
|
+
return datetime.strptime(text, fmt).date()
|
|
145
|
+
except ValueError:
|
|
146
|
+
continue
|
|
147
|
+
|
|
148
|
+
# Trailing timezone name that strptime cannot handle, e.g. "... UTC".
|
|
149
|
+
head = text.split(" ")[0]
|
|
150
|
+
if head != text:
|
|
151
|
+
return _parse_date(head)
|
|
152
|
+
return None
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _parse_cost(raw: str) -> float | None:
|
|
156
|
+
text = (raw or "").strip()
|
|
157
|
+
if not text:
|
|
158
|
+
return None
|
|
159
|
+
text = text.replace("$", "").replace("R$", "").replace("US$", "").strip()
|
|
160
|
+
|
|
161
|
+
# "1.234,56" (pt-BR) vs "1,234.56" (en-US): the last separator is decimal.
|
|
162
|
+
if "," in text and "." in text:
|
|
163
|
+
if text.rfind(",") > text.rfind("."):
|
|
164
|
+
text = text.replace(".", "").replace(",", ".")
|
|
165
|
+
else:
|
|
166
|
+
text = text.replace(",", "")
|
|
167
|
+
elif "," in text:
|
|
168
|
+
# Ambiguous. Treat as decimal only when it looks like one.
|
|
169
|
+
head, _, tail = text.rpartition(",")
|
|
170
|
+
text = f"{head}.{tail}" if len(tail) in (1, 2) else text.replace(",", "")
|
|
171
|
+
|
|
172
|
+
try:
|
|
173
|
+
value = float(text)
|
|
174
|
+
except ValueError:
|
|
175
|
+
return None
|
|
176
|
+
return value if value == value else None # reject NaN
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
# --------------------------------------------------------------------------
|
|
180
|
+
# Result type
|
|
181
|
+
# --------------------------------------------------------------------------
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
@dataclass
|
|
185
|
+
class BillingSeries:
|
|
186
|
+
"""A billing export normalised to daily totals.
|
|
187
|
+
|
|
188
|
+
Attributes:
|
|
189
|
+
days: calendar days present in the file, ascending and gap-free.
|
|
190
|
+
costs: total cost for each day in ``days``.
|
|
191
|
+
by_service: service name -> per-day cost, aligned with ``days``.
|
|
192
|
+
currency: currency code if the export declared one.
|
|
193
|
+
rows_parsed: line items successfully interpreted.
|
|
194
|
+
rows_skipped: line items rejected (unparseable date or cost).
|
|
195
|
+
"""
|
|
196
|
+
|
|
197
|
+
days: list[date]
|
|
198
|
+
costs: list[float]
|
|
199
|
+
by_service: dict[str, list[float]] = field(default_factory=dict)
|
|
200
|
+
currency: str | None = None
|
|
201
|
+
rows_parsed: int = 0
|
|
202
|
+
rows_skipped: int = 0
|
|
203
|
+
|
|
204
|
+
def __post_init__(self) -> None:
|
|
205
|
+
if len(self.days) != len(self.costs):
|
|
206
|
+
raise ValueError("days and costs must have the same length")
|
|
207
|
+
|
|
208
|
+
@property
|
|
209
|
+
def total_cost(self) -> float:
|
|
210
|
+
return float(sum(self.costs))
|
|
211
|
+
|
|
212
|
+
@property
|
|
213
|
+
def span_days(self) -> int:
|
|
214
|
+
return len(self.days)
|
|
215
|
+
|
|
216
|
+
def __len__(self) -> int: # pragma: no cover - trivial
|
|
217
|
+
return len(self.days)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
# --------------------------------------------------------------------------
|
|
221
|
+
# Entry point
|
|
222
|
+
# --------------------------------------------------------------------------
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _fill_gaps(daily: dict[date, float]) -> tuple[list[date], list[float]]:
|
|
226
|
+
"""Return a contiguous daily series, inserting 0.0 for missing days.
|
|
227
|
+
|
|
228
|
+
Gaps matter: a day with no line items is a day with no spend, and leaving
|
|
229
|
+
it out would distort both the weekly seasonality model and the averages.
|
|
230
|
+
"""
|
|
231
|
+
if not daily:
|
|
232
|
+
return [], []
|
|
233
|
+
start, end = min(daily), max(daily)
|
|
234
|
+
days: list[date] = []
|
|
235
|
+
costs: list[float] = []
|
|
236
|
+
current = start
|
|
237
|
+
one_day = (datetime.fromordinal(2) - datetime.fromordinal(1))
|
|
238
|
+
while current <= end:
|
|
239
|
+
days.append(current)
|
|
240
|
+
costs.append(round(daily.get(current, 0.0), 6))
|
|
241
|
+
current = (datetime.combine(current, datetime.min.time()) + one_day).date()
|
|
242
|
+
return days, costs
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def parse_billing_csv(csv_text: str, *, max_services: int = 25) -> BillingSeries:
|
|
246
|
+
"""Parse a cloud billing export into a daily :class:`BillingSeries`.
|
|
247
|
+
|
|
248
|
+
Args:
|
|
249
|
+
csv_text: raw contents of the CSV export.
|
|
250
|
+
max_services: keep only the N costliest services in ``by_service``.
|
|
251
|
+
|
|
252
|
+
Raises:
|
|
253
|
+
ParseError: the file is empty, has no header, or has no usable
|
|
254
|
+
date/cost column pair.
|
|
255
|
+
"""
|
|
256
|
+
if not csv_text or not csv_text.strip():
|
|
257
|
+
raise ParseError("Billing export is empty.")
|
|
258
|
+
|
|
259
|
+
reader = csv.DictReader(io.StringIO(csv_text))
|
|
260
|
+
headers = reader.fieldnames
|
|
261
|
+
if not headers:
|
|
262
|
+
raise ParseError("Billing export has no header row.")
|
|
263
|
+
|
|
264
|
+
date_col = _pick(headers, _DATE_COLUMNS, _DATE_HINTS)
|
|
265
|
+
cost_col = _pick(headers, _COST_COLUMNS, _COST_HINTS, exclude=_COST_EXCLUDE)
|
|
266
|
+
service_col = _pick(headers, _SERVICE_COLUMNS, _SERVICE_HINTS)
|
|
267
|
+
currency_col = _pick(headers, ("currency", "billingcurrency",
|
|
268
|
+
"lineitem/currencycode"), ("currency",))
|
|
269
|
+
|
|
270
|
+
if date_col is None or cost_col is None:
|
|
271
|
+
raise ParseError(
|
|
272
|
+
"Could not identify a date and a cost column. "
|
|
273
|
+
f"Columns found: {', '.join(headers[:20])}"
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
daily: dict[date, float] = defaultdict(float)
|
|
277
|
+
per_service: dict[str, dict[date, float]] = defaultdict(lambda: defaultdict(float))
|
|
278
|
+
currency: str | None = None
|
|
279
|
+
parsed = skipped = 0
|
|
280
|
+
|
|
281
|
+
for row in reader:
|
|
282
|
+
day = _parse_date(row.get(date_col, ""))
|
|
283
|
+
cost = _parse_cost(row.get(cost_col, ""))
|
|
284
|
+
if day is None or cost is None:
|
|
285
|
+
skipped += 1
|
|
286
|
+
continue
|
|
287
|
+
|
|
288
|
+
daily[day] += cost
|
|
289
|
+
if service_col:
|
|
290
|
+
name = (row.get(service_col) or "").strip() or "unattributed"
|
|
291
|
+
per_service[name][day] += cost
|
|
292
|
+
if currency is None and currency_col:
|
|
293
|
+
currency = (row.get(currency_col) or "").strip() or None
|
|
294
|
+
parsed += 1
|
|
295
|
+
|
|
296
|
+
if parsed == 0:
|
|
297
|
+
raise ParseError(
|
|
298
|
+
f"No line items could be parsed from {skipped} rows. "
|
|
299
|
+
f"Checked column '{date_col}' for dates and '{cost_col}' for costs."
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
days, costs = _fill_gaps(daily)
|
|
303
|
+
|
|
304
|
+
# Keep the costliest services; the long tail is noise for waste analysis.
|
|
305
|
+
by_service: dict[str, list[float]] = {}
|
|
306
|
+
ranked = sorted(per_service.items(), key=lambda kv: sum(kv[1].values()), reverse=True)
|
|
307
|
+
index = {day: i for i, day in enumerate(days)}
|
|
308
|
+
for name, series in ranked[:max_services]:
|
|
309
|
+
aligned = [0.0] * len(days)
|
|
310
|
+
for day, value in series.items():
|
|
311
|
+
aligned[index[day]] = round(value, 6)
|
|
312
|
+
by_service[name] = aligned
|
|
313
|
+
|
|
314
|
+
return BillingSeries(
|
|
315
|
+
days=days,
|
|
316
|
+
costs=costs,
|
|
317
|
+
by_service=by_service,
|
|
318
|
+
currency=currency,
|
|
319
|
+
rows_parsed=parsed,
|
|
320
|
+
rows_skipped=skipped,
|
|
321
|
+
)
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cloudsealed-jit
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Detect structural waste in cloud billing exports using robust day-of-week baselines
|
|
5
|
+
Author: Rodrigo Martinez Pinto
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/cloudsealed/JIT-Optimization-Engine
|
|
8
|
+
Project-URL: Source, https://github.com/cloudsealed/JIT-Optimization-Engine
|
|
9
|
+
Project-URL: Issues, https://github.com/cloudsealed/JIT-Optimization-Engine/issues
|
|
10
|
+
Keywords: finops,cloud-cost,aws,gcp,azure,anomaly-detection,billing
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: System Administrators
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: System :: Monitoring
|
|
19
|
+
Classifier: Topic :: Office/Business :: Financial
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: numpy>=1.24
|
|
24
|
+
Provides-Extra: jit
|
|
25
|
+
Requires-Dist: numba>=0.57; extra == "jit"
|
|
26
|
+
Provides-Extra: api
|
|
27
|
+
Requires-Dist: fastapi>=0.110; extra == "api"
|
|
28
|
+
Requires-Dist: uvicorn[standard]>=0.29; extra == "api"
|
|
29
|
+
Requires-Dist: pydantic>=2.6; extra == "api"
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: pytest>=7.4; extra == "dev"
|
|
32
|
+
Requires-Dist: httpx>=0.27; extra == "dev"
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
|
|
35
|
+
# cloudsealed-jit
|
|
36
|
+
|
|
37
|
+
Detects structural waste in cloud billing exports.
|
|
38
|
+
|
|
39
|
+
Given a billing export from AWS, GCP or Azure, it models what each day *should*
|
|
40
|
+
have cost, reports the days that did not match, and turns the excess into a
|
|
41
|
+
monthly figure. It is a library, a CLI and an HTTP service.
|
|
42
|
+
|
|
43
|
+
[](LICENSE)
|
|
44
|
+
[](https://www.python.org/downloads/)
|
|
45
|
+
|
|
46
|
+
---
|
|
47
|
+
|
|
48
|
+
## The problem
|
|
49
|
+
|
|
50
|
+
Cloud cost anomaly detection is usually done by comparing each day against the
|
|
51
|
+
period average and flagging anything beyond two or three standard deviations.
|
|
52
|
+
On billing data that method fails in two specific ways.
|
|
53
|
+
|
|
54
|
+
**Standard deviation is inflated by the very spikes you are looking for.** A
|
|
55
|
+
handful of large anomalies raises σ enough to pull themselves back inside the
|
|
56
|
+
threshold, and to hide every smaller anomaly with them. This is the masking
|
|
57
|
+
effect, and it gets worse as the anomalies get bigger.
|
|
58
|
+
|
|
59
|
+
**A flat average ignores the weekly cycle.** Most cloud bills have a pronounced
|
|
60
|
+
weekday/weekend shape. Measured against a flat mean, ordinary Mondays look like
|
|
61
|
+
overspend and ordinary Sundays look like savings.
|
|
62
|
+
|
|
63
|
+
## The method
|
|
64
|
+
|
|
65
|
+
**Baseline.** Expected spend for a day is a level term times a weekday term:
|
|
66
|
+
|
|
67
|
+
```
|
|
68
|
+
expected[i] = rolling_median(cost, 7)[i] × dow_factor[weekday(i)]
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
The rolling median follows growth and step changes without being dragged by
|
|
72
|
+
spikes. The weekday factor is the median ratio of observed spend to the level
|
|
73
|
+
term for that weekday. It is only estimated with at least two full weeks of
|
|
74
|
+
data; below that every factor is 1.0.
|
|
75
|
+
|
|
76
|
+
**Scoring.** Residuals are scored with a modified z-score built on the median
|
|
77
|
+
absolute deviation:
|
|
78
|
+
|
|
79
|
+
```
|
|
80
|
+
z = 0.6745 × (x − baseline) / MAD
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
The 0.6745 constant makes MAD a consistent estimator of σ for normal data, so
|
|
84
|
+
the score keeps the familiar "number of deviations" reading while tolerating
|
|
85
|
+
contamination in roughly half the sample. Days at or above |z| = 3.5 are
|
|
86
|
+
reported — the threshold recommended by Iglewicz & Hoaglin (1993).
|
|
87
|
+
|
|
88
|
+
**Waste.** Only positive excess counts. Waste percentage is the share of total
|
|
89
|
+
spend sitting above the baseline on anomalous days, which converts directly to
|
|
90
|
+
currency instead of being a count of unusual days.
|
|
91
|
+
|
|
92
|
+
**Recommendations.** Each carries a figure derived from the series itself,
|
|
93
|
+
normalised to 30 days, and states its assumption in the description. Estimates
|
|
94
|
+
that depend on facts the analyser cannot observe — whether a workload is
|
|
95
|
+
production, whether a commitment is acceptable — are labelled conditional
|
|
96
|
+
rather than presented as findings.
|
|
97
|
+
|
|
98
|
+
## Does it actually work better?
|
|
99
|
+
|
|
100
|
+
Yes, and it is measured, not asserted. `benchmarks/masking_benchmark.py` builds
|
|
101
|
+
synthetic bills whose anomalies are known by construction and scores this
|
|
102
|
+
method against the textbook mean+standard-deviation approach:
|
|
103
|
+
|
|
104
|
+
| scenario | textbook F1 | this method F1 |
|
|
105
|
+
|---|---|---|
|
|
106
|
+
| masking (scale estimator) | 0.667 | **0.923** |
|
|
107
|
+
| seasonality (baseline) | 0.667 | **1.000** |
|
|
108
|
+
| end-to-end | 0.667 | **1.000** |
|
|
109
|
+
|
|
110
|
+
Full derivation and reproduction steps in [METHODOLOGY.md](METHODOLOGY.md); the
|
|
111
|
+
design of the codebase is in [architecture.md](architecture.md). The benchmark
|
|
112
|
+
runs in CI (`--check`) and fails the build if the advantage ever regresses.
|
|
113
|
+
|
|
114
|
+
## Install
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
pip install cloudsealed-jit # library + CLI
|
|
118
|
+
pip install "cloudsealed-jit[jit]" # + numba-compiled kernels
|
|
119
|
+
pip install "cloudsealed-jit[jit,api]" # + HTTP service
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
`numba` is optional. Without it the kernels run on pure NumPy and the results
|
|
123
|
+
are identical; only large inputs get slower.
|
|
124
|
+
|
|
125
|
+
## Use
|
|
126
|
+
|
|
127
|
+
### CLI
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
cloudsealed-jit billing-export.csv
|
|
131
|
+
cloudsealed-jit billing-export.csv --json > findings.json
|
|
132
|
+
cat export.csv | cloudsealed-jit - --type cost-forecast
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
### Library
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
from cloudsealed_jit import parse_billing_csv, analyze
|
|
139
|
+
|
|
140
|
+
series = parse_billing_csv(open("export.csv").read())
|
|
141
|
+
result = analyze(series)
|
|
142
|
+
|
|
143
|
+
print(result.metrics.wastePercentage)
|
|
144
|
+
for r in result.recommendations:
|
|
145
|
+
print(r.title, r.potentialSavings)
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
### HTTP service
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
docker run -p 8091:8091 cloudsealed/jit-optimization-engine
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
```
|
|
155
|
+
GET /health
|
|
156
|
+
POST /v1/analyze-billing
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
curl -X POST localhost:8091/v1/analyze-billing \
|
|
161
|
+
-H 'Content-Type: application/json' \
|
|
162
|
+
-d '{"companyName":"Acme","csvContent":"date,cost\n2026-01-01,100\n..."}'
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
Set `JIT_OPTIMIZATION_API_KEY` to require an `X-Api-Key` header. Set
|
|
166
|
+
`JIT_MAX_CSV_BYTES` to change the 64 MB upload ceiling.
|
|
167
|
+
|
|
168
|
+
Response shape:
|
|
169
|
+
|
|
170
|
+
```jsonc
|
|
171
|
+
{
|
|
172
|
+
"anomalies": [
|
|
173
|
+
{ "date": "2026-01-31", "expectedCost": 99.0, "actualCost": 500.0,
|
|
174
|
+
"deviation": 405.05, "zScore": 7.82, "severity": "CRITICAL",
|
|
175
|
+
"description": "Spend above the day-of-week baseline by USD 401.00 (405.1%)." }
|
|
176
|
+
],
|
|
177
|
+
"metrics": {
|
|
178
|
+
"averageDailyCost": 106.32,
|
|
179
|
+
"stdDeviation": 51.69,
|
|
180
|
+
"sharpeRatio": 2.06, // spend stability: mean / stddev of daily cost
|
|
181
|
+
"wastePercentage": 6.29 // share of total spend above the baseline
|
|
182
|
+
},
|
|
183
|
+
"recommendations": [
|
|
184
|
+
{ "title": "...", "description": "...", "potentialSavings": 200.5, "effort": "MEDIUM" }
|
|
185
|
+
],
|
|
186
|
+
"summary": "..."
|
|
187
|
+
}
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
`sharpeRatio` is a **spend stability ratio** — mean daily cost divided by its
|
|
191
|
+
standard deviation, the reciprocal of the coefficient of variation. Higher
|
|
192
|
+
means more predictable spend. It is named for the field in the consuming API
|
|
193
|
+
contract; it is not a risk-adjusted return.
|
|
194
|
+
|
|
195
|
+
## Supported exports
|
|
196
|
+
|
|
197
|
+
| Provider | Date column | Cost column |
|
|
198
|
+
|---|---|---|
|
|
199
|
+
| AWS Cost and Usage Report | `lineItem/UsageStartDate` | `lineItem/UnblendedCost` |
|
|
200
|
+
| GCP billing export | `usage_start_time` | `cost` |
|
|
201
|
+
| Azure cost export | `Date`, `UsageDateTime` | `Cost`, `CostInBillingCurrency` |
|
|
202
|
+
| Generic | heuristic | heuristic |
|
|
203
|
+
|
|
204
|
+
Line items are aggregated to calendar days. Days with no line items are
|
|
205
|
+
inserted as zero-spend days rather than skipped. Rows that cannot be parsed are
|
|
206
|
+
counted and reported in the summary rather than dropped silently.
|
|
207
|
+
|
|
208
|
+
## Development
|
|
209
|
+
|
|
210
|
+
```bash
|
|
211
|
+
pip install -e ".[jit,api,dev]"
|
|
212
|
+
pytest
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
The test suite builds synthetic exports whose correct answer is known in
|
|
216
|
+
advance — a known spike at a known date, a known weekend-idle service, a stable
|
|
217
|
+
series that must produce no findings — so the assertions test behaviour rather
|
|
218
|
+
than the current output.
|
|
219
|
+
|
|
220
|
+
## License
|
|
221
|
+
|
|
222
|
+
MIT. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
cloudsealed_jit/__init__.py,sha256=IOHOiFIFwFwc4XjHnr7hFaOkjqCd5RWo5eRYrWDVRmg,604
|
|
2
|
+
cloudsealed_jit/analysis.py,sha256=Be3H0ksmiYxkx5prbGMZ_u-0Bb-7I7hQWYFh9K8YhCY,15355
|
|
3
|
+
cloudsealed_jit/api.py,sha256=N_jhv-RNgCJM2Nc6k3x3e4D8ufR1FgV-YpMbGqyYXYk,4913
|
|
4
|
+
cloudsealed_jit/cli.py,sha256=awxm-_39lPN3WB0SL86XQJukSM0So5CYcfJ0ixIotVw,1830
|
|
5
|
+
cloudsealed_jit/kernels.py,sha256=ul7is6QarWhmpiQfDaJ68IBmItF6GLoNuERsggIS5dc,4568
|
|
6
|
+
cloudsealed_jit/parsing.py,sha256=jSDw1E_zBELqVLioES5sexQOOcmCXVb4OlTC6rhNS0o,10483
|
|
7
|
+
cloudsealed_jit-0.2.0.dist-info/licenses/LICENSE,sha256=SY8phgxBGveojrUPkmyRDhp8HhA-SsX75HK_bnixdlo,1068
|
|
8
|
+
cloudsealed_jit-0.2.0.dist-info/METADATA,sha256=CnKW_5KIbYEBlphz1TWUZ9_kNxbwZWQzIwFyUjUe0TY,7803
|
|
9
|
+
cloudsealed_jit-0.2.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
10
|
+
cloudsealed_jit-0.2.0.dist-info/entry_points.txt,sha256=ATQRx9QOgOyuu41BTfFxDd4HZPfMYmYyw3iDnKMmfHw,61
|
|
11
|
+
cloudsealed_jit-0.2.0.dist-info/top_level.txt,sha256=r4c8dJeZHwe8UcAd8Cnok_K0Rxh7vpft7MKpogSzFNk,16
|
|
12
|
+
cloudsealed_jit-0.2.0.dist-info/RECORD,,
|