forecastops 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- forecastops/__init__.py +24 -0
- forecastops/adapters/__init__.py +2 -0
- forecastops/adapters/base.py +16 -0
- forecastops/adapters/darts.py +60 -0
- forecastops/adapters/dataframe.py +64 -0
- forecastops/adapters/gluonts.py +64 -0
- forecastops/adapters/nixtla.py +51 -0
- forecastops/adapters/prophet.py +38 -0
- forecastops/adapters/registry.py +59 -0
- forecastops/adapters/sklearn.py +28 -0
- forecastops/cli/__init__.py +2 -0
- forecastops/cli/main.py +229 -0
- forecastops/core/__init__.py +2 -0
- forecastops/core/capture.py +220 -0
- forecastops/core/compare.py +205 -0
- forecastops/core/config.py +110 -0
- forecastops/core/diff.py +90 -0
- forecastops/core/evaluate.py +328 -0
- forecastops/core/normalize.py +235 -0
- forecastops/core/report.py +182 -0
- forecastops/core/run.py +127 -0
- forecastops/core/schema.py +84 -0
- forecastops/core/validate.py +334 -0
- forecastops/core/wrappers.py +86 -0
- forecastops/otel/__init__.py +6 -0
- forecastops/otel/events.py +28 -0
- forecastops/otel/exporters.py +47 -0
- forecastops/otel/metrics.py +101 -0
- forecastops/otel/semconv.py +42 -0
- forecastops/otel/trace.py +78 -0
- forecastops/store/__init__.py +2 -0
- forecastops/store/duckdb_index.py +376 -0
- forecastops/store/local.py +52 -0
- forecastops/store/manifest.py +29 -0
- forecastops/store/parquet.py +64 -0
- forecastops/ui/__init__.py +2 -0
- forecastops/ui/queries.py +246 -0
- forecastops/ui/server.py +113 -0
- forecastops/ui/static/app.js +885 -0
- forecastops/ui/static/index.html +39 -0
- forecastops/ui/static/styles.css +749 -0
- forecastops-0.1.0.dist-info/METADATA +150 -0
- forecastops-0.1.0.dist-info/RECORD +46 -0
- forecastops-0.1.0.dist-info/WHEEL +4 -0
- forecastops-0.1.0.dist-info/entry_points.txt +2 -0
- forecastops-0.1.0.dist-info/licenses/LICENSE +202 -0
forecastops/__init__.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""ForecastOps public API."""
|
|
2
|
+
|
|
3
|
+
from forecastops.core.capture import capture
|
|
4
|
+
from forecastops.core.compare import compare
|
|
5
|
+
from forecastops.core.diff import diff
|
|
6
|
+
from forecastops.core.evaluate import evaluate
|
|
7
|
+
from forecastops.core.report import report
|
|
8
|
+
from forecastops.core.schema import ForecastSchema
|
|
9
|
+
from forecastops.core.wrappers import forecast, instrument, wrap
|
|
10
|
+
from forecastops.ui.server import ui
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"ForecastSchema",
|
|
14
|
+
"capture",
|
|
15
|
+
"compare",
|
|
16
|
+
"diff",
|
|
17
|
+
"evaluate",
|
|
18
|
+
"forecast",
|
|
19
|
+
"instrument",
|
|
20
|
+
"report",
|
|
21
|
+
"ui",
|
|
22
|
+
"wrap",
|
|
23
|
+
]
|
|
24
|
+
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any, Protocol
|
|
4
|
+
|
|
5
|
+
from forecastops.core.run import CaptureContext, DetectionResult, NormalizedForecast
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ForecastAdapter(Protocol):
|
|
9
|
+
name: str
|
|
10
|
+
|
|
11
|
+
def detect(self, obj: Any) -> DetectionResult:
|
|
12
|
+
...
|
|
13
|
+
|
|
14
|
+
def normalize(self, obj: Any, *, context: CaptureContext) -> NormalizedForecast:
|
|
15
|
+
...
|
|
16
|
+
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
from forecastops.adapters.base import ForecastAdapter
|
|
9
|
+
from forecastops.core.normalize import normalize_dataframe
|
|
10
|
+
from forecastops.core.run import CaptureContext, DetectionResult, NormalizedForecast
|
|
11
|
+
from forecastops.core.schema import ForecastSchema
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class DartsAdapter(ForecastAdapter):
|
|
15
|
+
name = "darts"
|
|
16
|
+
|
|
17
|
+
def detect(self, obj: Any) -> DetectionResult:
|
|
18
|
+
objects = obj if isinstance(obj, list) else [obj]
|
|
19
|
+
matched = bool(objects) and all(
|
|
20
|
+
hasattr(item, "time_index") and (hasattr(item, "values") or hasattr(item, "pd_dataframe"))
|
|
21
|
+
for item in objects
|
|
22
|
+
)
|
|
23
|
+
return DetectionResult(
|
|
24
|
+
matched,
|
|
25
|
+
0.75 if matched else 0.0,
|
|
26
|
+
"Darts TimeSeries-like object" if matched else "not a Darts TimeSeries-like object",
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
def normalize(self, obj: Any, *, context: CaptureContext) -> NormalizedForecast:
|
|
30
|
+
objects = obj if isinstance(obj, list) else [obj]
|
|
31
|
+
rows: list[pd.DataFrame] = []
|
|
32
|
+
for index, series in enumerate(objects):
|
|
33
|
+
if hasattr(series, "pd_dataframe"):
|
|
34
|
+
values = series.pd_dataframe()
|
|
35
|
+
if isinstance(values, pd.DataFrame):
|
|
36
|
+
value_column = values.columns[0]
|
|
37
|
+
frame = values.reset_index().rename(columns={values.index.name or "index": "target_time"})
|
|
38
|
+
frame["prediction"] = frame[value_column]
|
|
39
|
+
else:
|
|
40
|
+
raise ValueError("Darts pd_dataframe() did not return a pandas DataFrame")
|
|
41
|
+
else:
|
|
42
|
+
frame = pd.DataFrame(
|
|
43
|
+
{
|
|
44
|
+
"target_time": list(series.time_index),
|
|
45
|
+
"prediction": np.asarray(series.values()).reshape(-1),
|
|
46
|
+
}
|
|
47
|
+
)
|
|
48
|
+
frame["series_id"] = (
|
|
49
|
+
context.series_id[index]
|
|
50
|
+
if isinstance(context.series_id, list) and index < len(context.series_id)
|
|
51
|
+
else getattr(series, "components", [None])[0]
|
|
52
|
+
if hasattr(series, "components")
|
|
53
|
+
else context.series_id or f"series_{index}"
|
|
54
|
+
)
|
|
55
|
+
rows.append(frame[["series_id", "target_time", "prediction"]])
|
|
56
|
+
df = pd.concat(rows, ignore_index=True)
|
|
57
|
+
context.model_name = context.model_name or "darts"
|
|
58
|
+
schema = ForecastSchema(series_id="series_id", target_time="target_time", prediction="prediction")
|
|
59
|
+
return normalize_dataframe(df, context=context, adapter_name=self.name, schema=schema)
|
|
60
|
+
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
from forecastops.adapters.base import ForecastAdapter
|
|
8
|
+
from forecastops.core.normalize import normalize_array, normalize_dataframe
|
|
9
|
+
from forecastops.core.run import CaptureContext, DetectionResult, NormalizedForecast
|
|
10
|
+
from forecastops.core.schema import ForecastSchema
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class GenericDataFrameAdapter(ForecastAdapter):
|
|
14
|
+
name = "dataframe"
|
|
15
|
+
|
|
16
|
+
def detect(self, obj: Any) -> DetectionResult:
|
|
17
|
+
if not isinstance(obj, pd.DataFrame):
|
|
18
|
+
return DetectionResult(False, 0.0, "object is not a pandas DataFrame")
|
|
19
|
+
has_target = any(column in obj.columns for column in ["target_time", "ds", "timestamp"])
|
|
20
|
+
has_prediction = any(column in obj.columns for column in ["yhat", "prediction", "forecast"])
|
|
21
|
+
matched = has_target and has_prediction
|
|
22
|
+
return DetectionResult(
|
|
23
|
+
matched,
|
|
24
|
+
0.55 if matched else 0.0,
|
|
25
|
+
"generic dataframe columns found" if matched else "missing target or prediction column",
|
|
26
|
+
[] if matched else ["target_time", "prediction"],
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
def normalize(self, obj: Any, *, context: CaptureContext) -> NormalizedForecast:
|
|
30
|
+
return normalize_dataframe(obj, context=context, adapter_name=self.name)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class SchemaDataFrameAdapter(ForecastAdapter):
|
|
34
|
+
name = "schema"
|
|
35
|
+
|
|
36
|
+
def detect(self, obj: Any) -> DetectionResult:
|
|
37
|
+
matched = isinstance(obj, pd.DataFrame)
|
|
38
|
+
return DetectionResult(matched, 0.8 if matched else 0.0, "schema mapping supplied")
|
|
39
|
+
|
|
40
|
+
def normalize(self, obj: Any, *, context: CaptureContext) -> NormalizedForecast:
|
|
41
|
+
schema = context.schema
|
|
42
|
+
if not isinstance(schema, ForecastSchema):
|
|
43
|
+
raise ValueError("schema adapter requires ForecastSchema")
|
|
44
|
+
return normalize_dataframe(obj, context=context, adapter_name=self.name, schema=schema)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class ArrayAdapter(ForecastAdapter):
|
|
48
|
+
name = "array"
|
|
49
|
+
|
|
50
|
+
def detect(self, obj: Any) -> DetectionResult:
|
|
51
|
+
if isinstance(obj, pd.DataFrame):
|
|
52
|
+
return DetectionResult(False, 0.0, "dataframe should use dataframe adapters")
|
|
53
|
+
if hasattr(obj, "__array__") or isinstance(obj, (list, tuple)):
|
|
54
|
+
return DetectionResult(
|
|
55
|
+
True,
|
|
56
|
+
0.25,
|
|
57
|
+
"array-like object; requires explicit target_time and cutoff context",
|
|
58
|
+
["target_time", "cutoff"],
|
|
59
|
+
)
|
|
60
|
+
return DetectionResult(False, 0.0, "object is not array-like")
|
|
61
|
+
|
|
62
|
+
def normalize(self, obj: Any, *, context: CaptureContext) -> NormalizedForecast:
|
|
63
|
+
return normalize_array(obj, context=context)
|
|
64
|
+
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
from forecastops.adapters.base import ForecastAdapter
|
|
9
|
+
from forecastops.core.normalize import normalize_dataframe
|
|
10
|
+
from forecastops.core.run import CaptureContext, DetectionResult, NormalizedForecast
|
|
11
|
+
from forecastops.core.schema import ForecastSchema
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class GluonTSAdapter(ForecastAdapter):
|
|
15
|
+
name = "gluonts"
|
|
16
|
+
|
|
17
|
+
def detect(self, obj: Any) -> DetectionResult:
|
|
18
|
+
objects = obj if isinstance(obj, list) else [obj]
|
|
19
|
+
matched = bool(objects) and all(
|
|
20
|
+
hasattr(item, "start_date")
|
|
21
|
+
and hasattr(item, "prediction_length")
|
|
22
|
+
and (hasattr(item, "quantile") or hasattr(item, "samples"))
|
|
23
|
+
for item in objects
|
|
24
|
+
)
|
|
25
|
+
return DetectionResult(
|
|
26
|
+
matched,
|
|
27
|
+
0.75 if matched else 0.0,
|
|
28
|
+
"GluonTS Forecast-like object" if matched else "not a GluonTS Forecast-like object",
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
def normalize(self, obj: Any, *, context: CaptureContext) -> NormalizedForecast:
|
|
32
|
+
objects = obj if isinstance(obj, list) else [obj]
|
|
33
|
+
frames = []
|
|
34
|
+
for index, forecast in enumerate(objects):
|
|
35
|
+
prediction_length = int(forecast.prediction_length)
|
|
36
|
+
freq = getattr(forecast, "freq", None) or getattr(forecast.start_date, "freqstr", None) or "D"
|
|
37
|
+
start = pd.Timestamp(forecast.start_date)
|
|
38
|
+
target_time = pd.date_range(start=start, periods=prediction_length, freq=freq)
|
|
39
|
+
frame = pd.DataFrame({"target_time": target_time})
|
|
40
|
+
frame["series_id"] = getattr(forecast, "item_id", None) or context.series_id or f"item_{index}"
|
|
41
|
+
quantiles: dict[float, str] = {}
|
|
42
|
+
if hasattr(forecast, "quantile"):
|
|
43
|
+
for quantile in [0.1, 0.5, 0.9]:
|
|
44
|
+
column = f"q{int(quantile * 100)}"
|
|
45
|
+
frame[column] = np.asarray(forecast.quantile(quantile)).reshape(-1)
|
|
46
|
+
quantiles[quantile] = column
|
|
47
|
+
frame["prediction"] = frame["q50"]
|
|
48
|
+
else:
|
|
49
|
+
samples = np.asarray(forecast.samples)
|
|
50
|
+
frame["prediction"] = np.median(samples, axis=0).reshape(-1)
|
|
51
|
+
frame["q10"] = np.quantile(samples, 0.1, axis=0).reshape(-1)
|
|
52
|
+
frame["q90"] = np.quantile(samples, 0.9, axis=0).reshape(-1)
|
|
53
|
+
quantiles = {0.1: "q10", 0.5: "prediction", 0.9: "q90"}
|
|
54
|
+
frames.append(frame)
|
|
55
|
+
df = pd.concat(frames, ignore_index=True)
|
|
56
|
+
context.model_name = context.model_name or "gluonts"
|
|
57
|
+
schema = ForecastSchema(
|
|
58
|
+
series_id="series_id",
|
|
59
|
+
target_time="target_time",
|
|
60
|
+
prediction="prediction",
|
|
61
|
+
quantiles=quantiles,
|
|
62
|
+
)
|
|
63
|
+
return normalize_dataframe(df, context=context, adapter_name=self.name, schema=schema)
|
|
64
|
+
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
from forecastops.adapters.base import ForecastAdapter
|
|
8
|
+
from forecastops.core.normalize import normalize_dataframe
|
|
9
|
+
from forecastops.core.run import CaptureContext, DetectionResult, NormalizedForecast
|
|
10
|
+
from forecastops.core.schema import ForecastSchema
|
|
11
|
+
|
|
12
|
+
RESERVED_COLUMNS = {"unique_id", "ds", "cutoff", "cutoff_time", "actual", "y"}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class NixtlaAdapter(ForecastAdapter):
|
|
16
|
+
name = "nixtla"
|
|
17
|
+
|
|
18
|
+
def detect(self, obj: Any) -> DetectionResult:
|
|
19
|
+
if not isinstance(obj, pd.DataFrame):
|
|
20
|
+
return DetectionResult(False, 0.0, "object is not a pandas DataFrame")
|
|
21
|
+
columns = set(obj.columns)
|
|
22
|
+
model_cols = [column for column in obj.columns if column not in RESERVED_COLUMNS]
|
|
23
|
+
matched = {"unique_id", "ds"}.issubset(columns) and bool(model_cols)
|
|
24
|
+
return DetectionResult(
|
|
25
|
+
matched,
|
|
26
|
+
0.9 if matched else 0.0,
|
|
27
|
+
"Nixtla-style dataframe with unique_id, ds, and model columns"
|
|
28
|
+
if matched
|
|
29
|
+
else "missing unique_id/ds/model columns",
|
|
30
|
+
[] if matched else ["unique_id", "ds", "model_col"],
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
def normalize(self, obj: Any, *, context: CaptureContext) -> NormalizedForecast:
|
|
34
|
+
model_col = context.adapter_options.get("model_col")
|
|
35
|
+
if model_col is None:
|
|
36
|
+
candidates = [column for column in obj.columns if column not in RESERVED_COLUMNS]
|
|
37
|
+
if not candidates:
|
|
38
|
+
raise ValueError("nixtla adapter requires at least one model output column")
|
|
39
|
+
model_col = candidates[0]
|
|
40
|
+
if model_col not in obj:
|
|
41
|
+
raise ValueError(f"model_col {model_col!r} not present in dataframe")
|
|
42
|
+
schema = ForecastSchema(
|
|
43
|
+
series_id="unique_id",
|
|
44
|
+
cutoff_time="cutoff_time" if "cutoff_time" in obj else None,
|
|
45
|
+
target_time="ds",
|
|
46
|
+
prediction=model_col,
|
|
47
|
+
actual="actual" if "actual" in obj else "y" if "y" in obj else None,
|
|
48
|
+
)
|
|
49
|
+
context.model_name = context.model_name or str(model_col)
|
|
50
|
+
return normalize_dataframe(obj, context=context, adapter_name=self.name, schema=schema)
|
|
51
|
+
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
from forecastops.adapters.base import ForecastAdapter
|
|
8
|
+
from forecastops.core.normalize import normalize_dataframe
|
|
9
|
+
from forecastops.core.run import CaptureContext, DetectionResult, NormalizedForecast
|
|
10
|
+
from forecastops.core.schema import ForecastSchema
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ProphetAdapter(ForecastAdapter):
|
|
14
|
+
name = "prophet"
|
|
15
|
+
|
|
16
|
+
def detect(self, obj: Any) -> DetectionResult:
|
|
17
|
+
if not isinstance(obj, pd.DataFrame):
|
|
18
|
+
return DetectionResult(False, 0.0, "object is not a pandas DataFrame")
|
|
19
|
+
columns = set(obj.columns)
|
|
20
|
+
matched = {"ds", "yhat"}.issubset(columns)
|
|
21
|
+
confidence = 0.95 if matched and {"yhat_lower", "yhat_upper"} & columns else 0.85 if matched else 0.0
|
|
22
|
+
return DetectionResult(
|
|
23
|
+
matched,
|
|
24
|
+
confidence,
|
|
25
|
+
"Prophet-like dataframe with ds and yhat" if matched else "missing ds/yhat columns",
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
def normalize(self, obj: Any, *, context: CaptureContext) -> NormalizedForecast:
|
|
29
|
+
schema = ForecastSchema(
|
|
30
|
+
target_time="ds",
|
|
31
|
+
prediction="yhat",
|
|
32
|
+
lower="yhat_lower" if "yhat_lower" in obj else None,
|
|
33
|
+
upper="yhat_upper" if "yhat_upper" in obj else None,
|
|
34
|
+
)
|
|
35
|
+
if context.model_name is None:
|
|
36
|
+
context.model_name = "prophet"
|
|
37
|
+
return normalize_dataframe(obj, context=context, adapter_name=self.name, schema=schema)
|
|
38
|
+
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from forecastops.adapters.base import ForecastAdapter
|
|
6
|
+
from forecastops.adapters.darts import DartsAdapter
|
|
7
|
+
from forecastops.adapters.dataframe import (
|
|
8
|
+
ArrayAdapter,
|
|
9
|
+
GenericDataFrameAdapter,
|
|
10
|
+
SchemaDataFrameAdapter,
|
|
11
|
+
)
|
|
12
|
+
from forecastops.adapters.gluonts import GluonTSAdapter
|
|
13
|
+
from forecastops.adapters.nixtla import NixtlaAdapter
|
|
14
|
+
from forecastops.adapters.prophet import ProphetAdapter
|
|
15
|
+
from forecastops.adapters.sklearn import SklearnArrayAdapter
|
|
16
|
+
from forecastops.core.run import CaptureContext
|
|
17
|
+
|
|
18
|
+
_CUSTOM_ADAPTERS: dict[str, ForecastAdapter] = {}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def adapter(name: str):
|
|
22
|
+
def decorator(cls: type[ForecastAdapter]) -> type[ForecastAdapter]:
|
|
23
|
+
_CUSTOM_ADAPTERS[name] = cls()
|
|
24
|
+
return cls
|
|
25
|
+
|
|
26
|
+
return decorator
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def builtin_adapters() -> list[ForecastAdapter]:
|
|
30
|
+
return [
|
|
31
|
+
DartsAdapter(),
|
|
32
|
+
GluonTSAdapter(),
|
|
33
|
+
ProphetAdapter(),
|
|
34
|
+
NixtlaAdapter(),
|
|
35
|
+
GenericDataFrameAdapter(),
|
|
36
|
+
SklearnArrayAdapter(),
|
|
37
|
+
ArrayAdapter(),
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def resolve_adapter(obj: Any, adapter_name: str | None, context: CaptureContext) -> ForecastAdapter:
|
|
42
|
+
if adapter_name:
|
|
43
|
+
if adapter_name == "schema":
|
|
44
|
+
return SchemaDataFrameAdapter()
|
|
45
|
+
adapters = {adapter.name: adapter for adapter in [*_CUSTOM_ADAPTERS.values(), *builtin_adapters()]}
|
|
46
|
+
if adapter_name not in adapters:
|
|
47
|
+
raise ValueError(f"Unknown adapter {adapter_name!r}")
|
|
48
|
+
return adapters[adapter_name]
|
|
49
|
+
if context.schema is not None:
|
|
50
|
+
return SchemaDataFrameAdapter()
|
|
51
|
+
|
|
52
|
+
candidates: list[tuple[float, ForecastAdapter]] = []
|
|
53
|
+
for candidate in [*_CUSTOM_ADAPTERS.values(), *builtin_adapters()]:
|
|
54
|
+
result = candidate.detect(obj)
|
|
55
|
+
if result.matched:
|
|
56
|
+
candidates.append((result.confidence, candidate))
|
|
57
|
+
if not candidates:
|
|
58
|
+
raise ValueError("Could not detect a ForecastOps adapter. Supply adapter= or schema= explicitly.")
|
|
59
|
+
return sorted(candidates, key=lambda item: item[0], reverse=True)[0][1]
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from forecastops.adapters.dataframe import ArrayAdapter
|
|
6
|
+
from forecastops.core.run import CaptureContext, DetectionResult, NormalizedForecast
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class SklearnArrayAdapter(ArrayAdapter):
|
|
10
|
+
name = "sklearn"
|
|
11
|
+
|
|
12
|
+
def detect(self, obj: Any) -> DetectionResult:
|
|
13
|
+
result = super().detect(obj)
|
|
14
|
+
if not result.matched:
|
|
15
|
+
return result
|
|
16
|
+
return DetectionResult(
|
|
17
|
+
True,
|
|
18
|
+
0.2,
|
|
19
|
+
"array-like model predictions; explicit target_time and cutoff context required",
|
|
20
|
+
["target_time", "cutoff"],
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
def normalize(self, obj: Any, *, context: CaptureContext) -> NormalizedForecast:
|
|
24
|
+
context.model_name = context.model_name or "sklearn"
|
|
25
|
+
normalized = super().normalize(obj, context=context)
|
|
26
|
+
normalized.adapter_name = self.name
|
|
27
|
+
return normalized
|
|
28
|
+
|
forecastops/cli/main.py
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import pandas as pd
|
|
8
|
+
import typer
|
|
9
|
+
import yaml
|
|
10
|
+
|
|
11
|
+
from forecastops.adapters.registry import resolve_adapter
|
|
12
|
+
from forecastops.core.capture import capture
|
|
13
|
+
from forecastops.core.compare import compare as compare_run
|
|
14
|
+
from forecastops.core.config import load_config, write_default_config
|
|
15
|
+
from forecastops.core.diff import diff as diff_runs
|
|
16
|
+
from forecastops.core.evaluate import evaluate as evaluate_run
|
|
17
|
+
from forecastops.core.report import report as generate_report
|
|
18
|
+
from forecastops.core.run import CaptureContext
|
|
19
|
+
from forecastops.core.schema import ForecastSchema
|
|
20
|
+
from forecastops.core.validate import validate_forecast
|
|
21
|
+
from forecastops.store.duckdb_index import DuckDBIndex
|
|
22
|
+
from forecastops.store.local import LocalStore
|
|
23
|
+
from forecastops.store.parquet import read_artifact
|
|
24
|
+
from forecastops.ui.server import ui as launch_ui
|
|
25
|
+
|
|
26
|
+
app = typer.Typer(help="ForecastOps local forecast observability.")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@app.command()
|
|
30
|
+
def init(
|
|
31
|
+
store: Path = typer.Option(Path(".forecastops"), help="Local ForecastOps store path."),
|
|
32
|
+
config: Path = typer.Option(Path("forecastops.yaml"), help="Config file path."),
|
|
33
|
+
) -> None:
|
|
34
|
+
local_store = LocalStore.from_path(store)
|
|
35
|
+
local_store.init()
|
|
36
|
+
DuckDBIndex(local_store).init()
|
|
37
|
+
write_default_config(config)
|
|
38
|
+
typer.echo(f"Initialized {local_store.root}")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@app.command("capture")
|
|
42
|
+
def capture_file(
|
|
43
|
+
forecast_path: Path = typer.Argument(..., help="Forecast dataframe file."),
|
|
44
|
+
project: str = typer.Option("default", help="Project name."),
|
|
45
|
+
schema: Path | None = typer.Option(None, help="YAML schema mapping."),
|
|
46
|
+
adapter: str | None = typer.Option(None, help="Adapter name."),
|
|
47
|
+
cutoff: str | None = typer.Option(None, help="Forecast cutoff time."),
|
|
48
|
+
series_id: str | None = typer.Option(None, help="Single series id."),
|
|
49
|
+
actuals: Path | None = typer.Option(None, help="Actuals dataframe file."),
|
|
50
|
+
benchmark: Path | None = typer.Option(None, help="Benchmark dataframe file."),
|
|
51
|
+
benchmark_name: str = typer.Option("benchmark", help="Benchmark name."),
|
|
52
|
+
model_name: str | None = typer.Option(None, help="Model name."),
|
|
53
|
+
store: Path | None = typer.Option(None, help="Local store path."),
|
|
54
|
+
) -> None:
|
|
55
|
+
forecast = _read_frame(forecast_path)
|
|
56
|
+
run = capture(
|
|
57
|
+
forecast,
|
|
58
|
+
project=project,
|
|
59
|
+
schema=_read_schema(schema),
|
|
60
|
+
adapter=adapter,
|
|
61
|
+
cutoff=cutoff,
|
|
62
|
+
series_id=series_id,
|
|
63
|
+
actuals=_read_frame(actuals) if actuals else None,
|
|
64
|
+
benchmark=_read_frame(benchmark) if benchmark else None,
|
|
65
|
+
benchmark_name=benchmark_name,
|
|
66
|
+
model_name=model_name,
|
|
67
|
+
store=store,
|
|
68
|
+
)
|
|
69
|
+
typer.echo(json.dumps({"run_id": run.run_id, "status": run.status}, indent=2))
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@app.command()
|
|
73
|
+
def lint(
|
|
74
|
+
forecast_path: Path = typer.Argument(..., help="Forecast dataframe file."),
|
|
75
|
+
schema: Path | None = typer.Option(None, help="YAML schema mapping."),
|
|
76
|
+
adapter: str | None = typer.Option(None, help="Adapter name."),
|
|
77
|
+
project: str = typer.Option("lint", help="Project context."),
|
|
78
|
+
cutoff: str | None = typer.Option(None, help="Forecast cutoff time."),
|
|
79
|
+
series_id: str | None = typer.Option(None, help="Single series id."),
|
|
80
|
+
allow_insample: bool = typer.Option(False, help="Allow target_time <= cutoff_time."),
|
|
81
|
+
) -> None:
|
|
82
|
+
frame = _read_frame(forecast_path)
|
|
83
|
+
schema_obj = _read_schema(schema)
|
|
84
|
+
context = CaptureContext(
|
|
85
|
+
project=project,
|
|
86
|
+
run_id="lint",
|
|
87
|
+
cutoff=cutoff,
|
|
88
|
+
series_id=series_id,
|
|
89
|
+
schema=schema_obj,
|
|
90
|
+
model_name="lint",
|
|
91
|
+
)
|
|
92
|
+
adapter_impl = resolve_adapter(frame, adapter, context)
|
|
93
|
+
normalized = adapter_impl.normalize(frame, context=context).frame
|
|
94
|
+
events = validate_forecast(normalized, allow_insample=allow_insample)
|
|
95
|
+
for event in events:
|
|
96
|
+
typer.echo(
|
|
97
|
+
f"{event.severity:<5} {event.code}: {event.message}"
|
|
98
|
+
+ (f" ({event.affected_count})" if event.affected_count else "")
|
|
99
|
+
)
|
|
100
|
+
if any(event.is_error for event in events):
|
|
101
|
+
raise typer.Exit(1)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@app.command()
|
|
105
|
+
def evaluate(
|
|
106
|
+
run_id: str = typer.Argument(..., help="Run id."),
|
|
107
|
+
actuals: Path | None = typer.Option(None, help="Actuals dataframe file."),
|
|
108
|
+
store: Path | None = typer.Option(None, help="Local store path."),
|
|
109
|
+
) -> None:
|
|
110
|
+
index = DuckDBIndex(LocalStore.from_path(store))
|
|
111
|
+
run = index.run_by_id(run_id)
|
|
112
|
+
if not run:
|
|
113
|
+
raise typer.BadParameter(f"Run {run_id!r} not found")
|
|
114
|
+
result = evaluate_run(
|
|
115
|
+
read_artifact(run["forecast_artifact_uri"]),
|
|
116
|
+
run_id=run_id,
|
|
117
|
+
actuals=_read_frame(actuals) if actuals else None,
|
|
118
|
+
)
|
|
119
|
+
index.insert_metrics(result.metrics)
|
|
120
|
+
typer.echo(result.to_frame().to_string(index=False))
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@app.command()
|
|
124
|
+
def compare(
|
|
125
|
+
run_id: str = typer.Argument(..., help="Run id."),
|
|
126
|
+
benchmark: Path = typer.Option(..., help="Benchmark dataframe file."),
|
|
127
|
+
benchmark_name: str = typer.Option("benchmark", help="Benchmark name."),
|
|
128
|
+
store: Path | None = typer.Option(None, help="Local store path."),
|
|
129
|
+
) -> None:
|
|
130
|
+
index = DuckDBIndex(LocalStore.from_path(store))
|
|
131
|
+
run = index.run_by_id(run_id)
|
|
132
|
+
if not run:
|
|
133
|
+
raise typer.BadParameter(f"Run {run_id!r} not found")
|
|
134
|
+
result = compare_run(
|
|
135
|
+
read_artifact(run["forecast_artifact_uri"]),
|
|
136
|
+
benchmark=_read_frame(benchmark),
|
|
137
|
+
benchmark_name=benchmark_name,
|
|
138
|
+
)
|
|
139
|
+
index.insert_metrics(result.metrics)
|
|
140
|
+
typer.echo(result.to_frame().to_string(index=False))
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@app.command()
|
|
144
|
+
def diff(
|
|
145
|
+
base_run_id: str = typer.Argument(..., help="Base run id."),
|
|
146
|
+
candidate_run_id: str = typer.Argument(..., help="Candidate run id."),
|
|
147
|
+
store: Path | None = typer.Option(None, help="Local store path."),
|
|
148
|
+
) -> None:
|
|
149
|
+
index = DuckDBIndex(LocalStore.from_path(store))
|
|
150
|
+
base = index.run_by_id(base_run_id)
|
|
151
|
+
candidate = index.run_by_id(candidate_run_id)
|
|
152
|
+
if not base or not candidate:
|
|
153
|
+
raise typer.BadParameter("Both runs must exist")
|
|
154
|
+
result = diff_runs(base["forecast_artifact_uri"], candidate["forecast_artifact_uri"])
|
|
155
|
+
typer.echo(result.metric_deltas.to_string(index=False))
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
@app.command()
|
|
159
|
+
def report(
|
|
160
|
+
run_id: str | None = typer.Argument(None, help="Run id. Omit with --latest."),
|
|
161
|
+
latest: bool = typer.Option(False, help="Use latest run."),
|
|
162
|
+
out: Path | None = typer.Option(None, help="Output HTML path."),
|
|
163
|
+
store: Path | None = typer.Option(None, help="Local store path."),
|
|
164
|
+
) -> None:
|
|
165
|
+
output = generate_report(None if latest or run_id is None else run_id, out=out, store=store)
|
|
166
|
+
typer.echo(str(output))
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@app.command()
|
|
170
|
+
def ui(
|
|
171
|
+
host: str | None = typer.Option(None, help="Bind host."),
|
|
172
|
+
port: int | None = typer.Option(None, help="Bind port."),
|
|
173
|
+
store: Path | None = typer.Option(None, help="Local store path."),
|
|
174
|
+
no_open: bool = typer.Option(False, help="Do not open a browser window."),
|
|
175
|
+
allow_remote: bool = typer.Option(
|
|
176
|
+
False, help="Allow binding to a non-loopback host (no authentication; use with care)."
|
|
177
|
+
),
|
|
178
|
+
) -> None:
|
|
179
|
+
launch_ui(
|
|
180
|
+
host=host,
|
|
181
|
+
port=port,
|
|
182
|
+
store=store,
|
|
183
|
+
open_browser=not no_open,
|
|
184
|
+
allow_remote=allow_remote,
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@app.command()
|
|
189
|
+
def doctor(store: Path | None = typer.Option(None, help="Local store path.")) -> None:
|
|
190
|
+
config = load_config()
|
|
191
|
+
local_store = LocalStore.from_path(store or config.store)
|
|
192
|
+
index = DuckDBIndex(local_store)
|
|
193
|
+
index.init()
|
|
194
|
+
latest = index.latest_run_id()
|
|
195
|
+
typer.echo(
|
|
196
|
+
json.dumps(
|
|
197
|
+
{
|
|
198
|
+
"store": str(local_store.root),
|
|
199
|
+
"db": str(local_store.db_path),
|
|
200
|
+
"latest_run_id": latest,
|
|
201
|
+
"ui": {"host": config.ui_host, "port": config.ui_port},
|
|
202
|
+
"otel_enabled": config.otel_enabled,
|
|
203
|
+
},
|
|
204
|
+
indent=2,
|
|
205
|
+
)
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _read_frame(path: Path | None) -> pd.DataFrame:
|
|
210
|
+
if path is None:
|
|
211
|
+
raise ValueError("Path is required")
|
|
212
|
+
suffix = path.suffix.lower()
|
|
213
|
+
if suffix in {".parquet", ".pq"}:
|
|
214
|
+
return pd.read_parquet(path)
|
|
215
|
+
if suffix == ".csv":
|
|
216
|
+
return pd.read_csv(path)
|
|
217
|
+
if suffix in {".json", ".jsonl"}:
|
|
218
|
+
return pd.read_json(path, lines=suffix == ".jsonl")
|
|
219
|
+
raise ValueError(f"Unsupported dataframe file type: {path.suffix}")
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _read_schema(path: Path | None) -> ForecastSchema | None:
|
|
223
|
+
if path is None:
|
|
224
|
+
return None
|
|
225
|
+
with path.open("r", encoding="utf-8") as handle:
|
|
226
|
+
data: dict[str, Any] = yaml.safe_load(handle) or {}
|
|
227
|
+
if not data:
|
|
228
|
+
return None
|
|
229
|
+
return ForecastSchema.from_dict(data.get("schema", data))
|