nwp500-python 9.12.2__py3-none-any.whl → 9.13.0__py3-none-any.whl

nwp500/cli/__main__.py

@@ -4,6 +4,7 @@ import asyncio
 import functools
 import json
 import logging
+import math
 import sys
 from collections.abc import Awaitable, Callable
 from datetime import UTC, datetime
@@ -1134,14 +1135,14 @@ _tz_option = click.option(
 @click.option(
     "--bath",
     type=float,
-    help="The bath readiness is judged against, delivered, preferred unit "
-    "(default the parameters', 28.7 gal)",
+    help="The bath readiness is judged against, as the meter reads it, "
+    "preferred unit (default the parameters', 28.7 gal)",
 )
 @click.option(
     "--shower",
     type=float,
-    help="A shower, delivered, preferred unit (default the parameters', "
-    "12 gal)",
+    help="A shower, as the meter reads it, preferred unit (default the "
+    "parameters', 12 gal)",
 )
 @click.option(
     "--format",
@@ -1174,7 +1175,7 @@ def simulate_run(ctx: click.Context, output_format: str, **given: Any) -> None:
             live = await sim_cli.compose_live(
                 mqtt, ctx.obj["api"], device, inputs, now
             )
-            keeper = await sim_cli.read_book(inputs, now)
+            keeper = await sim_cli.read_tank(inputs, now)
         except sim_cli.InputError as e:
             raise click.UsageError(str(e)) from None
         walk(live, keeper)
@@ -1254,6 +1255,15 @@ async def simulate_record(
     await handle_recording(mqtt, device, history, every)
 
 
+def _finite_weeks(
+    ctx: click.Context, param: click.Parameter, value: float
+) -> float:
+    """``--tank-weeks`` as a finite number of weeks."""
+    if not math.isfinite(value):
+        raise click.BadParameter(f"{value} is not a number of weeks")
+    return value
+
+
 @simulate.command("calibrate")  # type: ignore[attr-defined]
 @click.argument("history", type=click.Path(exists=True, dir_okay=False))
 @click.option(
@@ -1291,6 +1301,25 @@ async def simulate_record(
     help="A TOML file naming where the air, its humidity, the hot water "
     "demand and the circulation are recorded (Home Assistant)",
 )
+@click.option(
+    "--holdout-from",
+    "holdout_from",
+    metavar="DATE",
+    help="Fit on the history before DATE (ISO 8601, local time without "
+    "an offset) and score the base and the fit on the events from it",
+)
+@click.option(
+    "--tank-weeks",
+    "tank_weeks",
+    type=click.FloatRange(min=0.0),
+    callback=_finite_weeks,
+    default=7.0,
+    show_default=True,
+    metavar="WEEKS",
+    help="Fit the tank on its events in the last WEEKS of the history "
+    "(to its last status before --holdout-from), the history before read "
+    "as their lead-in; 0 for the whole history",
+)
 @click.pass_context
 def simulate_calibrate(
     ctx: click.Context,
@@ -1301,10 +1330,16 @@ def simulate_calibrate(
     humidity: str | None,
     params: str | None,
     sources: str | None,
+    holdout_from: str | None,
+    tank_weeks: float,
 ) -> None:
     """Fit the heater's parameters to its recorded HISTORY.
 
     Prints what each fit found, and why a term kept the base's value.
+    With --holdout-from, the fit sees only the history before it, and
+    the base and the fit are scored on the recorded events after it.
+    The tank is fitted on the last --tank-weeks of what the fit sees,
+    the heat pump on all of it.
     """
     sim_cli = _simulator()
     from . import calibration as cal_cli
@@ -1320,21 +1355,31 @@ def simulate_calibrate(
             record = asyncio.run(
                 cal_cli.read_record(sim_cli.read_sources(sources), readings)
             )
-        done = cal_cli.calibrate(
-            readings,
-            air=None if air is None else cal_cli.read_readings(air, "air"),
-            ambient=ambient,
-            humidity=None
+        inputs: dict[str, Any] = {
+            "air": None if air is None else cal_cli.read_readings(air, "air"),
+            "ambient": ambient,
+            "humidity": None
             if humidity is None
             else cal_cli.read_readings(humidity, "humidity"),
-            params=None
+            "params": None
             if params is None
             else sim_cli.read_document(params, "parameters"),
-            record=record,
-        )
+            "record": record,
+            "tank_weeks": tank_weeks or None,
+        }
+        held = None
+        if holdout_from is None:
+            done = cal_cli.calibrate(readings, **inputs)
+        else:
+            held = cal_cli.holdout(
+                readings,
+                cal_cli.parse_instant(holdout_from, "--holdout-from"),
+                **inputs,
+            )
+            done = held.done
     except sim_cli.InputError as e:
         raise click.UsageError(str(e)) from None
-    cal_cli.run(done, output)
+    cal_cli.run(done, output, held=held)
 
 
 if __name__ == "__main__":

nwp500/cli/calibration.py

@@ -10,12 +10,14 @@ This module needs the ``sim`` extra.
 """
 
 import csv
+import functools
 import json
 import logging
 import math
 import sys
 from collections.abc import Sequence
-from dataclasses import replace
+from concurrent.futures import ProcessPoolExecutor
+from dataclasses import dataclass, replace
 from datetime import datetime
 from pathlib import Path
 from typing import Any, TextIO
@@ -23,12 +25,14 @@ from typing import Any, TextIO
 from rich.console import Console
 from rich.table import Table
 
+from nwp500 import sim, status_history
 from nwp500 import sim_calibrate as sc
-from nwp500 import status_history
+from nwp500 import sim_score as ss
+from nwp500.sim import tank_fit as tf
 from nwp500.sim.params import params_to_document
 from nwp500.sim_calibrate import Calibrated, Term, report
 
-from .simulate import InputError, read_params
+from .simulate import InputError, local_zone, read_params
 
 __author__ = "Emmanuel Levijarvi"
 __copyright__ = "Emmanuel Levijarvi"
@@ -102,9 +106,12 @@ _OPTION_NOTES = {
     sc.DEFAULT_BASE_NOTE: "refitting the 65 gal defaults (give --params "
     "for another tank size)",
     sc.NO_AIR_NOTE: "no air given (--air, --ambient or --sources): the "
-    "standby loss is not fitted",
+    "tank is not fitted",
 }
 
+#: The events each of the tank fit's workers takes at a time.
+_CHUNK = 8
+
 
 def calibrate(
     readings: Sequence[status_history.TimedStatus],
@@ -114,20 +121,25 @@ def calibrate(
     humidity: Sequence[Reading] | None = None,
     params: Any = None,
     record: Any = None,
+    tank_weeks: float | None = sc.TANK_WEEKS,
 ) -> Calibrated:
     """``sim_calibrate.calibrate`` with ``params`` a parameters
     document, its errors as ``InputError`` and its notes naming the
-    command's options."""
+    command's options, the tank's fit spread over the machine's
+    processors. ``tank_weeks`` as for ``sim_calibrate.calibrate``."""
     base = None if params is None else read_params(params, "--params")
     try:
-        done = sc.calibrate(
-            readings,
-            air=air,
-            ambient=ambient,
-            humidity=humidity,
-            params=base,
-            record=record,
-        )
+        with ProcessPoolExecutor() as pool:
+            done = sc.calibrate(
+                readings,
+                air=air,
+                ambient=ambient,
+                humidity=humidity,
+                params=base,
+                record=record,
+                map_=functools.partial(pool.map, chunksize=_CHUNK),
+                tank_weeks=tank_weeks,
+            )
     except sc.CalibrationInputError as e:
         raise InputError(str(e)) from None
     return replace(
@@ -135,6 +147,117 @@ def calibrate(
     )
 
 
+def parse_instant(text: str, what: str) -> datetime:
+    """An ISO 8601 date or instant; a date, or an instant without its
+    offset, in local time."""
+    try:
+        at = datetime.fromisoformat(text.strip())
+    except ValueError:
+        raise InputError(f"{what} {text!r} is not an ISO 8601 date") from None
+    return at.astimezone() if at.tzinfo is None else at
+
+
+@dataclass(frozen=True)
+class HeldOutScore:
+    """A calibration on the history before ``start``, and the base's
+    and the fit's scores on the events from it (``sim_score``)."""
+
+    start: datetime
+    done: Calibrated
+    base: ss.Score
+    fitted: ss.Score
+
+
+def holdout(
+    readings: Sequence[status_history.TimedStatus],
+    start: datetime,
+    *,
+    air: Sequence[Reading] | None = None,
+    ambient: float | None = None,
+    humidity: Sequence[Reading] | None = None,
+    params: Any = None,
+    record: Any = None,
+    tank_weeks: float | None = sc.TANK_WEEKS,
+) -> HeldOutScore:
+    """``calibrate`` on the statuses before ``start``, the tank on its
+    last ``tank_weeks`` before it, then the base and the fit scored on
+    the events from it. Raises ``InputError`` when
+    either side of ``start`` holds no status."""
+    before = [r for r in readings if r[0] < start]
+    if not before or len(before) == len(readings):
+        raise InputError(
+            f"--holdout-from {start.isoformat()} leaves no history on one "
+            "side of it"
+        )
+    done = calibrate(
+        before,
+        air=air,
+        ambient=ambient,
+        humidity=humidity,
+        params=params,
+        record=record,
+        tank_weeks=tank_weeks,
+    )
+    base = (
+        sim.default_params()
+        if params is None
+        else read_params(params, "--params")
+    )
+    try:
+        ready = sc.prepare(
+            readings, air=air, ambient=ambient, humidity=humidity, record=record
+        )
+    except sc.CalibrationInputError as e:
+        raise InputError(str(e)) from None
+    found = ss.events(ready, since=start)
+    # Each event starts from the layers the history before it leaves,
+    # which needs the air read somewhere in it.
+    air_f = ready.frame.get("air_f")
+    replayed = (
+        tf.tank_record(ready.frame)
+        if air_f is not None and bool(air_f.notna().any())
+        else None
+    )
+    # Local days, as ``--holdout-from`` is read.
+    tz = local_zone() or "UTC"
+    return HeldOutScore(
+        start,
+        done,
+        ss.score(found, base, tz=tz, record=replayed),
+        ss.score(found, done.calibration.params, tz=tz, record=replayed),
+    )
+
+
+def print_score(held: HeldOutScore, console: Console) -> None:
+    """The held-out score as a table on ``console``."""
+    events = held.fitted.events
+    console.print(
+        f"Held out from {held.start.isoformat()}: "
+        f"{len(events['recoveries'])} recoveries, "
+        f"{len(events['windows'])} quiet windows"
+    )
+    table = Table(title="Held-out score (lower is better, but a bias)")
+    for column in ("Metric", "Events", "Base", "Fitted", "Fitted 90 %"):
+        table.add_column(column)
+    for name, fitted in held.fitted.metrics.items():
+        base = held.base.metrics.get(name)
+        table.add_row(
+            name,
+            str(fitted.n),
+            "" if base is None else f"{base.value:.3g}",
+            f"{fitted.value:.3g}",
+            f"{fitted.lo:.3g} to {fitted.hi:.3g}",
+        )
+    console.print(table)
+    failed = sum(held.fitted.failed.values())
+    broken = sum(held.fitted.breaks.values())
+    if failed or broken:
+        console.print(
+            f"Fitted: {failed} events the simulator could not answer, "
+            f"{broken} predictions below the air"
+        )
+
+
 def _kept(t: Term) -> str:
     if t.fitted:
         return f"fitted: {t.note}" if t.note else "fitted"
@@ -145,8 +268,7 @@ def print_report(done: Calibrated, console: Console) -> None:
     """The report as tables on ``console``."""
     c = done.calibration
     console.print(
-        f"{done.n_statuses} statuses; quiet test: {c.standby.quiet_by}; "
-        f"{len(c.recoveries)} recoveries found"
+        f"{done.n_statuses} statuses; {len(c.recoveries)} recoveries found"
     )
     if done.n_gaps:
         minutes = done.max_gap.total_seconds() / 60
@@ -181,12 +303,19 @@ def print_report(done: Calibrated, console: Console) -> None:
 
 
 def run(
-    done: Calibrated, output: str | Path | None, out: TextIO | None = None
+    done: Calibrated,
+    output: str | Path | None,
+    out: TextIO | None = None,
+    held: HeldOutScore | None = None,
 ) -> None:
-    """Print the report and write the parameters to ``output``."""
+    """Print the report, and ``held``'s score, and write the parameters
+    to ``output``."""
     for note in done.notes:
         _logger.warning(note)
-    print_report(done, Console(file=out or sys.stdout))
+    console = Console(file=out or sys.stdout)
+    print_report(done, console)
+    if held is not None:
+        print_score(held, console)
     if output is not None:
         text = json.dumps(params_to_document(done.calibration.params), indent=2)
         Path(output).write_text(text + "\n", encoding="utf-8")

nwp500/cli/simulate.py

@@ -11,7 +11,6 @@ This module needs the ``sim`` extra; ``__main__`` imports it only when a
 """
 
 import csv
-import dataclasses
 import json
 import logging
 import os
@@ -43,7 +42,7 @@ if TYPE_CHECKING:
     from nwp500.api_client import NavienAPIClient
     from nwp500.models import Device
     from nwp500.mqtt import NavienMqttClient
-    from nwp500.sim_home_assistant import HomeAssistantBook
+    from nwp500.sim_home_assistant import HomeAssistantTank
 
 __author__ = "Emmanuel Levijarvi"
 __copyright__ = "Emmanuel Levijarvi"
@@ -130,8 +129,8 @@ class Inputs:
     #: Where the air, its humidity and the inlet are read live
     #: (``sim_home_assistant.Sources``), when the heater is.
     sources: Any = None
-    #: The bath and the shower readiness is judged against, delivered,
-    #: in the preferred unit; None: the parameters'.
+    #: The bath and the shower readiness is judged against, as the meter
+    #: reads them, in the preferred unit; None: the parameters'.
     bath: float | None = None
     shower: float | None = None
 
@@ -278,18 +277,18 @@ async def compose_live(
     return composed
 
 
-async def read_book(inputs: Inputs, now: datetime) -> HomeAssistantBook | None:
-    """The heat book's keeper (``sim_home_assistant.HomeAssistantBook``)
-    with the history read to ``now``, when the sources name the heater's
-    entities and the draw flow; None otherwise. Its ``book`` makes no
-    request, so it books after the client is closed."""
+async def read_tank(inputs: Inputs, now: datetime) -> HomeAssistantTank | None:
+    """The tank's keeper (``sim_home_assistant.HomeAssistantTank``) with
+    the history read to ``now``, when the sources name the heater's
+    entities and the draw flow; None otherwise. Its ``layers`` makes no
+    request, so it replays after the client is closed."""
     src = inputs.sources
     if src is None or src.heater is None or src.entities.draw_flow is None:
         return None
     from nwp500 import sim_home_assistant as ha
 
     async with ha.HomeAssistantClient(src.url, src.token) as client:
-        keeper = ha.HomeAssistantBook(
+        keeper = ha.HomeAssistantTank(
             client,
             src.heater,
             src.entities.draw_flow,
@@ -300,34 +299,37 @@ async def read_book(inputs: Inputs, now: datetime) -> HomeAssistantBook | None:
     return keeper
 
 
-def booked(
-    keeper: HomeAssistantBook | None, built: Built, now: datetime
-) -> sim.Booked | None:
-    """The tank's booked heat at the scenario's start, from ``keeper``
-    (``read_book``): only for a scenario that starts ``now``, whose
-    state is the heater's now. Its gallons convert at the scenario's
-    inlet where one is given, as the walk takes it. Its notes are
-    logged."""
-    if keeper is None:
-        return None
-    start = built.scenario.start
-    if start != now:
+def layered(
+    keeper: HomeAssistantTank | None, built: Built, now: datetime
+) -> Built:
+    """``built`` starting from the tank's layers its history leaves
+    (``HomeAssistantTank.layers``), the scenario's probes laid over
+    them: only for a scenario that starts ``now``, whose state is the
+    heater's now, and gives no layers of its own. Without them, ``built``
+    as it is, which the simulator starts from the two probes. The notes
+    are logged."""
+    state = built.scenario.state
+    if keeper is None or state.layers_f is not None:
+        return built
+    if built.scenario.start != now:
         _logger.warning(
-            "book: the scenario starts at %s, not now: walked without "
-            "the heat book",
-            start.isoformat(),
+            "layers: the scenario starts at %s, not now: walked from "
+            "the two probes",
+            built.scenario.start.isoformat(),
         )
-        return None
-    answer = keeper.book(now, built.params)
+        return built
+    answer = keeper.layers(now, built.params, (state.upper_f, state.lower_f))
     for note in answer.notes:
-        if answer.booked is None:
+        if answer.layers_f is None:
             _logger.warning(note)
         else:
             _logger.info(note)
-    inlet = built.scenario.environment.inlet_f
-    if answer.booked is None or inlet is None:
-        return answer.booked
-    return dataclasses.replace(answer.booked, inlet_f=inlet)
+    if answer.layers_f is None:
+        return built
+    scenario = built.scenario.model_copy(
+        update={"state": state.model_copy(update={"layers_f": answer.layers_f})}
+    )
+    return Built(scenario, built.params, built.notes)
 
 
 def window(inputs: Inputs, now: datetime) -> tuple[datetime, timedelta]:
@@ -413,7 +415,6 @@ def energy_totals(
 
 def hot_water_lines(
     built: Built,
-    book: sim.Booked | None,
     bath_gal: float | None = None,
     shower_gal: float | None = None,
 ) -> list[str]:
@@ -425,7 +426,6 @@ def hot_water_lines(
         r = sim.time_to_ready(
             built.scenario,
             built.params,
-            book=book,
             bath_gal=bath_gal,
             shower_gal=shower_gal,
         )
@@ -453,23 +453,28 @@ def print_table(
     built: Built,
     result: sim.SimulationResult,
     console: Console,
-    book: sim.Booked | None = None,
     bath_gal: float | None = None,
     shower_gal: float | None = None,
 ) -> None:
-    """The heating intervals, the energy by source, the readiness (from
-    ``book`` when given) with the bath and the showers, the draw windows
+    """The heating intervals, the energy by source, the readiness with
+    the bath and the showers, the draw windows
     (and which ran cold), the reasons and the defaults."""
     s = built.scenario
     tz = s.tz
-    ready = sim.readiness(s.state, s.settings, built.params, book=book)
+    ready = sim.readiness(
+        s.state,
+        s.settings,
+        built.params,
+        inlet_f=s.environment.inlet_f,
+        on=s.start.date(),
+    )
     console.print(
         f"[bold]{_local(s.start, tz)} to {_local(s.until, tz)}[/bold] ({tz}): "
         f"tank {_temp(s.state.upper_f)} upper, {_temp(s.state.lower_f)} "
         f"lower, setpoint {_temp(s.settings.setpoint_f)}; "
         f"{_volume(ready.gal)} deliverable now ({ready.basis})"
     )
-    for line in hot_water_lines(built, book, bath_gal, shower_gal):
+    for line in hot_water_lines(built, bath_gal, shower_gal):
         console.print(line)
 
     heating = Table(title="Heating")
@@ -565,16 +570,15 @@ def run(
     now: datetime,
     output_format: str,
     out: TextIO | None = None,
-    keeper: HomeAssistantBook | None = None,
+    keeper: HomeAssistantTank | None = None,
 ) -> sim.SimulationResult:
     """Build the scenario, walk it and print the result: from the tank's
-    booked heat when ``keeper`` (``read_book``) books one."""
-    built = build(inputs, live, now)
+    layers when ``keeper`` (``read_tank``) gives them."""
+    built = layered(keeper, build(inputs, live, now), now)
     for note in built.notes:
         _logger.warning(note)
-    book = booked(keeper, built, now)
     try:
-        result = sim.simulate(built.scenario, built.params, book=book)
+        result = sim.simulate(built.scenario, built.params)
     except SimulationError as e:
         raise InputError(f"the scenario cannot be simulated: {e}") from None
     stream = out or sys.stdout
@@ -588,7 +592,6 @@ def run(
             built,
             result,
             Console(file=stream),
-            book,
             None if inputs.bath is None else sa.to_sim_volume(inputs.bath),
             None if inputs.shower is None else sa.to_sim_volume(inputs.shower),
         )

nwp500/sim/__init__.py

@@ -10,12 +10,14 @@
   can deliver now; ``time_to_ready(scenario, params)``: whether it can
   run a bath, when it can in each heating mode, and the showers it is
   ready for;
-- ``element_mode(state, settings, params, mode=..., at=...)``: a
-  recovery in an element mode;
+- ``element_mode(scenario, params, mode=...)``: a stint in an element
+  mode, the scenario walked with the mode written at its start;
+- ``carry_on(walks, tail, params)``: each walk's end walked on through
+  one tail, so changes are weighed with what each leaves behind;
 - ``default_params(volume_code)``: one NWP500-65's parameters with no
   mixing valve; ``site_example_params`` that installation whole, valve
   included (``with_valve`` sets one); the parameters of one NWP500-65,
-  with the tank volume scaled for the 50 and 80 gal models.
+  with the tank resized to the 50 and 80 gal models' (``defaults.sized``).
 
 ``nwp500.sim_adapters`` builds a scenario's parts from the library's
 models; ``sim.codec`` reads and writes scenarios as JSON documents.
@@ -32,9 +34,11 @@ from nwp500.sim.api import (
     standby,
     time_to_ready,
 )
-from nwp500.sim.book import Booked, full_tank_book
+from nwp500.sim.carry import SETTLED_F, Carried, carry_on
 from nwp500.sim.defaults import (
     NO_VALVE,
+    SITE_UNDILUTED_GPM,
+    SITE_UNDILUTED_SHARE,
     SITE_VALVE,
     default_params,
     site_example_params,
@@ -57,8 +61,8 @@ from nwp500.sim.inputs import (
     TouPeriod,
     Write,
 )
+from nwp500.sim.margins import StartMargin, start_margins
 from nwp500.sim.params import (
-    BookParams,
     CirculationParams,
     HeaterParams,
     ReadinessParams,
@@ -72,6 +76,7 @@ from nwp500.sim.ready import (
 )
 from nwp500.sim.result import (
     ActiveCycle,
+    ElementInterval,
     ElementModeResult,
     Forecast,
     RecoveryResult,
@@ -83,14 +88,17 @@ __all__ = (
     "CirculationParams",
     "CirculationRun",
     "NO_VALVE",
+    "SITE_UNDILUTED_GPM",
+    "SITE_UNDILUTED_SHARE",
     "SITE_VALVE",
+    "SETTLED_F",
     "BathReady",
-    "BookParams",
-    "Booked",
+    "Carried",
     "Deliverable",
     "Draw",
     "DrawWindow",
     "DrawWindowResult",
+    "ElementInterval",
     "ElementModeResult",
     "Environment",
     "HeaterParams",
@@ -110,12 +118,13 @@ __all__ = (
     "Settings",
     "ShowersReady",
     "SimulationResult",
+    "StartMargin",
     "TouPeriod",
     "Trace",
     "Write",
+    "carry_on",
     "default_params",
     "element_mode",
-    "full_tank_book",
     "active_cycle",
     "forecast",
     "next_recovery",
@@ -124,6 +133,7 @@ __all__ = (
     "simulate",
     "site_example_params",
     "standby",
+    "start_margins",
     "recovery_as_of",
     "time_to_ready",
     "with_valve",