corethink 1.0.3__tar.gz → 1.0.4__tar.gz

{corethink-1.0.3 → corethink-1.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: corethink
-Version: 1.0.3
+Version: 1.0.4
 Summary: The CoreThink Workbench from your terminal: environments, training, endpoints and more.
 Author-email: CoreThink <support@corethink.ai>
 License-Expression: MIT

{corethink-1.0.3 → corethink-1.0.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "corethink"
-version = "1.0.3"
+version = "1.0.4"
 description = "The CoreThink Workbench from your terminal: environments, training, endpoints and more."
 readme = "README.md"
 requires-python = ">=3.10"

{corethink-1.0.3 → corethink-1.0.4}/src/corethink/_version.py

@@ -1,3 +1,3 @@
 """The version of the `corethink` package (read by the build and by `corethink --version`)."""
 
-__version__ = "1.0.3"
+__version__ = "1.0.4"

{corethink-1.0.3 → corethink-1.0.4}/src/corethink/commands/train.py

@@ -1081,6 +1081,8 @@ def estimate_lines(estimate: dict[str, Any] | None, budget: float | None = None)
             text += " The limit is lower; it may stop early."
         elif estimate.get("costUsd") is None and first and (first.get("lowUsd") or 0) > budget:
             text += " The limit is below the low end; it will likely stop early."
+        elif estimate.get("costUsd") is None and first and (first.get("typicalUsd") or 0) > budget:
+            text += " The limit is below the likely cost; it may stop early."
     if estimate.get("waiting"):
         text += f" {plural(estimate['waiting'], 'run')} waiting ahead."
     lines = [text]
@@ -1212,6 +1214,16 @@ def _estimate(plan: Plan) -> dict[str, Any] | None:
         return None
 
 
+def eval_tasks(plan: Plan) -> int:
+    """Tasks in the final check: at most those held out from training."""
+    usecase = plan.usecase or {}
+    if usecase.get("split"):
+        return int(usecase["split"].get("test") or 0)
+    if usecase.get("scenarios"):
+        return min(plan.check_tasks, withheld_tasks(int(usecase["scenarios"]), plan.share))
+    return plan.check_tasks
+
+
 def _limits_panel(plan: Plan, budget: float, available: float | None, estimate: dict[str, Any] | None) -> None:
     """The console's "Limits for this run" box, on stderr (stdout is for the run itself)."""
     usecase = plan.usecase
@@ -1233,7 +1245,7 @@ def _limits_panel(plan: Plan, budget: float, available: float | None, estimate:
         ("Hyperparameters", "Advanced" if advanced else "Automatic"),
         ("Spending", f"max {ui.usd(budget, 0 if float(budget).is_integer() else 2)}"),
         ("Trains on", trains),
-        ("Eval set tasks", str(split.get("test") if split else plan.check_tasks)),
+        ("Eval set tasks", str(eval_tasks(plan))),
     ]
     if plan.method:
         rows.append(("Training method", METHOD_LABEL.get(plan.method, plan.method)))

{corethink-1.0.3 → corethink-1.0.4}/src/corethink.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: corethink
-Version: 1.0.3
+Version: 1.0.4
 Summary: The CoreThink Workbench from your terminal: environments, training, endpoints and more.
 Author-email: CoreThink <support@corethink.ai>
 License-Expression: MIT

{corethink-1.0.3 → corethink-1.0.4}/tests/test_train.py

@@ -979,3 +979,25 @@ def test_start_from_an_earlier_run_repeats_its_environment_and_settings(form, ru
 
 def test_start_needs_an_environment_or_a_run(form, run):
     assert run("train", "start", "--yes").code == 2
+
+
+def test_a_first_runs_limit_below_its_likely_cost_is_flagged():
+    from corethink.commands.train import estimate_lines
+
+    first = {"typicalUsd": 57.3, "lowUsd": 40.1, "highUsd": 85.9, "drivers": [], "assumptions": []}
+    estimate = {"costUsd": None, "minutes": None, "firstRun": first}
+    assert "may stop early" not in estimate_lines(estimate, 86)[0]
+    assert estimate_lines(estimate, 45)[0].endswith("The limit is below the likely cost; it may stop early.")
+    assert estimate_lines(estimate, 30)[0].endswith("The limit is below the low end; it will likely stop early.")
+
+
+def test_the_eval_set_is_at_most_the_tasks_held_out():
+    from corethink.commands.train import Plan, eval_tasks
+
+    made = Plan.__new__(Plan)
+    made.usecase, made.share, made.check_tasks = {"scenarios": 80}, 0.6, 40
+    assert eval_tasks(made) == 32  # 80 tasks, 48 to train on: 32 held out, not the 40 asked for
+    made.usecase = {"scenarios": 200}
+    assert eval_tasks(made) == 40
+    made.usecase = {"scenarios": 200, "split": {"train": 150, "test": 50}}
+    assert eval_tasks(made) == 50