fastworkflow 3.2.0__tar.gz → 3.2.1__tar.gz

{fastworkflow-3.2.0 → fastworkflow-3.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fastworkflow
-Version: 3.2.0
+Version: 3.2.1
 Summary: A framework for rapidly building large-scale, deterministic, interactive workflows with a fault-tolerant, conversational UX
 License: Apache-2.0
 Keywords: fastworkflow,ai,workflow,llm,openai

{fastworkflow-3.2.0 → fastworkflow-3.2.1}/fastworkflow/model_pipeline_training.py

@@ -1,5 +1,5 @@
 import os
-from typing import Callable, ClassVar, Optional
+from typing import Callable, ClassVar, Iterable, Optional
 from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification
 from torch.optim import AdamW
 from sklearn.decomposition import PCA
@@ -55,50 +55,91 @@ class TrainingDataError(ValueError):
     """Raised before model fitting when a label cannot be trained and evaluated."""
 
 
+def _name_labels(
+    labels: Iterable[int],
+    decode_label: Optional[Callable[[int], str]] = None,
+) -> str:
+    """Render labels as a plain comma-separated list, sorted for a stable message.
+
+    Deliberately not a list repr. `decode_label` returns a numpy string, so a repr
+    renders as `[np.str_('cmd')]`, and the report prints through a rich console that
+    reads a bracketed run with no spaces as console markup and deletes it - which
+    silently dropped the one detail these messages exist to carry.
+    """
+    names = sorted(
+        str(decode_label(label)) if decode_label else str(label) for label in labels
+    )
+    return ", ".join(names)
+
+
 def split_training_data(
     dataset: list[tuple[str, int]],
     decode_label: Optional[Callable[[int], str]] = None,
 ) -> tuple[list[tuple[str, int]], list[tuple[str, int]]]:
     """Create a deterministic, class-aware train/evaluation split.
 
-    Every label must contribute at least one row to both sets. This replaces the
-    unstratified 25% split that could randomly put every row for a small class in the
-    evaluation set, leaving the classifier unable to learn that command.
+    Every label that *can* appear on both sides does. A label with fewer rows than
+    `heldout_evaluation.MIN_TRAINING_ROWS_PER_LABEL` cannot be stratified, so it trains
+    and goes unmeasured rather than failing the run: that is already what the persona
+    holdout does when its own split would starve a label, and aborting here instead
+    spends the entire LLM and GPU budget only to publish nothing - over a label whose
+    evaluation coverage the developer may never have wanted. Being unmeasured is
+    reported, not fatal.
+
+    Splitting still replaces the unstratified 25% split that could randomly put every
+    row for a small class in the evaluation set, leaving the classifier unable to learn
+    that command.
 
     ``decode_label`` turns the encoded label back into its command name for the
-    error message. Without it the abort names labels the developer cannot map to
-    commands without reverse-engineering the LabelEncoder, which is the state
-    this argument exists to prevent.
+    warning and error messages. Without it those messages name labels the developer
+    cannot map to commands without reverse-engineering the LabelEncoder, which is the
+    state this argument exists to prevent.
     """
     label_counts = Counter(label for _, label in dataset)
-    starved = sorted(
+    unmeasurable = {
         label
         for label, count in label_counts.items()
         if count < heldout_evaluation.MIN_TRAINING_ROWS_PER_LABEL
-    )
-    if starved:
-        named = [decode_label(label) for label in starved] if decode_label else starved
-        raise TrainingDataError(
-            f"Each intent label needs at least "
-            f"{heldout_evaluation.MIN_TRAINING_ROWS_PER_LABEL} rows (one for training "
-            f"and one for evaluation); labels with fewer rows: {named}"
+    }
+    # Partition rather than filter, so the unmeasurable rows still reach the model. They
+    # are appended in dataset order to keep the split reproducible.
+    splittable = [row for row in dataset if row[1] not in unmeasurable]
+    unmeasured_rows = [row for row in dataset if row[1] in unmeasurable]
+    if unmeasurable:
+        logger.warning(
+            f"{len(unmeasurable)} intent label(s) have fewer than "
+            f"{heldout_evaluation.MIN_TRAINING_ROWS_PER_LABEL} rows, so they train "
+            f"without an evaluation row and are unmeasured on the routing axis: "
+            f"{_name_labels(unmeasurable, decode_label)}"
         )
 
+    label_counts = Counter(label for _, label in splittable)
     label_count = len(label_counts)
-    test_size = max((len(dataset) + 3) // 4, label_count)
-    if len(dataset) - test_size < label_count:
+    if not label_count:
+        # Nothing can be measured, so there is no evaluation set to tune a threshold
+        # against and no score to publish. That is a different failure from one thin
+        # label and stays fatal.
+        raise TrainingDataError(
+            f"No intent label has the "
+            f"{heldout_evaluation.MIN_TRAINING_ROWS_PER_LABEL} rows needed to form an "
+            f"evaluation set; every label is below the floor: "
+            f"{_name_labels(unmeasurable, decode_label)}"
+        )
+
+    test_size = max((len(splittable) + 3) // 4, label_count)
+    if len(splittable) - test_size < label_count:
         raise TrainingDataError(
-            f"Cannot split {len(dataset)} rows across {label_count} labels while keeping "
-            "at least one row per label in both training and evaluation sets."
+            f"Cannot split {len(splittable)} rows across {label_count} labels while "
+            "keeping at least one row per label in both training and evaluation sets."
         )
 
     train_data, test_data = train_test_split(
-        dataset,
+        splittable,
         test_size=test_size,
         random_state=42,
-        stratify=[label for _, label in dataset],
+        stratify=[label for _, label in splittable],
     )
-    return list(train_data), list(test_data)
+    return list(train_data) + unmeasured_rows, list(test_data)
 
 
 def save_label_encoder(filepath):

{fastworkflow-3.2.0 → fastworkflow-3.2.1}/fastworkflow/train/training_report.py

@@ -39,10 +39,11 @@ available at the end of a training run and can be retained with its artifacts.
 
 The two floors, and how much to trust them
 ------------------------------------------
-See `DEFAULT_MIN_TRAINING_ROWS` and `DEFAULT_MIN_SEED_UTTERANCES`. One is derived
-from a structural property of the training code; the other is an observation from
-a single workflow. They are labelled differently in the output on purpose, and
-only the structural floor can prevent publication.
+See `DEFAULT_MIN_TRAINING_ROWS` and `DEFAULT_MIN_SEED_UTTERANCES`. The row floor is
+derived from the minimum needed to both train and evaluate a label; falling below it
+means that label trains without evaluation coverage. The seed floor is an observation
+from a single workflow. Both are advisory because neither makes the trained label
+unusable.
 
 `TrainingReport.has_blocking_problems` rejects structurally unusable data before
 publication; fallback and seed-count findings remain advisory.
@@ -148,14 +149,17 @@ class RowStatus(str, Enum):
     #: Generation degraded (R3a fallback). The command trained on its command name
     #: plus hand-written seeds plus whatever arrived before the failure.
     FELL_BACK = "fell_back"
-    #: Fewer rows than the floor. May or may not have fallen back.
+    #: Fewer rows than the train-and-evaluate floor. The rows still train, but the
+    #: label has no evaluation coverage and is therefore reported as unmeasured.
     BELOW_FLOOR = "below_floor"
     #: The command has `Signature.Input`, but its generator returned no rows in at
     #: least one context. This is an explicit trainer skip, not missing provenance.
     NO_UTTERANCES = "no_utterances"
-    #: The command should have generated utterances but has no provenance record.
-    #: Either it was never reached (a training run that died part-way) or provenance
-    #: was lost. Distinct from EXCLUDED, which is by design.
+    #: The command has no provenance record AND no context provenance showing it was
+    #: trained: either it was never reached (a training run that died part-way) or
+    #: provenance was lost. Distinct from EXCLUDED, which is by design, and from a
+    #: command that supplies hand-written utterances without calling the generator -
+    #: that one has nothing to record and is judged on its row count.
     MISSING = "missing"
     #: No `Signature.Input`, so `model_pipeline_training._requires_utterances` returns
     #: False and the command is deliberately not an intent-detection label at all. It
@@ -183,13 +187,12 @@ _STATUS_SEVERITY: dict[RowStatus, int] = {
     RowStatus.OK: 7,
 }
 
-#: Statuses that make a training dataset structurally unusable. EXCLUDED and
-#: THIN_SEEDS are deliberately absent: the first is a design choice, the second rests
-#: on one workflow's curve and is too weak a basis for rejecting a run. FELL_BACK is
-#: reported loudly but does not fail when enough rows remain.
+#: Statuses that make a training dataset structurally unusable. EXCLUDED is a design
+#: choice. THIN_SEEDS and BELOW_FLOOR are advisory: the former rests on one workflow's
+#: curve, while the latter means the label trained but could not also contribute an
+#: evaluation row. FELL_BACK is reported loudly but does not fail when rows remain.
 BLOCKING_STATUSES: frozenset[RowStatus] = frozenset(
     {
-        RowStatus.BELOW_FLOOR,
         RowStatus.NO_UTTERANCES,
         RowStatus.MISSING,
     }
@@ -671,6 +674,9 @@ def _classify_status(
     put a non-command at the top of a list of broken commands. A reserved label that
     genuinely fell back is still surfaced, because a degraded escalation class
     degrades the workflow like anything else.
+
+    Absent generation provenance is only a defect when nothing else shows the command
+    trained; see the comment on the `has_provenance` branch below.
     """
     if kind is CommandKind.RESERVED and not fell_back:
         return RowStatus.NOT_APPLICABLE
@@ -690,7 +696,17 @@ def _classify_status(
     if not has_provenance:
         if trains_as_label is False:
             return RowStatus.EXCLUDED
-        return RowStatus.MISSING
+        if not has_included_context:
+            # No generation record AND no sign the trainer ever reached this command:
+            # the run died part-way or provenance was lost. That is what MISSING means.
+            return RowStatus.MISSING
+        # An `UtteranceProvenance` record describes *generation*, and only
+        # `generate_diverse_utterances` writes one. A command whose `generate_utterances`
+        # returns hand-written rows directly never calls the generator, so it has nothing
+        # to record - exactly the reasoning `NOT_APPLICABLE` already applies to reserved
+        # labels. The trainer's own context provenance proves this command trained, and
+        # carries the row counts, so judge it on the floor like anything else instead of
+        # blocking publication over a record that was never owed.
     if rows_by_context and any(
         context_rows < min_rows for context_rows in rows_by_context.values()
     ):
@@ -1063,11 +1079,11 @@ def format_report(report: TrainingReport) -> str:
             f"  BELOW ROW FLOOR ({len(below_floor)}) — fewer than {report.min_rows} rows."
         )
         lines.append(
-            "  The class-aware split needs at least one row for training and one for "
-            "evaluation."
+            "  These labels trained, but the class-aware split could not also reserve "
+            "an evaluation row."
         )
         lines.append(
-            "  A label below that floor cannot be learned and evaluated in the same run."
+            "  They are unmeasured on the routing axis; this does not block publication."
         )
         lines.extend(_detail_lines(below_floor, width))
 
@@ -1212,7 +1228,7 @@ def format_report(report: TrainingReport) -> str:
 
     lines.append("")
     lines.append(
-        f"  Floors: rows >= {report.min_rows} (structural — one train and one "
+        f"  Floors: rows >= {report.min_rows} (evaluation coverage — one train and one "
         f"evaluation row per label),"
     )
     lines.append(
@@ -1220,9 +1236,7 @@ def format_report(report: TrainingReport) -> str:
         f"one workflow's"
     )
     lines.append("  observation). These are trainer policy, not workflow configuration.")
-    lines.append(
-        "  Structural failures stop publication; advisory findings do not."
-    )
+    lines.append("  Missing or empty training data stops publication; advisory findings do not.")
     lines.append(rule)
     return "\n".join(lines)
 

{fastworkflow-3.2.0 → fastworkflow-3.2.1}/pyproject.toml

@@ -9,7 +9,7 @@ repository = "https://github.com/radiantlogicinc/fastworkflow"
 
 [tool.poetry]
 name = "fastworkflow"
-version = "3.2.0"
+version = "3.2.1"
 description = "A framework for rapidly building large-scale, deterministic, interactive workflows with a fault-tolerant, conversational UX"
 authors = ["Dhar Rawal <drawal@radiantlogic.com>"]
 license = "Apache-2.0"