forecasting-tools 0.2.89__tar.gz → 0.2.90__tar.gz

{forecasting_tools-0.2.89 → forecasting_tools-0.2.90}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: forecasting-tools
-Version: 0.2.89
+Version: 0.2.90
 Summary: AI forecasting and research tools to help humans reason about and forecast the future
 License: MIT
 License-File: LICENSE

{forecasting_tools-0.2.89 → forecasting_tools-0.2.90}/forecasting_tools/ai_models/general_llm.py

@@ -359,16 +359,15 @@ class GeneralLlm(
             return typeguard.check_type(citations, list[str])
 
         # OpenRouter returns Perplexity citations as url_citation annotations
-        # rather than in model_extra["citations"]. The annotations are the
-        # flat source URL list duplicated (once with titles, once without),
-        # NOT one-per-occurrence. All start_index/end_index are 0.
-        # We deduplicate to reconstruct the original indexed list where
-        # urls[i] corresponds to citation [i+1] in the text.
+        # rather than in model_extra["citations"]. URLs may be duplicated
+        # (e.g. once with titles, once without). We deduplicate while
+        # preserving first-seen order so urls[i] corresponds to [i+1].
         message = choices[0].message
         annotations = getattr(message, "annotations", None)
         if not annotations:
             return []
-        all_urls: list[str] = []
+        seen: set[str] = set()
+        unique_urls: list[str] = []
         for annotation in annotations:
             if not isinstance(annotation, dict):
                 continue
@@ -376,23 +375,9 @@ class GeneralLlm(
                 continue
             url_info = annotation.get("url_citation", {})
             url = url_info.get("url", "")
-            if url:
-                all_urls.append(url)
-
-        seen: set[str] = set()
-        unique_urls: list[str] = []
-        for url in all_urls:
-            if url not in seen:
+            if url and url not in seen:
                 seen.add(url)
                 unique_urls.append(url)
-
-        num_unique = len(unique_urls)
-        num_total = len(all_urls)
-        if num_total != num_unique and num_total != num_unique * 2:
-            raise ValueError(
-                f"Expected annotations to contain each URL once or twice, "
-                f"but got {num_total} total URLs and {num_unique} unique URLs"
-            )
         return unique_urls
 
     def _normalize_response(

forecasting_tools-0.2.90/forecasting_tools/calibration_adjustment/.gitignore

@@ -0,0 +1,2 @@
+# Don't leak data used in notebook
+*.csv

forecasting_tools-0.2.90/forecasting_tools/calibration_adjustment/__init__.py

@@ -0,0 +1,18 @@
+from forecasting_tools.calibration_adjustments.calibration_adjuster import (
+    CalibrationAdjuster as CalibrationAdjuster,
+)
+from forecasting_tools.calibration_adjustments.constant_shift_adjuster import (
+    ConstantShiftAdjuster as ConstantShiftAdjuster,
+)
+from forecasting_tools.calibration_adjustments.decision_tree_adjuster import (
+    DecisionTreeAdjuster as DecisionTreeAdjuster,
+)
+from forecasting_tools.calibration_adjustments.k_means_adjuster import (
+    KMeansAdjuster as KMeansAdjuster,
+)
+from forecasting_tools.calibration_adjustments.logistic_recalibration_adjuster import (
+    LogisticRecalibrationAdjuster as LogisticRecalibrationAdjuster,
+)
+from forecasting_tools.calibration_adjustments.step_adjuster import (
+    StepAdjuster as StepAdjuster,
+)

forecasting_tools-0.2.90/forecasting_tools/calibration_adjustment/_training_data.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import json
+import logging
+
+import numpy as np
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+_VALID_BINARY_RESOLUTIONS = {"yes", "no"}
+
+
+def extract_bernoulli_observations(
+    forecasts: pd.DataFrame,
+) -> tuple[np.ndarray, np.ndarray]:
+    """Extract (prediction, outcome) pairs from a forecasts DataFrame.
+
+    Each binary forecast contributes one observation:
+        (probability_yes, 1.0 if resolution=='yes' else 0.0)
+
+    Each multiple-choice forecast contributes one observation per option:
+        (p_option_i, 1.0 if resolution==option_i else 0.0)
+
+    Rows that are not resolved binary or multiple-choice are skipped.
+    Malformed rows (missing probability, unparsable JSON, length
+    mismatches between options and probabilities) are also skipped.
+
+    Returns:
+        predictions: 1-D float array of predicted probabilities.
+        outcomes:    1-D float array of resolved outcomes (0.0 or 1.0).
+
+    Raises:
+        ValueError: if no usable observations are found.
+    """
+    predictions: list[float] = []
+    outcomes: list[float] = []
+
+    for _, row in forecasts.iterrows():
+        q_type = row.get("type")
+        resolution = row.get("resolution")
+
+        if pd.isna(resolution) or resolution is None:
+            continue
+
+        if q_type == "binary":
+            _process_binary_row(row, resolution, predictions, outcomes)
+        elif q_type == "multiple_choice":
+            _process_mc_row(row, resolution, predictions, outcomes)
+
+    if not predictions:
+        raise ValueError(
+            "No usable resolved binary or multiple-choice rows in DataFrame"
+        )
+
+    return np.array(predictions, dtype=np.float64), np.array(outcomes, dtype=np.float64)
+
+
+def _process_binary_row(
+    row: pd.Series,
+    resolution: str,
+    predictions: list[float],
+    outcomes: list[float],
+) -> None:
+    if resolution not in _VALID_BINARY_RESOLUTIONS:
+        return
+
+    prob = row.get("probability_yes")
+    if prob is None or (isinstance(prob, float) and np.isnan(prob)):
+        return
+
+    try:
+        prob = float(prob)
+    except (ValueError, TypeError):
+        return
+
+    if not 0.0 <= prob <= 1.0:
+        return
+
+    predictions.append(prob)
+    outcomes.append(1.0 if resolution == "yes" else 0.0)
+
+
+def _process_mc_row(
+    row: pd.Series,
+    resolution: str,
+    predictions: list[float],
+    outcomes: list[float],
+) -> None:
+    raw_probs = row.get("probability_yes_per_category")
+    raw_options = row.get("options")
+
+    if pd.isna(raw_probs) or pd.isna(raw_options):
+        return
+
+    try:
+        probs = json.loads(raw_probs) if isinstance(raw_probs, str) else raw_probs
+        options = (
+            json.loads(raw_options) if isinstance(raw_options, str) else raw_options
+        )
+    except (json.JSONDecodeError, TypeError):
+        return
+
+    if not isinstance(probs, list) or not isinstance(options, list):
+        return
+    if len(probs) != len(options):
+        logger.warning(
+            "Skipping MC row: options length (%d) != probabilities length (%d)",
+            len(options),
+            len(probs),
+        )
+        return
+
+    for option, prob in zip(options, probs):
+        try:
+            prob = float(prob)
+        except (ValueError, TypeError):
+            continue
+        if not 0.0 <= prob <= 1.0:
+            continue
+
+        predictions.append(prob)
+        outcomes.append(1.0 if resolution == option else 0.0)