tradeblocks-mcp 4.2.0 → 4.3.0

package/src/tools/profile-analysis.ts

@@ -35,6 +35,8 @@ import {
   runtimeFieldCatalog,
   summarizeDatasetAge,
   datasetAgeOutput,
+  maxMissedSessionsSchema,
+  type DatasetProvenance,
 } from "../db/dataset-schemas.ts";
 
 // =============================================================================
@@ -177,7 +179,8 @@ interface TradeWithFields {
   trade: Trade;
   market: Record<string, unknown>;
   builtIn: Record<string, unknown> | undefined;
-  provenance: Record<string, { sourceTimestamp: string; availableAt: string }>;
+  provenance: Record<string, DatasetProvenance>;
+  rejected: Record<string, DatasetProvenance>;
 }
 
 function partitionMarketFilters(entryFilters: EntryFilter[], catalog: { field: string }[]) {
@@ -212,6 +215,7 @@ async function loadTradesAndMarket(
   baseDir: string,
   blockId: string,
   strategyName: string,
+  maxMissedSessions?: number,
 ): Promise<{
   rows: TradeWithFields[];
   unmatchedCount: number;
@@ -244,7 +248,9 @@ async function loadTradesAndMarket(
   const marketRecords = resultToRecords(result);
   const marketMap = recordsByTickerDate(marketRecords);
   const datasets = await listDatasets(conn);
-  const datasetMatches = trades.map((trade) => resolveTradeDatasets(datasets, trade));
+  const datasetMatches = trades.map((trade) =>
+    resolveTradeDatasets(datasets, trade, maxMissedSessions),
+  );
 
   // Preserve built-in coverage separately from the per-field joined values.
   const rows: TradeWithFields[] = [];
@@ -260,6 +266,7 @@ async function loadTradesAndMarket(
       builtIn,
       market: { ...builtIn, ...datasetValues },
       provenance: datasetMatches[index].provenance,
+      rejected: datasetMatches[index].rejected,
     });
     if (!builtIn) unmatchedCount++;
   }
@@ -330,6 +337,7 @@ function findBucket(
 export const analyzeStructureFitSchema = z.object({
   blockId: z.string().describe("Block ID to analyze"),
   strategyName: z.string().describe("Strategy name matching a stored profile"),
+  maxMissedSessions: maxMissedSessionsSchema,
   minTrades: z
     .number()
     .optional()
@@ -359,6 +367,7 @@ export async function handleAnalyzeStructureFit(
     baseDir,
     blockId,
     strategyName,
+    input.maxMissedSessions,
   );
 
   const catalog = await runtimeFieldCatalog(conn);
@@ -382,6 +391,7 @@ export async function handleAnalyzeStructureFit(
     return createToolOutput(`Uncovered dataset fields: ${uncovered.join(", ")}`, {
       error: "uncovered_fields",
       fields: uncovered,
+      ...datasetAgeOutput(datasetAge, uncovered),
       warnings,
     });
 
@@ -611,6 +621,7 @@ export async function handleAnalyzeStructureFit(
 export const validateEntryFiltersSchema = z.object({
   blockId: z.string().describe("Block ID to analyze"),
   strategyName: z.string().describe("Strategy name matching a stored profile"),
+  maxMissedSessions: maxMissedSessionsSchema,
   minTrades: z
     .number()
     .optional()
@@ -675,6 +686,7 @@ export async function handleValidateEntryFilters(
     baseDir,
     blockId,
     strategyName,
+    input.maxMissedSessions,
   );
   const predicates: FilterPredicate[] = filters.map((filter) =>
     buildFilterPredicate(filter, catalog),
@@ -682,9 +694,10 @@ export async function handleValidateEntryFilters(
   const matched = rows.filter(({ market }) =>
     predicates.every((predicate) => predicate.hasData(market)),
   );
+  const ageRows = input.maxMissedSessions === undefined ? matched : rows;
   const datasetAge = summarizeDatasetAge(
-    matched.map(({ trade }) => trade),
-    matched,
+    ageRows.map(({ trade }) => trade),
+    ageRows,
   );
   const uncovered = filters
     .filter(
@@ -697,6 +710,7 @@ export async function handleValidateEntryFilters(
     return createToolOutput(`Uncovered dataset fields: ${uncovered.join(", ")}`, {
       error: "uncovered_fields",
       fields: uncovered,
+      ...datasetAgeOutput(datasetAge, uncovered),
       warnings,
     });
 

package/src/tools/reports/fields.ts

@@ -14,7 +14,7 @@ import { createToolOutput } from "../../utils/output-formatter.ts";
 import { filterByStrategy, filterByDateRange } from "../shared/filters.ts";
 import { enrichAnalysisTrades } from "./helpers.ts";
 import { withSyncedBlock } from "../middleware/sync-middleware.ts";
-import { datasetAgeOutput } from "../../db/dataset-schemas.ts";
+import { datasetAgeOutput, maxMissedSessionsSchema } from "../../db/dataset-schemas.ts";
 
 /**
  * Register field-related report tools
@@ -28,6 +28,7 @@ export function registerFieldTools(server: McpServer, baseDir: string): void {
         "Get detailed statistics for a specific field including min/max/avg/median/stdDev, percentiles, and histogram",
       inputSchema: z.object({
         blockId: z.string().describe("Block folder name"),
+        maxMissedSessions: maxMissedSessionsSchema,
         field: z
           .string()
           .describe(
@@ -46,7 +47,15 @@ export function registerFieldTools(server: McpServer, baseDir: string): void {
     },
     withSyncedBlock(
       baseDir,
-      async ({ blockId, field, strategy, startDate, endDate, histogramBuckets }) => {
+      async ({
+        blockId,
+        field,
+        strategy,
+        startDate,
+        endDate,
+        histogramBuckets,
+        maxMissedSessions,
+      }) => {
         try {
           const block = await loadBlock(baseDir, blockId);
           let trades = block.trades;
@@ -67,7 +76,7 @@ export function registerFieldTools(server: McpServer, baseDir: string): void {
           }
 
           // Enrich trades
-          const resolved = await enrichAnalysisTrades(baseDir, trades);
+          const resolved = await enrichAnalysisTrades(baseDir, trades, maxMissedSessions);
           const enrichedTrades = resolved.trades;
 
           // Extract field values
@@ -80,14 +89,10 @@ export function registerFieldTools(server: McpServer, baseDir: string): void {
           }
 
           if (values.length === 0) {
-            return {
-              content: [
-                {
-                  type: "text",
-                  text: `Field "${field}" has no valid numeric values in the filtered trades.`,
-                },
-              ],
-            };
+            return createToolOutput(
+              `Field "${field}" has no valid numeric values in the filtered trades.`,
+              { blockId, field, ...datasetAgeOutput(resolved.datasetAge, [field]) },
+            );
           }
 
           const analysis = fieldStatisticsFromValues(values, histogramBuckets)!;

package/src/tools/reports/helpers.ts

@@ -52,8 +52,12 @@ export interface EnrichedTrade extends Trade {
   staticDatasetFields?: Record<string, Record<string, number | string>>;
 }
 
-export async function enrichAnalysisTrades(baseDir: string, trades: Trade[]) {
-  const resolved = await resolveAnalysisFields(baseDir, trades);
+export async function enrichAnalysisTrades(
+  baseDir: string,
+  trades: Trade[],
+  maxMissedSessions?: number,
+) {
+  const resolved = await resolveAnalysisFields(baseDir, trades, maxMissedSessions);
   const enriched = enrichTrades(trades);
   for (let i = 0; i < enriched.length; i++) {
     const datasets: Record<string, Record<string, number | string>> = {};

package/src/tools/reports/predictive.ts

@@ -16,7 +16,7 @@ import { getConnection } from "../../db/connection.ts";
 import { getProfile } from "../../db/profile-schemas.ts";
 import { requireCoveredField } from "../../utils/trade-analysis-fields.ts";
 import { sweepThresholds } from "../../utils/threshold-sweep.ts";
-import { datasetAgeOutput } from "../../db/dataset-schemas.ts";
+import { datasetAgeOutput, maxMissedSessionsSchema } from "../../db/dataset-schemas.ts";
 
 /**
  * Register predictive-related report tools
@@ -30,6 +30,7 @@ export function registerPredictiveTools(server: McpServer, baseDir: string): voi
         "Identify which trade entry conditions predict profitability by calculating Pearson correlations between all numeric fields and a target field (usually P/L). Returns fields ranked by predictive strength.",
       inputSchema: z.object({
         blockId: z.string().describe("Block folder name"),
+        maxMissedSessions: maxMissedSessionsSchema,
         strategy: z.string().optional().describe("Pre-filter by strategy name (case-insensitive)"),
         strategyName: z
           .string()
@@ -67,6 +68,7 @@ export function registerPredictiveTools(server: McpServer, baseDir: string): voi
         targetField,
         minSamples,
         includeCustomFields,
+        maxMissedSessions,
       }) => {
         try {
           const block = await loadBlock(baseDir, blockId);
@@ -96,7 +98,7 @@ export function registerPredictiveTools(server: McpServer, baseDir: string): voi
           }
 
           // Enrich trades
-          const resolved = await enrichAnalysisTrades(baseDir, trades);
+          const resolved = await enrichAnalysisTrades(baseDir, trades, maxMissedSessions);
           const enrichedTrades = resolved.trades;
 
           const {
@@ -187,6 +189,7 @@ export function registerPredictiveTools(server: McpServer, baseDir: string): voi
         "Sweep filter thresholds for a field and show performance at each threshold. Use after find_predictive_fields to determine optimal filter values. Returns outcome curves and identifies sweet spots where filtering improves performance.",
       inputSchema: z.object({
         blockId: z.string().describe("Block folder name"),
+        maxMissedSessions: maxMissedSessionsSchema,
         field: z
           .string()
           .describe("Field to sweep thresholds on (e.g., 'openingVix', 'durationHours')"),
@@ -224,6 +227,7 @@ export function registerPredictiveTools(server: McpServer, baseDir: string): voi
         strategy,
         startDate,
         endDate,
+        maxMissedSessions,
       }) => {
         try {
           const block = await loadBlock(baseDir, blockId);
@@ -245,13 +249,13 @@ export function registerPredictiveTools(server: McpServer, baseDir: string): voi
           }
 
           // Enrich trades
-          const resolved = await enrichAnalysisTrades(baseDir, trades);
+          const resolved = await enrichAnalysisTrades(baseDir, trades, maxMissedSessions);
           const enrichedTrades = resolved.trades;
           if (
             resolved.catalog.some((entry) => entry.field === field) ||
             (field.includes(".") && !field.startsWith("custom.") && !field.startsWith("daily."))
           )
-            requireCoveredField(field, resolved.catalog, resolved.values);
+            requireCoveredField(field, resolved.catalog, resolved.values, resolved.datasetAge);
 
           // Extract field values
           const fieldValues: number[] = [];
@@ -288,6 +292,13 @@ export function registerPredictiveTools(server: McpServer, baseDir: string): voi
             winRate: Math.round(baselineWinRate * 10000) / 10000,
             avgPl: Math.round(baselineAvgPl * 100) / 100,
             totalPl: Math.round(baselineTotalPl * 100) / 100,
+            uncoveredCount: trades.length - tradesWithField.length,
+            uncoveredTotalPl:
+              Math.round(
+                enrichedTrades
+                  .filter((trade) => getTradeFieldValue(trade, field) === null)
+                  .reduce((sum, trade) => sum + trade.pl, 0) * 100,
+              ) / 100,
           };
 
           const sweeps = sweepThresholds(

package/src/tools/shared/field-analysis.ts

@@ -4,35 +4,63 @@ import { sweepThresholds } from "../../utils/threshold-sweep.ts";
 import { computeSliceStats } from "../../utils/analysis-stats.ts";
 import { createToolOutput } from "../../utils/output-formatter.ts";
 import { datasetAgeOutput } from "../../db/dataset-schemas.ts";
+import {
+  suggestionEvidence,
+  rankSuggestions,
+  objectiveOutput,
+  type FilterObjective,
+  type SuggestionEvidence,
+} from "./filter-objective.ts";
 
 export async function suggestFieldFilters(
   baseDir: string,
   blockId: string,
   trades: Trade[],
   fields: string[],
-  minImprovementPct: number,
+  minImprovementPct: number | undefined,
+  objective: FilterObjective,
+  maxMissedSessions?: number,
 ) {
-  const { catalog, values, datasetAge } = await resolveAnalysisFields(baseDir, trades);
-  const suggestions: {
+  const { catalog, values, datasetAge } = await resolveAnalysisFields(
+    baseDir,
+    trades,
+    maxMissedSessions,
+  );
+  const suggestions: (SuggestionEvidence & {
     filter: string;
     condition: { field: string; operator: string; value: number | string; lagged: boolean };
-    tradesRemoved: number;
     winnersRemoved: number;
     losersRemoved: number;
     newWinRate: number;
     newTotalPl: number;
     winRateDelta: number;
     plDelta: number;
-    confidence: string;
-  }[] = [];
+  })[] = [];
   const coverage: Record<string, { covered: number; uncovered: number }> = {};
+  const fieldBaselines: Record<
+    string,
+    {
+      count: number;
+      totalPl: number;
+      uncoveredCount: number;
+      uncoveredTotalPl: number;
+    }
+  > = {};
   for (const field of fields) {
-    requireCoveredField(field, catalog, values);
+    requireCoveredField(field, catalog, values, datasetAge);
     const pool = trades
       .map((trade, i) => ({ trade, value: values[i][field] }))
       .filter((item) => item.value !== undefined);
     coverage[field] = { covered: pool.length, uncovered: trades.length - pool.length };
     const baseline = computeSliceStats(pool.map(({ trade }) => trade.pl));
+    fieldBaselines[field] = {
+      count: pool.length,
+      totalPl: baseline.totalPl,
+      uncoveredCount: trades.length - pool.length,
+      uncoveredTotalPl: computeSliceStats(
+        trades.filter((_, i) => values[i][field] === undefined).map((trade) => trade.pl),
+      ).totalPl,
+    };
     const entry = catalog.find((entry) => entry.field === field)!;
     const candidates =
       entry.type === "numeric"
@@ -53,7 +81,15 @@ export async function suggestFieldFilters(
       if (!kept.length || !excluded.size) continue;
       const stats = computeSliceStats(kept.map(({ trade }) => trade.pl));
       const improvement = stats.winRate - baseline.winRate;
-      if (improvement < minImprovementPct) continue;
+      const evidence = suggestionEvidence(
+        objective,
+        improvement,
+        stats.totalPl - baseline.totalPl,
+        kept.length,
+        excluded.size,
+        minImprovementPct ?? 3,
+      );
+      if (!evidence) continue;
       const winnersRemoved = candidate.excluded.filter(({ trade }) => trade.pl > 0).length;
       suggestions.push({
         filter: `Skip when ${field} ${candidate.operator} ${candidate.value}`,
@@ -63,27 +99,29 @@ export async function suggestFieldFilters(
           value: candidate.value,
           lagged: entry.timing === "close",
         },
-        tradesRemoved: excluded.size,
+        ...evidence,
         winnersRemoved,
         losersRemoved: excluded.size - winnersRemoved,
         newWinRate: stats.winRate,
         newTotalPl: stats.totalPl,
         winRateDelta: improvement,
         plDelta: stats.totalPl - baseline.totalPl,
-        confidence: kept.length >= 100 ? "high" : kept.length >= 30 ? "medium" : "low",
       });
     }
   }
-  suggestions.sort((a, b) => b.winRateDelta - a.winRateDelta || b.plDelta - a.plDelta);
+  suggestions.sort((a, b) => rankSuggestions(objective, a, b));
   return createToolOutput(`Field filter analysis: ${blockId} | ${suggestions.length} suggestions`, {
     blockId,
     fields,
+    ...objectiveOutput(objective, minImprovementPct),
     currentStats: computeSliceStats(trades.map((trade) => trade.pl)),
     suggestedFilters: suggestions,
     compositeSuggestions: [],
     coverage,
+    fieldBaselines,
+    comparisonPopulation: "fieldBaselines",
     ...datasetAgeOutput(datasetAge, fields),
-    minImprovementThreshold: minImprovementPct,
+    minImprovementThreshold: objective === "winRate" ? (minImprovementPct ?? 3) : null,
   });
 }
 
@@ -93,9 +131,14 @@ export async function analyzeFieldRegimes(
   trades: Trade[],
   field: string,
   bins?: number[],
+  maxMissedSessions?: number,
 ) {
-  const { catalog, values, datasetAge } = await resolveAnalysisFields(baseDir, trades);
-  requireCoveredField(field, catalog, values);
+  const { catalog, values, datasetAge } = await resolveAnalysisFields(
+    baseDir,
+    trades,
+    maxMissedSessions,
+  );
+  requireCoveredField(field, catalog, values, datasetAge);
   const entry = catalog.find((entry) => entry.field === field)!;
   if (bins && entry.type !== "numeric") throw new Error(`Bins require a numeric field: ${field}`);
   if (bins?.some((value, i) => !Number.isFinite(value) || (i > 0 && value <= bins[i - 1])))

package/src/tools/shared/filter-objective.ts

@@ -0,0 +1,73 @@
+export type FilterObjective = "totalPl" | "winRate";
+
+export interface SuggestionEvidence {
+  objective: FilterObjective;
+  tradesKept: number;
+  keptFraction: number;
+  tradesRemoved: number;
+  confidence: "high" | "medium" | "low";
+  warnings: string[];
+}
+
+/** Both suggestion paths use the same admission and sample-size support rule. */
+export function suggestionEvidence(
+  objective: FilterObjective,
+  winRateDelta: number,
+  plDelta: number,
+  kept: number,
+  removed: number,
+  minImprovementPct: number,
+): SuggestionEvidence | null {
+  if (
+    !kept ||
+    !removed ||
+    (objective === "totalPl" ? plDelta <= 0 : winRateDelta < minImprovementPct)
+  )
+    return null;
+  const keptFraction = kept / (kept + removed);
+  const lowRetention = keptFraction < 0.2;
+  return {
+    objective,
+    tradesKept: kept,
+    keptFraction,
+    tradesRemoved: removed,
+    confidence: lowRetention
+      ? "low"
+      : removed >= 10 && kept >= 20
+        ? "high"
+        : removed >= 5 && kept >= 10
+          ? "medium"
+          : "low",
+    warnings: lowRetention
+      ? ["Low retention: fewer than 20% of covered trades kept; sample-size support capped at low."]
+      : [],
+  };
+}
+
+export function rankSuggestions(
+  objective: FilterObjective,
+  a: { winRateDelta: number; plDelta: number },
+  b: { winRateDelta: number; plDelta: number },
+): number {
+  return objective === "totalPl"
+    ? b.plDelta - a.plDelta || b.winRateDelta - a.winRateDelta
+    : b.winRateDelta - a.winRateDelta || b.plDelta - a.plDelta;
+}
+
+export function objectiveOutput(objective: FilterObjective, minImprovementPct?: number) {
+  return {
+    objective,
+    rankingRule:
+      objective === "totalPl"
+        ? "Positive P&L improvement, ranked by P&L improvement then win-rate gain"
+        : "Win-rate gain >= minImprovementPct, ranked by win-rate gain then P&L improvement",
+    supportNote:
+      "confidence is sample-size support, not statistical confidence; keptFraction < 0.2 caps it at low",
+    ...(objective === "totalPl" && minImprovementPct !== undefined
+      ? {
+          objectiveNote:
+            "minImprovementPct is ignored under totalPl; only positive P&L improvement is admitted.",
+        }
+      : {}),
+  };
+}

package/src/utils/trade-analysis-fields.ts

@@ -5,15 +5,22 @@ import {
   resolveTradeDatasets,
   runtimeFieldCatalog,
   summarizeDatasetAge,
+  type DatasetFieldAge,
 } from "../db/dataset-schemas.ts";
 import { buildLookaheadFreeQuery, CLOSE_KNOWN_FIELDS } from "./field-timing.ts";
 import { marketTickerDateKey, resolveTradeTicker } from "./ticker.ts";
 
-export async function resolveAnalysisFields(baseDir: string, trades: Trade[]) {
+export async function resolveAnalysisFields(
+  baseDir: string,
+  trades: Trade[],
+  maxMissedSessions?: number,
+) {
   const conn = await getConnection(baseDir);
   const datasets = await listDatasets(conn);
   const catalog = await runtimeFieldCatalog(conn);
-  const datasetMatches = trades.map((trade) => resolveTradeDatasets(datasets, trade));
+  const datasetMatches = trades.map((trade) =>
+    resolveTradeDatasets(datasets, trade, maxMissedSessions),
+  );
   const keys = trades.map((trade) => {
     const date = new Date(trade.dateOpened);
     const midnightUtc = date.getUTCHours() === 0 && date.getUTCMinutes() === 0;
@@ -64,9 +71,17 @@ export function requireCoveredField(
   field: string,
   catalog: { field: string }[],
   values: Record<string, number | string>[],
+  ages?: Record<string, DatasetFieldAge>,
 ) {
   if (!catalog.some((entry) => entry.field === field))
     throw new Error(`Unknown analysis field: ${field}`);
-  if (!values.some((row) => row[field] !== undefined))
-    throw new Error(`Uncovered analysis field: ${field}; no value available at trade entry`);
+  if (!values.some((row) => row[field] !== undefined)) {
+    const age = ages?.[field];
+    const reason = age?.rejectedAttachments
+      ? `; ${age.rejectedAttachments} refused by maxMissedSessions; ${age.unsupportedSessionAge} session ages unsupported`
+      : "";
+    throw new Error(
+      `Uncovered analysis field: ${field}; no value available at trade entry${reason}`,
+    );
+  }
 }