productfoo 0.0.0-stage → 1.0.1

package/dist-public/src/discoveryLearningRouting.js

@@ -0,0 +1,405 @@
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import path from "node:path";
+import { parse } from "./vendorYaml.js";
+export const LEARNING_ROUTING_POLICY = {
+    semantic_minimum: 0.45,
+    semantic_margin: 0.12,
+    catch_all_shared_max: 5,
+    synthesis_source_overlap_max: 0.82,
+};
+const STOPWORDS = new Set([
+    "about", "after", "again", "before", "being", "between", "could", "every",
+    "should", "their", "there", "these", "those", "through", "with", "without",
+    "would", "from", "into", "only", "that", "this", "when", "where", "which",
+]);
+// These are common product-label words, not distinctive concepts. They can
+// still contribute to paragraph evidence, but never establish title custody.
+const GENERIC_TITLE_TOKENS = new Set([
+    "agent", "business", "company", "control", "enterprise", "management",
+    "plane", "platform", "product", "system", "workflow",
+]);
+const CLUSTER_CONCEPTS = new Map([
+    ["access", "governance"], ["approval", "governance"], ["credential", "governance"],
+    ["gated", "governance"], ["govern", "governance"], ["guardrail", "governance"],
+    ["inspectable", "governance"], ["permission", "governance"], ["regulated", "governance"],
+    ["security", "governance"], ["backlog", "portfolio"], ["command", "portfolio"],
+    ["coordination", "portfolio"], ["portfolio", "portfolio"], ["prioritization", "portfolio"],
+    ["steering", "portfolio"], ["visibility", "portfolio"], ["adoption", "adoption"],
+    ["blocked", "adoption"], ["change", "adoption"], ["distribution", "adoption"],
+    ["onboarding", "adoption"], ["restriction", "adoption"], ["automation", "process"],
+    ["mining", "process"], ["process", "process"], ["recurring", "process"],
+    ["variation", "process"], ["workflow", "process"],
+]);
+function normalizePath(root, file) {
+    return path.relative(root, file).split(path.sep).join("/");
+}
+function titleCase(value) {
+    return value
+        .replace(/\.md$/i, "")
+        .replace(/^\d{4}-\d{2}-\d{2}-/, "")
+        .replace(/-v\d+$/i, "")
+        .split(/[-_]+/)
+        .filter(Boolean)
+        .map((word) => `${word.charAt(0).toUpperCase()}${word.slice(1)}`)
+        .join(" ");
+}
+function frontmatter(text) {
+    if (!text.startsWith("---\n"))
+        return { meta: {}, body: text };
+    const closing = text.indexOf("\n---", 4);
+    if (closing === -1)
+        return { meta: {}, body: text };
+    const after = text.indexOf("\n", closing + 1);
+    let meta = {};
+    try {
+        const parsed = parse(text.slice(4, closing));
+        if (parsed && typeof parsed === "object" && !Array.isArray(parsed))
+            meta = parsed;
+    }
+    catch {
+        meta = {};
+    }
+    return { meta, body: after === -1 ? "" : text.slice(after + 1) };
+}
+function contentBlocks(body) {
+    const blocks = [];
+    for (const paragraph of body.split(/\n{2,}/)) {
+        const trimmed = paragraph.trim();
+        if (!trimmed)
+            continue;
+        if (trimmed.length > 1500) {
+            for (const line of trimmed.split("\n"))
+                if (line.trim())
+                    blocks.push(line.trim());
+        }
+        else {
+            blocks.push(trimmed);
+        }
+    }
+    return blocks;
+}
+function markdownFiles(dir) {
+    if (!existsSync(dir))
+        return [];
+    const files = [];
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+        const full = path.join(dir, entry.name);
+        if (entry.isDirectory())
+            files.push(...markdownFiles(full));
+        else if (entry.isFile() && entry.name.endsWith(".md"))
+            files.push(full);
+    }
+    return files;
+}
+function destinationFromFile(root, file, type, ref) {
+    const text = readFileSync(file, "utf8");
+    const parsed = frontmatter(text);
+    const metadataTitle = typeof parsed.meta.title === "string"
+        ? parsed.meta.title
+        : typeof parsed.meta.idea === "string"
+            ? parsed.meta.idea
+            : null;
+    const heading = /^#\s+(.+)$/m.exec(parsed.body)?.[1]?.trim() ?? null;
+    return {
+        type,
+        ref,
+        path: normalizePath(root, file),
+        title: metadataTitle?.trim() || heading || titleCase(path.basename(type === "backlog" ? path.dirname(file) : file)),
+        text: parsed.body,
+        blocks: contentBlocks(parsed.body),
+    };
+}
+function isArchivedPath(root, file) {
+    return normalizePath(root, file).split("/").some((segment) => segment.toLowerCase() === "archive");
+}
+function isActivePacket(file) {
+    const { meta } = frontmatter(readFileSync(file, "utf8"));
+    const lifecycle = typeof meta.lifecycle === "string" ? meta.lifecycle.trim().toLowerCase() : "";
+    return !new Set(["complete", "completed", "done", "killed", "parked", "retired", "stopped"]).has(lifecycle);
+}
+export function buildLearningDestinationCatalog(root) {
+    const destinations = [];
+    for (const file of markdownFiles(path.join(root, "ideas"))) {
+        if (isArchivedPath(root, file))
+            continue;
+        destinations.push(destinationFromFile(root, file, "idea", normalizePath(root, file)));
+    }
+    for (const file of markdownFiles(path.join(root, "research"))) {
+        if (isArchivedPath(root, file))
+            continue;
+        destinations.push(destinationFromFile(root, file, "research", normalizePath(root, file)));
+    }
+    const specsDir = path.join(root, "specs");
+    if (existsSync(specsDir)) {
+        for (const entry of readdirSync(specsDir, { withFileTypes: true })) {
+            if (!entry.isDirectory())
+                continue;
+            const file = path.join(specsDir, entry.name, "spec.md");
+            if (existsSync(file) && isActivePacket(file))
+                destinations.push(destinationFromFile(root, file, "backlog", entry.name));
+        }
+    }
+    return destinations.sort((left, right) => left.path.localeCompare(right.path));
+}
+function normalizedToken(token) {
+    if (/^evaluat|^evals?$/.test(token))
+        return "eval";
+    if (/^orchestrat/.test(token))
+        return "orchestr";
+    if (/^rout(?:e|ed|es|ing)?$/.test(token))
+        return "route";
+    if (token.length > 5 && token.endsWith("ies"))
+        return `${token.slice(0, -3)}y`;
+    if (token.length > 6 && token.endsWith("ing"))
+        return token.slice(0, -3);
+    if (token.length > 5 && token.endsWith("ed"))
+        return token.slice(0, -2);
+    if (token.length > 5 && token.endsWith("es"))
+        return token.slice(0, -2);
+    if (token.length > 4 && token.endsWith("s") && !token.endsWith("ss"))
+        return token.slice(0, -1);
+    return token;
+}
+function tokens(text) {
+    return new Set((text.toLowerCase().match(/[a-z0-9]+/g) ?? [])
+        .filter((token) => token.length >= 4 && !STOPWORDS.has(token))
+        .map(normalizedToken));
+}
+function normalizedText(text) {
+    return (text.toLowerCase().match(/[a-z0-9]+/g) ?? []).map(normalizedToken).join(" ");
+}
+function clusterTokens(finding) {
+    return tokens(`${finding.headline} ${finding.paraphrase}`);
+}
+function clusterConcepts(finding) {
+    const concepts = new Set();
+    for (const token of clusterTokens(finding)) {
+        const concept = CLUSTER_CONCEPTS.get(token);
+        if (concept)
+            concepts.add(concept);
+    }
+    return concepts;
+}
+function relatedClusterFindings(left, right) {
+    const leftConcepts = clusterConcepts(left);
+    const rightConcepts = clusterConcepts(right);
+    for (const concept of leftConcepts)
+        if (rightConcepts.has(concept))
+            return true;
+    const leftTokens = clusterTokens(left);
+    const rightTokens = clusterTokens(right);
+    let overlap = 0;
+    for (const token of leftTokens)
+        if (rightTokens.has(token))
+            overlap += 1;
+    return overlap >= 3;
+}
+function clusterKey(findings) {
+    const sharedConcepts = [...clusterConcepts(findings[0] ?? { finding_id: "", inbox_scope_id: "", headline: "", paraphrase: "" })]
+        .filter((concept) => findings.every((finding) => clusterConcepts(finding).has(concept)))
+        .sort();
+    return `cluster:${sharedConcepts[0] ?? "emerging"}:${findings[0]?.finding_id ?? "unknown"}`;
+}
+export function clusterUnmatchedLearnings(input) {
+    const findings = input
+        .filter((finding) => /^F-\d{3}$/.test(finding.finding_id) && finding.inbox_scope_id.trim() && (finding.headline.trim() || finding.paraphrase.trim()))
+        .map((finding) => ({ ...finding }))
+        .sort((left, right) => left.finding_id.localeCompare(right.finding_id));
+    const visited = new Set();
+    const clusters = [];
+    for (const seed of findings) {
+        if (visited.has(seed.finding_id))
+            continue;
+        const members = [];
+        const queue = [seed];
+        visited.add(seed.finding_id);
+        while (queue.length > 0) {
+            const current = queue.shift();
+            members.push(current);
+            for (const candidate of findings) {
+                if (visited.has(candidate.finding_id) || !relatedClusterFindings(current, candidate))
+                    continue;
+                visited.add(candidate.finding_id);
+                queue.push(candidate);
+            }
+        }
+        members.sort((left, right) => left.finding_id.localeCompare(right.finding_id));
+        const sourceScopeIds = [...new Set(members.map((finding) => finding.inbox_scope_id))].sort();
+        clusters.push({
+            cluster_key: clusterKey(members),
+            status: sourceScopeIds.length >= 2 ? "eligible_for_synthesis" : "emerging",
+            source_finding_ids: members.map((finding) => finding.finding_id),
+            source_scope_ids: sourceScopeIds,
+            findings: members,
+        });
+    }
+    return clusters.sort((left, right) => left.cluster_key.localeCompare(right.cluster_key));
+}
+function tokenCoverage(left, right) {
+    const leftTokens = tokens(left);
+    const rightTokens = tokens(right);
+    if (leftTokens.size === 0 || rightTokens.size === 0)
+        return 0;
+    let overlap = 0;
+    for (const token of leftTokens)
+        if (rightTokens.has(token))
+            overlap += 1;
+    return overlap / Math.min(leftTokens.size, rightTokens.size);
+}
+export function validateHypothesisSynthesis(input) {
+    if (input.cluster.status !== "eligible_for_synthesis" || input.cluster.source_scope_ids.length < 2) {
+        return { ok: false, reason: "independent_calls_required" };
+    }
+    const actualSources = [...input.cluster.source_finding_ids].sort();
+    const requestedSources = [...new Set(input.synthesis.source_finding_ids)].sort();
+    if (requestedSources.length !== input.synthesis.source_finding_ids.length || requestedSources.join("\n") !== actualSources.join("\n")) {
+        return { ok: false, reason: "source_set_mismatch" };
+    }
+    for (const key of ["claim", "dissent", "falsifier", "next_evidence"]) {
+        const value = input.synthesis[key]?.trim();
+        if (!value || value.length > 1500)
+            return { ok: false, reason: `invalid_${key}` };
+    }
+    if (!new Set(["low", "medium"]).has(input.synthesis.starting_confidence)) {
+        return { ok: false, reason: "invalid_starting_confidence" };
+    }
+    const claim = input.synthesis.claim.trim();
+    const normalizedClaim = normalizedText(claim);
+    for (const finding of input.cluster.findings) {
+        for (const source of [finding.headline, finding.paraphrase, finding.proposal_text ?? ""]) {
+            const normalizedSource = normalizedText(source);
+            if (!normalizedSource)
+                continue;
+            const contained = normalizedClaim === normalizedSource
+                || (normalizedClaim.length >= 40 && normalizedSource.includes(normalizedClaim))
+                || (normalizedSource.length >= 40 && normalizedClaim.includes(normalizedSource));
+            if (contained || tokenCoverage(claim, source) >= LEARNING_ROUTING_POLICY.synthesis_source_overlap_max) {
+                return { ok: false, reason: "source_like_claim" };
+            }
+        }
+    }
+    return {
+        ok: true,
+        value: {
+            source_finding_ids: actualSources,
+            source_scope_ids: [...input.cluster.source_scope_ids],
+            claim,
+            dissent: input.synthesis.dissent.trim(),
+            falsifier: input.synthesis.falsifier.trim(),
+            next_evidence: input.synthesis.next_evidence.trim(),
+            starting_confidence: input.synthesis.starting_confidence,
+        },
+    };
+}
+function blockScore(findingTokens, block) {
+    if (findingTokens.size === 0)
+        return 0;
+    const blockTokens = tokens(block);
+    let overlap = 0;
+    for (const token of findingTokens)
+        if (blockTokens.has(token))
+            overlap += 1;
+    if (overlap < 3)
+        return 0;
+    const findingCoverage = overlap / Math.min(findingTokens.size, 12);
+    const blockPrecision = overlap / Math.min(blockTokens.size, 12);
+    return Math.min(1, (findingCoverage * 0.6) + (blockPrecision * 0.4));
+}
+function titleScore(findingTokens, destination) {
+    const coreTitle = destination.title.split(/[:—]/, 1)[0] ?? destination.title;
+    const titleTokens = new Set([...tokens(coreTitle)].filter((token) => !GENERIC_TITLE_TOKENS.has(token)));
+    let titleOverlap = 0;
+    for (const token of titleTokens)
+        if (findingTokens.has(token))
+            titleOverlap += 1;
+    if (titleOverlap < 2 || titleTokens.size === 0)
+        return 0;
+    // A title echo is a useful routing label only when the document body also
+    // supports the learning. This keeps a two-word coincidence from taking custody.
+    const bodySupported = destination.blocks.some((block) => {
+        const blockTokens = tokens(block);
+        let overlap = 0;
+        for (const token of findingTokens)
+            if (blockTokens.has(token))
+                overlap += 1;
+        return overlap >= 3;
+    });
+    if (!bodySupported)
+        return 0;
+    const titleCoverage = titleOverlap / titleTokens.size;
+    const pairCoverage = Math.min(1, titleOverlap / 2);
+    return Math.min(1, (titleCoverage * 0.75) + (pairCoverage * 0.25));
+}
+function semanticCandidates(findingText, destinations) {
+    const findingTokens = tokens(findingText);
+    return destinations
+        .map((destination) => ({
+        destination,
+        score: Math.max(titleScore(findingTokens, destination), destination.blocks.reduce((best, block) => Math.max(best, blockScore(findingTokens, block)), 0)),
+    }))
+        .filter((candidate) => candidate.score > 0)
+        .sort((left, right) => right.score - left.score || left.destination.path.localeCompare(right.destination.path));
+}
+function explicitRefs(finding) {
+    return Array.isArray(finding.explicit_refs)
+        ? finding.explicit_refs.filter((ref) => typeof ref === "string" && ref.trim().length > 0)
+        : [];
+}
+function includesExactRef(destination, ref) {
+    if (destination.ref === ref || destination.path === ref)
+        return true;
+    const escaped = ref.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    return new RegExp(`(^|[^A-Za-z0-9])${escaped}([^A-Za-z0-9]|$)`).test(destination.text);
+}
+function textField(record, key) {
+    const value = record?.[key];
+    return typeof value === "string" ? value.trim() : "";
+}
+export function routeLearning(input) {
+    for (const ref of explicitRefs(input.finding)) {
+        const exact = input.destinations.find((destination) => includesExactRef(destination, ref));
+        if (exact) {
+            return {
+                kind: "home",
+                destination: exact,
+                basis: "exact_ref",
+                placement_confidence: "high",
+                why: `The signal explicitly references ${ref} in ${exact.title}.`,
+                score: 1,
+                margin: 1,
+            };
+        }
+    }
+    const findingText = [textField(input.finding, "headline"), textField(input.finding, "paraphrase")]
+        .filter(Boolean)
+        .join(" ");
+    const candidates = semanticCandidates(findingText, input.destinations);
+    const eligible = candidates.filter((candidate) => (input.sharedTargetCounts?.get(candidate.destination.path) ?? 0) <= LEARNING_ROUTING_POLICY.catch_all_shared_max);
+    const best = eligible[0];
+    const runnerUp = eligible[1];
+    const margin = best ? best.score - (runnerUp?.score ?? 0) : 0;
+    if (best && best.score >= LEARNING_ROUTING_POLICY.semantic_minimum && margin >= LEARNING_ROUTING_POLICY.semantic_margin) {
+        return {
+            kind: "home",
+            destination: best.destination,
+            basis: "semantic",
+            placement_confidence: best.score >= 0.7 && margin >= 0.2 ? "high" : "medium",
+            why: `${best.destination.title} is the closest supported next-action match and clears the routing margin.`,
+            score: best.score,
+            margin,
+        };
+    }
+    if (best && best.score >= LEARNING_ROUTING_POLICY.semantic_minimum && runnerUp) {
+        return {
+            kind: "unverified",
+            reason: "conflict",
+            candidates: eligible.slice(0, 2),
+            why: "Two destinations are similarly supported, so the primary home needs human judgment.",
+        };
+    }
+    return {
+        kind: "unmatched",
+        candidates: (eligible.length ? eligible : candidates).slice(0, 2),
+        why: "No existing Idea, Research, or Backlog destination cleared admission; keep this signal emerging until an independent call corroborates it.",
+    };
+}

package/dist-public/src/discoveryMatching.d.ts

@@ -0,0 +1,26 @@
+import type { DiscoverySession } from "./discoverySources.js";
+export type DiscoveryMatchTarget = {
+    provider: string;
+    expected_local_date: string | null;
+    timezone: string;
+    required_participants: string[];
+    required_title_tokens: string[];
+};
+export type DiscoveryCandidateRef = {
+    candidate_index: number;
+    source_fingerprint: string;
+};
+export type DiscoveryMatchResult = {
+    status: "none" | "unique" | "ambiguous";
+    matches: DiscoverySession[];
+    candidates: DiscoveryCandidateRef[];
+};
+export declare function sourceFingerprint(session: DiscoverySession): string;
+export declare function matchDiscoverySessions(target: DiscoveryMatchTarget, sessions: DiscoverySession[]): DiscoveryMatchResult;
+export type DiscoveryDuplicateProposal = {
+    left_fingerprint: string;
+    right_fingerprint: string;
+    confidence: "medium" | "high";
+    action: "review-canonical-source";
+};
+export declare function findProbableDiscoveryDuplicates(sessions: DiscoverySession[]): DiscoveryDuplicateProposal[];

package/dist-public/src/discoveryMatching.js

@@ -0,0 +1,73 @@
+import { createHash } from "node:crypto";
+function normalize(value) {
+    return value.trim().toLowerCase().replace(/\s+/g, " ");
+}
+function localDate(iso, timeZone) {
+    const date = new Date(iso);
+    if (!Number.isFinite(date.getTime()))
+        return null;
+    try {
+        const parts = new Intl.DateTimeFormat("en-US", { timeZone, year: "numeric", month: "2-digit", day: "2-digit" }).formatToParts(date);
+        const map = Object.fromEntries(parts.map((part) => [part.type, part.value]));
+        return `${map.year}-${map.month}-${map.day}`;
+    }
+    catch {
+        return null;
+    }
+}
+function sessionMatches(target, session) {
+    if (session.completion_state !== "completed")
+        return false;
+    if (normalize(session.provider) !== normalize(target.provider))
+        return false;
+    if (target.expected_local_date && localDate(session.completed_at, target.timezone) !== target.expected_local_date)
+        return false;
+    const title = normalize(session.title);
+    if (!target.required_title_tokens.every((token) => title.includes(normalize(token))))
+        return false;
+    const participants = session.participants.map(normalize);
+    if (!target.required_participants.every((required) => participants.includes(normalize(required))))
+        return false;
+    return true;
+}
+export function sourceFingerprint(session) {
+    const identity = [
+        normalize(session.provider),
+        normalize(session.connection_alias),
+        session.runtime_handle,
+        new Date(session.completed_at).toISOString(),
+        normalize(session.title),
+        session.stable_url ?? "",
+    ].join("\n");
+    return `sha256:${createHash("sha256").update(identity).digest("hex")}`;
+}
+export function matchDiscoverySessions(target, sessions) {
+    const matches = sessions.filter((session) => sessionMatches(target, session));
+    const candidates = matches.map((session, index) => ({ candidate_index: index + 1, source_fingerprint: sourceFingerprint(session) }));
+    return { status: matches.length === 0 ? "none" : matches.length === 1 ? "unique" : "ambiguous", matches, candidates };
+}
+function participantOverlap(left, right) {
+    const rightNames = new Set(right.participants.map(normalize));
+    return left.participants.map(normalize).some((name) => rightNames.has(name));
+}
+export function findProbableDiscoveryDuplicates(sessions) {
+    const proposals = [];
+    for (let left = 0; left < sessions.length; left += 1) {
+        for (let right = left + 1; right < sessions.length; right += 1) {
+            const a = sessions[left];
+            const b = sessions[right];
+            if (normalize(a.provider) === normalize(b.provider))
+                continue;
+            const minutes = Math.abs(Date.parse(a.completed_at) - Date.parse(b.completed_at)) / 60_000;
+            if (minutes > 15 || normalize(a.title) !== normalize(b.title) || !participantOverlap(a, b))
+                continue;
+            proposals.push({
+                left_fingerprint: sourceFingerprint(a),
+                right_fingerprint: sourceFingerprint(b),
+                confidence: minutes <= 5 ? "high" : "medium",
+                action: "review-canonical-source",
+            });
+        }
+    }
+    return proposals;
+}

package/dist-public/src/discoveryOutlook.d.ts

@@ -0,0 +1,12 @@
+import { type DiscoveryCommandOutcome } from "./discoveryCommand.js";
+import { type SynthesisTransport } from "./discoverySynthesis.js";
+import { type FetchLike } from "./outlookNango.js";
+export type OutlookDiscoveryDeps = {
+    fetchImpl?: FetchLike;
+    env?: NodeJS.ProcessEnv;
+    now?: () => string;
+    sleep?: (ms: number) => Promise<void>;
+    transport?: SynthesisTransport;
+    fetchTimeoutMs?: number;
+};
+export declare function runOutlookDiscovery(root: string, args: string[], deps?: OutlookDiscoveryDeps): Promise<DiscoveryCommandOutcome>;