althing 1.7.0__py3-none-any.whl

althing/pack_diff.py

@@ -0,0 +1,271 @@
+"""Diff two persona packs side-by-side (GH-308).
+
+Compares packs by persona name (added / removed / unchanged / changed) and
+summarizes composition deltas (age range, role distribution, gender split
+when present). Loads either built-in / user-saved pack IDs or file paths.
+
+Name-based matching is intentionally simple — renames will surface as
+``added`` + ``removed`` rather than a single change. The README documents
+this so users can interpret results correctly.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from althing.mcp.data import get_persona_pack
+
+# ---------------------------------------------------------------------------
+# Data models
+# ---------------------------------------------------------------------------
+
+# Persona-level fields the diff inspects when both packs contain a persona of
+# the same name. Adding new fields here is the supported way to broaden the
+# field-level diff without touching the matching logic.
+_FIELD_KEYS: tuple[str, ...] = (
+    "age",
+    "occupation",
+    "background",
+    "personality_traits",
+    "gender",
+)
+
+_SENIORITY_PREFIXES = {"junior", "senior", "staff", "principal", "lead", "chief"}
+
+
+@dataclass
+class CompositionStats:
+    """Aggregate demographic snapshot for a single pack.
+
+    ``gender_split`` is empty when no persona declares a ``gender`` field
+    (the common case for the bundled packs). ``role_distribution`` is keyed
+    on a one-word bucket derived from each persona's ``occupation``.
+    """
+
+    persona_count: int
+    age_min: int | None
+    age_max: int | None
+    age_mean: float | None
+    gender_split: dict[str, int] = field(default_factory=dict)
+    role_distribution: dict[str, int] = field(default_factory=dict)
+
+
+@dataclass
+class PersonaChange:
+    """Field-level diff for a persona present in both packs.
+
+    ``changed`` maps each differing field to ``{"a": <a-value>, "b": <b-value>}``.
+    For ``personality_traits`` the values are the raw lists; the renderer
+    surfaces added/removed traits separately for readability.
+    """
+
+    name: str
+    changed: dict[str, dict[str, Any]]
+
+
+@dataclass
+class PackDiff:
+    pack_a_id: str
+    pack_b_id: str
+    pack_a_name: str
+    pack_b_name: str
+    composition_a: CompositionStats
+    composition_b: CompositionStats
+    added: list[str]
+    removed: list[str]
+    unchanged: list[str]
+    changed: list[PersonaChange]
+
+
+# ---------------------------------------------------------------------------
+# Loading
+# ---------------------------------------------------------------------------
+
+
+def load_pack(source: str) -> tuple[dict[str, Any], str]:
+    """Load a persona pack from a file path or built-in / user-saved pack ID.
+
+    File paths win when *source* points to an existing file — this lets
+    callers diff unpublished packs without first installing them. Otherwise
+    the loader falls back to :func:`althing.mcp.data.get_persona_pack`,
+    which checks user-saved packs first and then bundled packs.
+
+    Returns ``(pack_dict, source_label)`` — the label is the file stem for
+    paths and the original ID for installed packs, used for human-readable
+    output.
+    """
+    p = Path(source)
+    if p.is_file():
+        data = yaml.safe_load(p.read_text(encoding="utf-8"))
+        if not isinstance(data, dict):
+            raise ValueError(f"Pack file does not contain a mapping: {source}")
+        return data, p.stem
+    return get_persona_pack(source), source
+
+
+# ---------------------------------------------------------------------------
+# Composition stats
+# ---------------------------------------------------------------------------
+
+
+def _normalize_traits(traits: Any) -> list[str]:
+    """Collapse traits to a sorted list of stripped lowercase strings."""
+    if traits is None:
+        return []
+    if isinstance(traits, str):
+        items = [t.strip().lower() for t in traits.split(",") if t.strip()]
+    elif isinstance(traits, list):
+        items = [str(t).strip().lower() for t in traits if str(t).strip()]
+    else:
+        return []
+    return sorted(items)
+
+
+def _role_bucket(occupation: str) -> str:
+    """Coarse single-word bucket for an occupation string.
+
+    Drops seniority prefixes (Junior, Senior, Staff, Principal, Lead, Chief)
+    and returns the first remaining alphabetical token, lowercased. Falls
+    back to the raw stripped string when no tokens match.
+    """
+    tokens = re.findall(r"[A-Za-z][A-Za-z0-9-]+", occupation)
+    if not tokens:
+        return occupation.strip().lower()
+    cleaned = [t.lower() for t in tokens if t.lower() not in _SENIORITY_PREFIXES]
+    if not cleaned:
+        cleaned = [t.lower() for t in tokens]
+    return cleaned[0]
+
+
+def _compose(personas: list[dict[str, Any]]) -> CompositionStats:
+    ages = [int(p["age"]) for p in personas if isinstance(p.get("age"), int)]
+    age_min = min(ages) if ages else None
+    age_max = max(ages) if ages else None
+    age_mean = round(sum(ages) / len(ages), 1) if ages else None
+
+    gender_counter: Counter[str] = Counter()
+    for p in personas:
+        g = p.get("gender")
+        if g:
+            gender_counter[str(g).lower().strip()] += 1
+
+    role_counter: Counter[str] = Counter()
+    for p in personas:
+        occ = p.get("occupation")
+        if occ:
+            role_counter[_role_bucket(str(occ))] += 1
+
+    return CompositionStats(
+        persona_count=len(personas),
+        age_min=age_min,
+        age_max=age_max,
+        age_mean=age_mean,
+        gender_split=dict(gender_counter),
+        role_distribution=dict(role_counter),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Field-level diff
+# ---------------------------------------------------------------------------
+
+
+def _persona_field_diff(a: dict[str, Any], b: dict[str, Any]) -> dict[str, dict[str, Any]]:
+    """Return the subset of ``_FIELD_KEYS`` that differ between two personas.
+
+    ``personality_traits`` is compared after normalization (sorted, lowercase,
+    stripped) so cosmetic ordering or casing changes don't trip the diff.
+    Other fields are compared with raw equality and ``None == None`` is not
+    flagged.
+    """
+    changed: dict[str, dict[str, Any]] = {}
+    for key in _FIELD_KEYS:
+        av = a.get(key)
+        bv = b.get(key)
+        if key == "personality_traits":
+            if _normalize_traits(av) != _normalize_traits(bv):
+                changed[key] = {"a": av, "b": bv}
+        else:
+            if av is None and bv is None:
+                continue
+            if av != bv:
+                changed[key] = {"a": av, "b": bv}
+    return changed
+
+
+def trait_delta(change: PersonaChange) -> tuple[list[str], list[str]]:
+    """Return ``(added, removed)`` traits for a persona change, or empty lists."""
+    payload = change.changed.get("personality_traits")
+    if not payload:
+        return [], []
+    a = set(_normalize_traits(payload.get("a")))
+    b = set(_normalize_traits(payload.get("b")))
+    return sorted(b - a), sorted(a - b)
+
+
+# ---------------------------------------------------------------------------
+# Main entry point
+# ---------------------------------------------------------------------------
+
+
+def compute_pack_diff(
+    pack_a: dict[str, Any],
+    pack_b: dict[str, Any],
+    *,
+    pack_a_id: str = "",
+    pack_b_id: str = "",
+) -> PackDiff:
+    """Compute a side-by-side diff of two persona packs.
+
+    Personas are matched by ``name`` (exact). Renames appear as one entry in
+    ``added`` and one in ``removed``. ``unchanged`` lists matched personas
+    whose tracked fields are all equal; ``changed`` lists matched personas
+    with at least one differing field.
+    """
+    personas_a = pack_a.get("personas") or []
+    personas_b = pack_b.get("personas") or []
+    if not isinstance(personas_a, list):
+        personas_a = []
+    if not isinstance(personas_b, list):
+        personas_b = []
+
+    by_name_a = {p["name"]: p for p in personas_a if isinstance(p, dict) and p.get("name")}
+    by_name_b = {p["name"]: p for p in personas_b if isinstance(p, dict) and p.get("name")}
+
+    names_a = set(by_name_a)
+    names_b = set(by_name_b)
+
+    added = sorted(names_b - names_a)
+    removed = sorted(names_a - names_b)
+    common = sorted(names_a & names_b)
+
+    unchanged: list[str] = []
+    changed: list[PersonaChange] = []
+    for name in common:
+        diff = _persona_field_diff(by_name_a[name], by_name_b[name])
+        if diff:
+            changed.append(PersonaChange(name=name, changed=diff))
+        else:
+            unchanged.append(name)
+
+    a_label = pack_a_id or str(pack_a.get("id") or "") or "pack_a"
+    b_label = pack_b_id or str(pack_b.get("id") or "") or "pack_b"
+
+    return PackDiff(
+        pack_a_id=a_label,
+        pack_b_id=b_label,
+        pack_a_name=str(pack_a.get("name") or a_label),
+        pack_b_name=str(pack_b.get("name") or b_label),
+        composition_a=_compose([p for p in personas_a if isinstance(p, dict)]),
+        composition_b=_compose([p for p in personas_b if isinstance(p, dict)]),
+        added=added,
+        removed=removed,
+        unchanged=unchanged,
+        changed=changed,
+    )

althing/packs/__init__.py

@@ -0,0 +1,3 @@
+"""Bundled persona packs shipped with althing."""
+
+from __future__ import annotations

althing/packs/ai-eval-buyers.yaml

@@ -0,0 +1,247 @@
+name: AI & ML Evaluation Buyers
+description: >
+  ML engineers, AI researchers, eval platform leaders, devtool PMs, and
+  infra engineers across startup, enterprise, and research settings.
+  Buyer audience for LLM evaluation frameworks, benchmark platforms,
+  model-ops tooling, and AI-platform infrastructure. Useful for testing
+  ML/AI tooling products and positioning.
+
+personas:
+  - name: Carla Mendes
+    age: 32
+    occupation: ML engineer at a multimodal startup
+    background: >
+      Runs daily evals on vision-language models. Writes eval code in
+      Python + pytest. Frustrated by how long it takes to spin up a
+      reproducible benchmark; lives in spreadsheets.
+    personality_traits:
+      - applied ML
+      - CLI-native
+      - spreadsheet-escaper
+
+  - name: Jinhua Zhao
+    age: 38
+    occupation: Senior research engineer, agent labs
+    background: >
+      Designs and runs agent benchmarks for a research org. Needs
+      reproducibility, determinism, and long-horizon trace capture.
+      Local-first philosophy matters more than dashboards.
+    personality_traits:
+      - agent researcher
+      - reproducibility-obsessed
+      - local-first
+
+  - name: Mohammed Al-Sabah
+    age: 41
+    occupation: Head of AI evaluation at an enterprise fintech
+    background: >
+      Owns model approval gates before prod. Needs frameworks that
+      produce audit-trail-ready evidence. Regulatory exposure high.
+    personality_traits:
+      - enterprise buyer
+      - audit-minded
+      - compliance-heavy
+
+  - name: Kaitlyn Park
+    age: 28
+    occupation: ML platform engineer at a seed-stage startup
+    background: >
+      Builds ML infra for a 6-person ML team. Cost-conscious, no budget
+      for commercial eval platforms. Evaluates open-source alternatives
+      by deployment simplicity and local-mode depth.
+    personality_traits:
+      - platform engineer
+      - budget-constrained
+      - OSS-native
+
+  - name: Sergei Volodin
+    age: 36
+    occupation: Prompt engineer at a developer-tools company
+    background: >
+      Iterates on system prompts for production agents. Needs quick
+      A/B eval harnesses. Hates tools that require elaborate setup for
+      what should be a one-liner.
+    personality_traits:
+      - prompt engineer
+      - iteration-speed
+      - setup-averse
+
+  - name: Priya Iyer
+    age: 33
+    occupation: Applied AI researcher, consulting firm
+    background: >
+      Runs AI POCs for client engagements. Needs frameworks that slot
+      into different client data stacks without heavy onboarding. Portable
+      BYOK model matters because clients insist on their own providers.
+    personality_traits:
+      - consultant researcher
+      - BYOK-first
+      - portable
+
+  - name: Marcus Liu
+    age: 42
+    occupation: Director of evaluation at a foundation-model lab
+    background: >
+      Compares benchmark results across internal models and competitors.
+      Already uses lm-eval-harness, HELM, and custom harnesses. Looking
+      for gaps newer eval tools fill vs what he has.
+    personality_traits:
+      - benchmark veteran
+      - comparison-buyer
+      - domain-deep
+
+  - name: Ines Costa
+    age: 29
+    occupation: ML engineer at an open-source LLM startup
+    background: >
+      Ships model releases every 6 weeks. Needs eval pipelines that run
+      in CI and block releases on regressions. Automation and CLI
+      ergonomics are table stakes.
+    personality_traits:
+      - release-cadence-driven
+      - CI-integrator
+      - automator
+
+  - name: Ahmad Qureshi
+    age: 35
+    occupation: Academic researcher, on sabbatical in industry
+    background: >
+      Evaluates tools through a publications lens. Wants reproducibility
+      hooks (seed control, config versioning) so he can cite results
+      defensibly in papers. Less interested in dashboards.
+    personality_traits:
+      - academic rigor
+      - reproducibility-first
+      - paper-oriented
+
+  - name: Rebecca Chen
+    age: 30
+    occupation: Staff engineer at a gaming AI startup
+    background: >
+      Evaluates NPC-behavior LLMs. Benchmarks need multi-turn game
+      contexts and custom scoring. Looking for frameworks flexible
+      enough to plug in domain-specific scorers.
+    personality_traits:
+      - game AI
+      - custom-scorer needer
+      - multi-turn-focused
+
+  - name: Tobias Braun
+    age: 47
+    occupation: VP engineering at a 400-person AI platform
+    background: >
+      Evaluates enterprise tooling for the team. Wants SSO, RBAC, audit
+      logs, and SOC2 readiness. Features matter less than "will it
+      survive a procurement cycle".
+    personality_traits:
+      - VPE buyer
+      - procurement-aware
+      - enterprise-hardened
+
+  - name: Himanshu Patel
+    age: 31
+    occupation: Devtool PM at a developer cloud company
+    background: >
+      Scoping a "model eval" feature for his product. Torn between
+      building vs integrating an eval framework. Wants to know the
+      open-source story deeply before committing to build.
+    personality_traits:
+      - devtool PM
+      - integrate-vs-build
+      - OSS-diligent
+
+  - name: Lin Xiao
+    age: 34
+    occupation: Senior applied scientist, conversational AI
+    background: >
+      Evaluates dialog quality. Benchmarks involve human calibration
+      and inter-rater agreement. Needs frameworks that support hybrid
+      human + automated scoring workflows.
+    personality_traits:
+      - hybrid-eval
+      - dialog-quality
+      - rater-aware
+
+  - name: Alejandro Ruiz
+    age: 39
+    occupation: Principal engineer, evaluation platforms
+    background: >
+      Ten years building internal eval tooling at FAANG. Now at a
+      Series-D startup. Evaluates externals with a "did we already
+      build this internally?" lens.
+    personality_traits:
+      - eval-veteran
+      - internal-vs-external buyer
+      - pragmatic
+
+  - name: Noemi Rossi
+    age: 27
+    occupation: Junior ML engineer at a healthtech startup
+    background: >
+      First ML job. Overwhelmed by the eval-tool landscape. Wants a
+      tool that teaches her methodology as she uses it — opinionated
+      defaults more valuable than infinite configurability.
+    personality_traits:
+      - junior
+      - learning-oriented
+      - opinionated-defaults wanted
+
+  - name: Daniel Okonkwo
+    age: 40
+    occupation: Security engineer at an AI safety org
+    background: >
+      Evaluates models for adversarial robustness and jailbreak
+      resistance. Needs red-team eval plugins. Interested in whether
+      a framework's extension points support security-focused probes.
+    personality_traits:
+      - AI safety
+      - red-team-oriented
+      - extensibility-focused
+
+  - name: Elena Georgescu
+    age: 36
+    occupation: Lead data scientist, marketing analytics
+    background: >
+      Less ML-engineer-y, more analyst. Evaluates text-generation models
+      for tone and brand-fit. Methodology-curious about how qualitative
+      outputs are scored.
+    personality_traits:
+      - analyst-tilted
+      - qualitative-focused
+      - methodology curious
+
+  - name: Kiran Joshi
+    age: 32
+    occupation: Infrastructure engineer, LLM inference platform
+    background: >
+      Evaluates inference backends (vLLM, TGI, custom). Cares about
+      throughput + cost-per-eval. Wants eval tooling to tell him when
+      a regression is model-quality vs infra-latency.
+    personality_traits:
+      - infra engineer
+      - cost-per-eval
+      - root-cause-oriented
+
+  - name: Yasmin Benjelloun
+    age: 29
+    occupation: Research software engineer, NLP lab
+    background: >
+      Writes evaluation harnesses for professors. Inheriting grad-student
+      code is a recurring pain point. Interested in whether structured
+      eval tools produce reusable, hand-off-able code.
+    personality_traits:
+      - research SWE
+      - code-continuity-focused
+      - lab-reality
+
+  - name: Gregor Novak
+    age: 44
+    occupation: Principal AI architect, consulting
+    background: >
+      Designs AI systems for F500 clients. Evaluates eval tools across
+      a dozen projects. Needs tools that scale from POC to prod without
+      rewriting. Wary of tools that only work at one end of that spectrum.
+    personality_traits:
+      - architect consultant
+      - poc-to-prod seeker
+      - vendor-weary