swarm-notes 0.3.4__tar.gz → 0.3.6__tar.gz

{swarm_notes-0.3.4 → swarm_notes-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: swarm-notes
-Version: 0.3.4
+Version: 0.3.6
 Summary: Automated research paper tracking and knowledge synthesis
 Author: LM
 Author-email: LM <hi@leima.is>
@@ -140,6 +140,17 @@ for biorxiv/medrxiv papers.
 For CI/CD, add `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` as repository secrets
 (same place as `LLM_API_KEY`).
 
+### Fallback model
+
+Set `llm_fallback_model` in the config (or the `LLM_FALLBACK_MODEL` environment variable) to a second model. Every agent call tries `llm_model` first and switches to the fallback when that call fails, for example on a rate limit, a 4xx/5xx error or an outage:
+
+```yaml
+llm_model: "google-gla:gemini-flash-lite-latest"
+llm_fallback_model: "anthropic:claude-haiku-5-5"
+```
+
+The fallback provider needs its own key, for example `ANTHROPIC_API_KEY` for Claude.
+
 ## CI/CD Setup
 
 ### Add the required secret

{swarm_notes-0.3.4 → swarm_notes-0.3.6}/README.md

@@ -121,6 +121,17 @@ for biorxiv/medrxiv papers.
 For CI/CD, add `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` as repository secrets
 (same place as `LLM_API_KEY`).
 
+### Fallback model
+
+Set `llm_fallback_model` in the config (or the `LLM_FALLBACK_MODEL` environment variable) to a second model. Every agent call tries `llm_model` first and switches to the fallback when that call fails, for example on a rate limit, a 4xx/5xx error or an outage:
+
+```yaml
+llm_model: "google-gla:gemini-flash-lite-latest"
+llm_fallback_model: "anthropic:claude-haiku-5-5"
+```
+
+The fallback provider needs its own key, for example `ANTHROPIC_API_KEY` for Claude.
+
 ## CI/CD Setup
 
 ### Add the required secret

{swarm_notes-0.3.4 → swarm_notes-0.3.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "swarm-notes"
-version = "0.3.4"
+version = "0.3.6"
 description = "Automated research paper tracking and knowledge synthesis"
 readme = "README.md"
 authors = [

{swarm_notes-0.3.4 → swarm_notes-0.3.6}/src/swarm_notes/analyst.py

@@ -8,14 +8,16 @@ from __future__ import annotations
 
 import json
 import logging
+import re
+from collections import Counter
 from typing import Annotated, Literal
 
 from pydantic import BaseModel, Field
 from pydantic_ai import Agent
 
-from swarm_notes.config import settings
+from swarm_notes.config import get_model, settings
 from swarm_notes.router import SkillSpec
-from swarm_notes.vault_manager import get_existing_tags
+from swarm_notes.vault_manager import get_tag_counts, select_relevant
 from swarm_notes.watcher import RawPaper
 
 logger = logging.getLogger(__name__)
@@ -222,8 +224,10 @@ class PaperAnalysis(PaperAnalysisBase):
 # Upper bound on how many existing tags are listed in the prompt. Tags are
 # passed as prompt text rather than as an enum in the output schema: a schema
 # enum with hundreds of values makes Gemini reject the request with
-# 400 INVALID_ARGUMENT.
-MAX_PROMPT_TAGS = 1000
+# 400 INVALID_ARGUMENT. The tags listed are the ones most relevant to the
+# paper (see ``select_relevant``), so the prompt stays the same size as the
+# vault grows.
+MAX_PROMPT_TAGS = 200
 
 
 # ---------------------------------------------------------------------------
@@ -291,11 +295,21 @@ def analyse(paper: RawPaper, skill: SkillSpec) -> PaperAnalysis:
         Structured extraction result.
     """
     taxonomy = _load_taxonomy()
-    existing_tags = get_existing_tags()
-    system_prompt = _build_system_prompt(skill, taxonomy, existing_tags)
+    tag_counts = get_tag_counts()
+    existing_tags = sorted(tag_counts)
+    # List one spelling per tag (the most used), so variants such as
+    # "time-series" and "time series" don't take up several slots.
+    canonical_tags = sorted(set(_canonical_tags(existing_tags, tag_counts).values()))
+    prompt_tags = select_relevant(
+        canonical_tags,
+        f"{paper.title}\n{paper.abstract}",
+        MAX_PROMPT_TAGS,
+        tag_counts,
+    )
+    system_prompt = _build_system_prompt(skill, taxonomy, prompt_tags)
 
     agent: Agent[None, PaperAnalysis] = Agent(
-        model=settings.llm_model,
+        model=get_model(),
         output_type=PaperAnalysis,
         system_prompt=system_prompt,
     )
@@ -313,7 +327,7 @@ def analyse(paper: RawPaper, skill: SkillSpec) -> PaperAnalysis:
 
     result = agent.run_sync(user_message)
     analysis: PaperAnalysis = result.output
-    analysis.tags = _normalise_tags(analysis.tags, existing_tags)
+    analysis.tags = _normalise_tags(analysis.tags, existing_tags, tag_counts)
 
     # Ensure metadata from the raw paper is preserved accurately
     analysis.arxiv_id = paper.arxiv_id
@@ -332,19 +346,39 @@ def analyse(paper: RawPaper, skill: SkillSpec) -> PaperAnalysis:
     return analysis
 
 
-def _normalise_tags(tags: list[str], existing_tags: list[str]) -> list[str]:
-    """Map tags onto the vault's existing spelling (case-insensitive) and drop duplicates."""
-    canonical = {t.lower(): t for t in existing_tags}
+def _tag_key(tag: str) -> str:
+    """Key that treats case, spaces, hyphens and underscores as the same."""
+    return re.sub(r"[\s_\-]+", " ", tag.strip().lower()).strip()
+
+
+def _canonical_tags(existing_tags: list[str], counts: Counter[str] | None = None) -> dict[str, str]:
+    """Map each tag key to the vault's most used spelling of it."""
+    counts = counts or Counter()
+    canonical: dict[str, str] = {}
+    for t in sorted(existing_tags, key=lambda t: (-counts.get(t, 0), t)):
+        canonical.setdefault(_tag_key(t), t)
+    return canonical
+
+
+def _normalise_tags(
+    tags: list[str],
+    existing_tags: list[str],
+    counts: Counter[str] | None = None,
+) -> list[str]:
+    """Map tags onto the vault's existing spelling and drop duplicates.
+
+    "Time-Series", "time series" and "time_series" all map to whichever
+    spelling the vault already uses most.
+    """
+    canonical = _canonical_tags(existing_tags, counts)
     result: list[str] = []
     seen: set[str] = set()
-    for tag in tags:
-        tag = tag.strip()
-        if not tag:
+    for tag in (part.strip() for t in tags for part in t.split(",")):
+        key = _tag_key(tag)
+        if not key or key in seen:
             continue
-        tag = canonical.get(tag.lower(), tag)
-        if tag.lower() not in seen:
-            seen.add(tag.lower())
-            result.append(tag)
+        seen.add(key)
+        result.append(canonical.get(key, tag))
     return result
 
 

{swarm_notes-0.3.4 → swarm_notes-0.3.6}/src/swarm_notes/config.py

@@ -153,6 +153,9 @@ class Settings(BaseModel):
     openai_api_key: str = Field(default_factory=lambda: os.environ.get("OPENAI_API_KEY", os.environ.get("LLM_API_KEY", "")))
     gemini_api_key: str = Field(default_factory=lambda: os.environ.get("GEMINI_API_KEY", os.environ.get("GOOGLE_API_KEY", "")))
     llm_model: str = Field(default="openai:gpt-4o-mini")
+    # Optional second model, tried whenever a call to llm_model fails (rate
+    # limit, 4xx/5xx, outage). Empty disables the fallback.
+    llm_fallback_model: str = Field(default_factory=lambda: os.environ.get("LLM_FALLBACK_MODEL", ""))
     
     # Watcher config
     paper_search: PaperSearchSettings = Field(default_factory=PaperSearchSettings)
@@ -334,6 +337,15 @@ class Settings(BaseModel):
 
 settings = Settings.load_from_yaml()
 
+def get_model() -> Any:
+    """Return the model for agents: llm_model, falling back to llm_fallback_model on errors."""
+    if not settings.llm_fallback_model:
+        return settings.llm_model
+    from pydantic_ai.models.fallback import FallbackModel
+
+    return FallbackModel(settings.llm_model, settings.llm_fallback_model)
+
+
 def load_yaml_config(yaml_path: str | Path) -> None:
     """Override configuration variables from a YAML file."""
     global settings

{swarm_notes-0.3.4 → swarm_notes-0.3.6}/src/swarm_notes/critic.py

@@ -10,9 +10,9 @@ from pydantic import BaseModel, Field
 from pydantic_ai import Agent
 
 from swarm_notes.analyst import ConceptLink, DatasetLink, OpenQuestion, PaperAnalysis, RejectedCandidate
-from swarm_notes.config import settings
+from swarm_notes.config import get_model, settings
 from swarm_notes.router import SkillSpec
-from swarm_notes.vault_manager import get_existing_entities
+from swarm_notes.vault_manager import get_existing_entities, select_relevant
 from swarm_notes.watcher import RawPaper
 
 logger = logging.getLogger(__name__)
@@ -61,13 +61,29 @@ class CriticReview(BaseModel):
     )
 
 
-def _build_system_prompt(skill: SkillSpec) -> str:
-    existing_concepts = get_existing_entities("concepts")
-    existing_concepts_str = ", ".join(existing_concepts) if existing_concepts else "(No concepts currently exist in the vault)"
-    existing_open_questions = get_existing_entities("open_questions")
-    existing_open_questions_str = ", ".join(existing_open_questions) if existing_open_questions else "(No open questions currently exist in the vault)"
-    existing_datasets = get_existing_entities("datasets")
-    existing_datasets_str = ", ".join(existing_datasets) if existing_datasets else "(No datasets currently exist in the vault)"
+# Upper bound on how many existing concepts, open questions and datasets are
+# listed in the critic prompt (per type). The vault holds thousands of them, so
+# only the ones most relevant to the paper are shown.
+MAX_PROMPT_ENTITIES = 150
+
+
+def _existing_entities_str(entity_type: str, context: str | None, empty: str) -> str:
+    names = get_existing_entities(entity_type)
+    if context is not None:
+        names = select_relevant(names, context, MAX_PROMPT_ENTITIES)
+    return ", ".join(names) if names else empty
+
+
+def _build_system_prompt(skill: SkillSpec, context: str | None = None) -> str:
+    existing_concepts_str = _existing_entities_str(
+        "concepts", context, "(No concepts currently exist in the vault)"
+    )
+    existing_open_questions_str = _existing_entities_str(
+        "open_questions", context, "(No open questions currently exist in the vault)"
+    )
+    existing_datasets_str = _existing_entities_str(
+        "datasets", context, "(No datasets currently exist in the vault)"
+    )
 
     prompt = f"""You are a highly selective research archivist protecting a long-lived ML knowledge vault.
 
@@ -115,6 +131,16 @@ REVIEW POLICY:
     return prompt
 
 
+def _relevance_context(analysis: PaperAnalysis, paper: RawPaper) -> str:
+    """Text used to pick which existing vault entities the critic sees."""
+    candidates = [
+        *(f"{c.slug} {c.one_liner}" for c in analysis.concepts),
+        *(f"{q.slug} {q.description}" for q in analysis.open_questions),
+        *(f"{d.name} {d.description}" for d in analysis.datasets),
+    ]
+    return "\n".join([paper.title, paper.abstract, *candidates])
+
+
 def review_analysis(analysis: PaperAnalysis, paper: RawPaper, skill: SkillSpec) -> PaperAnalysis:
     """Review candidate concepts, open questions, and datasets before writing to the vault."""
     if not analysis.concepts and not analysis.open_questions and not analysis.datasets:
@@ -122,9 +148,9 @@ def review_analysis(analysis: PaperAnalysis, paper: RawPaper, skill: SkillSpec)
         return analysis
 
     agent: Agent[None, CriticReview] = Agent(
-        model=settings.llm_model,
+        model=get_model(),
         output_type=CriticReview,
-        system_prompt=_build_system_prompt(skill),
+        system_prompt=_build_system_prompt(skill, _relevance_context(analysis, paper)),
     )
 
     payload = {

{swarm_notes-0.3.4 → swarm_notes-0.3.6}/src/swarm_notes/discussant.py

@@ -5,7 +5,7 @@ import logging
 from pydantic_ai import Agent
 
 from swarm_notes.analyst import PaperAnalysis
-from swarm_notes.config import settings
+from swarm_notes.config import get_model
 from swarm_notes.router import SkillSpec
 
 logger = logging.getLogger(__name__)
@@ -27,7 +27,7 @@ def discuss_papers(analyses: list[PaperAnalysis], skill: SkillSpec | None = None
         system_prompt += f"\n\nSKILL-SPECIFIC CONTEXT:\n{skill.discussant_context}"
 
     discussant_agent = Agent(
-        model=settings.llm_model,
+        model=get_model(),
         system_prompt=system_prompt
     )
 

{swarm_notes-0.3.4 → swarm_notes-0.3.6}/src/swarm_notes/domain_expert.py

@@ -6,7 +6,7 @@ from bs4 import BeautifulSoup
 from pydantic_ai import Agent
 
 from swarm_notes.analyst import OpenQuestion
-from swarm_notes.config import settings
+from swarm_notes.config import get_model, settings
 from swarm_notes.router import SkillSpec
 
 logger = logging.getLogger(__name__)
@@ -61,7 +61,7 @@ def extract_open_questions(
         system_prompt += f"\n\nSKILL-SPECIFIC CONTEXT:\n{skill.domain_expert_context}"
 
     agent: Agent[None, list[OpenQuestion]] = Agent(
-        model=settings.llm_model,
+        model=get_model(),
         output_type=list[OpenQuestion],
         system_prompt=system_prompt
     )

{swarm_notes-0.3.4 → swarm_notes-0.3.6}/src/swarm_notes/vault_manager.py

@@ -9,7 +9,9 @@ partial corruption of the live vault.
 from __future__ import annotations
 
 import logging
+import re
 import shutil
+from collections import Counter
 from pathlib import Path
 
 import frontmatter
@@ -88,42 +90,117 @@ def _merge_directory(src: Path, dst: Path) -> None:
             logger.debug("Staged %s → %s", src_file, dst_file)
 
 
-def get_existing_tags() -> list[str]:
-    """Return a deduplicated list of all tags currently found in live vault papers."""
-    if not settings.vault_papers_dir.exists():
+_ENTITY_DIR_SETTINGS = {
+    "concepts": "vault_concepts_dir",
+    "datasets": "vault_datasets_dir",
+    "open_questions": "vault_open_questions_dir",
+}
+
+# Cache of parsed paper frontmatter, keyed on the papers directory and its
+# state. A run only writes to tmp_vault/, so the live vault is parsed once per
+# run instead of once per prompt.
+_paper_metadata_cache: dict[tuple, list[dict]] = {}
+
+
+def _load_paper_metadata() -> list[dict]:
+    """Return the frontmatter of every live vault paper, parsed once per vault state."""
+    papers_dir = settings.vault_papers_dir
+    if not papers_dir.exists():
         return []
-    tags_set = set()
-    for f in settings.vault_papers_dir.glob("*.md"):
+    files = sorted(papers_dir.glob("*.md"))
+    key = (str(papers_dir), papers_dir.stat().st_mtime_ns, len(files))
+    cached = _paper_metadata_cache.get(key)
+    if cached is not None:
+        return cached
+    metadata: list[dict] = []
+    for f in files:
         try:
-            post = frontmatter.load(f)
-            file_tags = post.metadata.get("tags", [])
-            if isinstance(file_tags, list):
-                for t in file_tags:
-                    if isinstance(t, str) and t.strip():
-                        tags_set.add(t.strip())
-            elif isinstance(file_tags, str) and file_tags.strip():
-                tags_set.add(file_tags.strip())
+            metadata.append(frontmatter.load(f).metadata)
         except Exception as exc:
-            logger.warning("VaultManager: failed to parse tags from %s: %s", f, exc)
-    return sorted(list(tags_set))
+            logger.warning("VaultManager: failed to parse frontmatter from %s: %s", f, exc)
+    _paper_metadata_cache.clear()
+    _paper_metadata_cache[key] = metadata
+    return metadata
+
+
+def get_tag_counts() -> Counter[str]:
+    """Return how many live vault papers use each tag."""
+    counts: Counter[str] = Counter()
+    for meta in _load_paper_metadata():
+        file_tags = meta.get("tags", [])
+        if isinstance(file_tags, str):
+            file_tags = [file_tags]
+        if not isinstance(file_tags, list):
+            continue
+        # Some older notes hold several tags in one comma-joined string.
+        split = {part.strip() for t in file_tags if isinstance(t, str) for part in t.split(",")}
+        for t in split - {""}:
+            counts[t] += 1
+    return counts
+
+
+def get_existing_tags() -> list[str]:
+    """Return a deduplicated list of all tags currently found in live vault papers."""
+    return sorted(get_tag_counts())
 
 
 def get_existing_entities(entity_type: str) -> list[str]:
-    """Return a deduplicated list of entity names (e.g. concepts, datasets, open_questions) from live vault papers."""
-    if not settings.vault_papers_dir.exists():
-        return []
-    entity_set = set()
-    for f in settings.vault_papers_dir.glob("*.md"):
-        try:
-            post = frontmatter.load(f)
-            entities = post.metadata.get(entity_type, [])
-            if isinstance(entities, list):
-                for e in entities:
-                    if isinstance(e, dict) and "name" in e:
-                        entity_set.add(e["name"])
-        except Exception as exc:
-            pass
-    return sorted(list(entity_set))
+    """Return entity names (e.g. concepts, datasets, open_questions) known to the vault.
+
+    Names come from paper frontmatter and from the slugs of the standalone
+    entity notes (``vault/concepts/<slug>.md`` and so on).
+    """
+    entity_set: set[str] = set()
+    for meta in _load_paper_metadata():
+        entities = meta.get(entity_type, [])
+        if isinstance(entities, list):
+            for e in entities:
+                if isinstance(e, dict) and e.get("name"):
+                    entity_set.add(str(e["name"]))
+    dir_setting = _ENTITY_DIR_SETTINGS.get(entity_type)
+    entity_dir = getattr(settings, dir_setting, None) if dir_setting else None
+    if entity_dir is not None and entity_dir.exists():
+        entity_set.update(f.stem for f in entity_dir.glob("*.md") if not f.name.startswith("_"))
+    return sorted(entity_set)
+
+
+_WORD_RE = re.compile(r"[a-z0-9]+")
+_STOPWORDS = frozenset({
+    "and", "are", "based", "for", "from", "into", "its", "new", "our",
+    "the", "their", "this", "that", "towards", "using", "via", "with",
+})
+
+
+def _words(text: str) -> set[str]:
+    return {w for w in _WORD_RE.findall(text.lower()) if len(w) > 2 and w not in _STOPWORDS}
+
+
+def select_relevant(
+    names: list[str],
+    text: str,
+    limit: int,
+    counts: Counter[str] | None = None,
+) -> list[str]:
+    """Pick at most *limit* names, the ones sharing the most words with *text* first.
+
+    Names with no word in common are used only to fill the remaining slots,
+    most frequent first (by *counts*). This keeps prompts a fixed size however
+    large the vault grows, while still showing the names a paper is likely to
+    reuse.
+    """
+    if len(names) <= limit:
+        return list(names)
+    text_words = _words(text)
+    counts = counts or Counter()
+
+    def rank(name: str) -> tuple:
+        name_words = _words(name)
+        shared = len(name_words & text_words)
+        # More shared words first, then names that are mostly shared words
+        # ("forecasting" over "forecasting-for-sparse-retail-demand").
+        return (-shared, -shared / max(len(name_words), 1), -counts.get(name, 0), name)
+
+    return sorted(names, key=rank)[:limit]
 
 
 def make_storage_id(source: str, paper_id: str) -> str: