swarm-notes 0.2.0__tar.gz → 0.3.0__tar.gz

{swarm_notes-0.2.0 → swarm_notes-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: swarm-notes
-Version: 0.2.0
+Version: 0.3.0
 Summary: Automated research paper tracking and knowledge synthesis
 Author: LM
 Author-email: LM <hi@leima.is>

{swarm_notes-0.2.0 → swarm_notes-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "swarm-notes"
-version = "0.2.0"
+version = "0.3.0"
 description = "Automated research paper tracking and knowledge synthesis"
 readme = "README.md"
 authors = [

{swarm_notes-0.2.0 → swarm_notes-0.3.0}/src/swarm_notes/analyst.py

@@ -15,7 +15,7 @@ from pydantic_ai import Agent
 
 from swarm_notes.config import settings
 from swarm_notes.router import SkillSpec
-from swarm_notes.vault_manager import get_existing_concept_slugs
+from swarm_notes.vault_manager import get_existing_tags
 from swarm_notes.watcher import RawPaper
 
 logger = logging.getLogger(__name__)
@@ -124,8 +124,14 @@ class RejectedCandidate(BaseModel):
     )
 
 
-class PaperAnalysis(BaseModel):
-    """Structured extraction output for a single academic paper."""
+class DatasetLink(BaseModel):
+    """A dataset mentioned in the paper."""
+    name: str = Field(description="Name of the dataset (e.g. 'european-space-agency-anomaly-dataset').")
+    description: str = Field(description="Description of the dataset and how it is used in the paper.")
+
+
+class PaperAnalysisBase(BaseModel):
+    """Structured extraction output for a single academic paper (Base schema)."""
 
     title: str = Field(description="Clean, full title of the paper.")
     authors: list[str] = Field(description="List of author full names.")
@@ -148,13 +154,6 @@ class PaperAnalysis(BaseModel):
         description="Bullet-point list of 3-5 key contributions."
     )
 
-    tags: list[str] = Field(
-        description=(
-            "Relevant tags from the provided taxonomy. "
-            "Choose only tags that genuinely apply. Maximum 10 tags."
-        )
-    )
-
     architectures: list[str] = Field(
         default_factory=list,
         description=(
@@ -163,7 +162,7 @@ class PaperAnalysis(BaseModel):
         ),
     )
 
-    datasets: list[str] = Field(
+    datasets: list[DatasetLink] = Field(
         default_factory=list,
         description=(
             "Names of CRITICAL datasets used for evaluation or training "
@@ -209,35 +208,32 @@ class PaperAnalysis(BaseModel):
     )
 
 
+class PaperAnalysis(PaperAnalysisBase):
+    """Structured extraction output for a single academic paper."""
+    tags: list[str] = Field(default_factory=list, description="All assigned tags.")
+
+
 # ---------------------------------------------------------------------------
 # Agent factory
 # ---------------------------------------------------------------------------
 
 def _build_system_prompt(skill: SkillSpec, taxonomy: dict) -> str:
-    tags_list = ", ".join(taxonomy.get("tags", []))
     architectures_list = ", ".join(taxonomy.get("architectures", []))
     domains_list = ", ".join(taxonomy.get("domains", []))
     
-    existing_concepts_list = get_existing_concept_slugs()
-    existing_concepts_str = ", ".join(existing_concepts_list) if existing_concepts_list else "(No concepts currently exist in the vault)"
-
     base_prompt = f"""You are an expert academic paper analyst specialising in machine learning research.
 
 Your task is to analyse the provided paper abstract and extract structured information.
 
-TAXONOMY (use ONLY these values for tags/architectures/domains):
-- Available tags: {tags_list}
+TAXONOMY (use ONLY these values for architectures/domains):
 - Available architectures: {architectures_list}
 - Available domains: {domains_list}
 
-EXISTING CONCEPTS IN VAULT:
-{existing_concepts_str}
-
 SKILL CONTEXT ({skill.name}):
 {skill.extra_system_prompt}
 
 IMPORTANT RULES:
-1. Select tags ONLY from the taxonomy tag list above.
+1. Select tags according to the dynamic schema.
 2. Select domain ONLY from the taxonomy domains list above.
 3. Select architectures ONLY from the taxonomy architectures list above (or leave empty).
 4. The summary must be 3-5 sentences, technically precise but accessible.
@@ -246,11 +242,10 @@ IMPORTANT RULES:
 7. Default to zero concepts unless a concept is central to the paper's novelty and likely to matter beyond this single paper.
 8. For each proposed concept, fill importance_reason, reusability_reason, and evidence_excerpt with concrete, specific justification from the title or abstract.
 9. Reject generic terminology, routine datasets, benchmark names, and umbrella phrases unless the paper makes them a distinct reusable idea.
-10. For concepts, ALWAYS check the EXISTING CONCEPTS IN VAULT list. If a highly synonymous concept already exists, you MUST exactly reuse its slug. Only create entirely new concepts if they represent a genuinely novel idea.
-11. Prefer 0-2 excellent concept candidates over 5 weak ones.
-12. Datasets are also candidate proposals for archival notes. Include only critical, named datasets that are central to evaluation claims.
-13. Reject generic or aggregate dataset labels like "real-world datasets", "benchmark datasets", "10 datasets", or unnamed proprietary buckets.
-14. Prefer 0-2 datasets. If uncertain a dataset is important enough for a standalone note, omit it.
+10. Prefer 0-2 excellent concept candidates over 5 weak ones.
+11. Datasets are also candidate proposals for archival notes. Include only critical, named datasets that are central to evaluation claims.
+12. Reject generic or aggregate dataset labels like "real-world datasets", "benchmark datasets", "10 datasets", or unnamed proprietary buckets.
+13. Prefer 0-2 datasets. If uncertain a dataset is important enough for a standalone note, omit it.
 """
     if skill.analyst_system_prompt_override:
         return skill.analyst_system_prompt_override
@@ -274,10 +269,45 @@ def analyse(paper: RawPaper, skill: SkillSpec) -> PaperAnalysis:
     """
     taxonomy = _load_taxonomy()
     system_prompt = _build_system_prompt(skill, taxonomy)
+    existing_tags = get_existing_tags()
+    
+    from pydantic import create_model
+
+    fields = {}
+    if existing_tags:
+        ExistingTags = Literal[tuple(existing_tags)]  # type: ignore
+        fields["existing_tags"] = (
+            list[ExistingTags],
+            Field(
+                default_factory=list,
+                description="Tags chosen from the existing knowledge base. Pick only tags that genuinely apply."
+            )
+        )
+        fields["new_tags"] = (
+            list[str],
+            Field(
+                default_factory=list,
+                description="Use this ONLY if the concept cannot be accurately described by any of the existing_tags. Keep new tags to an absolute minimum."
+            )
+        )
+    else:
+        fields["new_tags"] = (
+            list[str],
+            Field(
+                default_factory=list,
+                description="List of new tags for this paper."
+            )
+        )
+        
+    DynamicPaperAnalysis = create_model(
+        "DynamicPaperAnalysis",
+        __base__=PaperAnalysisBase,
+        **fields
+    )
 
-    agent: Agent[None, PaperAnalysis] = Agent(
+    agent: Agent[None, DynamicPaperAnalysis] = Agent(
         model=settings.llm_model,
-        output_type=PaperAnalysis,
+        output_type=DynamicPaperAnalysis,
         system_prompt=system_prompt,
     )
 
@@ -293,7 +323,17 @@ def analyse(paper: RawPaper, skill: SkillSpec) -> PaperAnalysis:
     logger.info("Analyst: analysing paper %s with skill %s", paper.arxiv_id, skill.name)
 
     result = agent.run_sync(user_message)
-    analysis: PaperAnalysis = result.output
+    dynamic_analysis = result.output
+    
+    # Merge existing and new tags
+    combined_tags = []
+    if hasattr(dynamic_analysis, "existing_tags"):
+        combined_tags.extend(getattr(dynamic_analysis, "existing_tags", []))
+    if hasattr(dynamic_analysis, "new_tags"):
+        combined_tags.extend(getattr(dynamic_analysis, "new_tags", []))
+        
+    analysis_dict = dynamic_analysis.model_dump(exclude={"existing_tags", "new_tags"})
+    analysis = PaperAnalysis(**analysis_dict, tags=combined_tags)
 
     # Ensure metadata from the raw paper is preserved accurately
     analysis.arxiv_id = paper.arxiv_id

{swarm_notes-0.2.0 → swarm_notes-0.3.0}/src/swarm_notes/critic.py

@@ -9,10 +9,10 @@ from pathlib import Path
 from pydantic import BaseModel, Field
 from pydantic_ai import Agent
 
-from swarm_notes.analyst import ConceptLink, OpenQuestion, PaperAnalysis, RejectedCandidate
+from swarm_notes.analyst import ConceptLink, DatasetLink, OpenQuestion, PaperAnalysis, RejectedCandidate
 from swarm_notes.config import settings
 from swarm_notes.router import SkillSpec
-from swarm_notes.vault_manager import get_existing_concept_slugs
+from swarm_notes.vault_manager import get_existing_entities
 from swarm_notes.watcher import RawPaper
 
 logger = logging.getLogger(__name__)
@@ -44,7 +44,7 @@ class CriticReview(BaseModel):
             "Return an empty list if none qualify."
         ),
     )
-    approved_datasets: list[str] = Field(
+    approved_datasets: list[DatasetLink] = Field(
         default_factory=list,
         description=(
             "Only critical named datasets that deserve standalone notes. "
@@ -62,11 +62,11 @@ class CriticReview(BaseModel):
 
 
 def _build_system_prompt(skill: SkillSpec) -> str:
-    existing_concepts = get_existing_concept_slugs()
+    existing_concepts = get_existing_entities("concepts")
     existing_concepts_str = ", ".join(existing_concepts) if existing_concepts else "(No concepts currently exist in the vault)"
-    existing_open_questions = _get_existing_slugs(settings.vault_open_questions_dir)
+    existing_open_questions = get_existing_entities("open_questions")
     existing_open_questions_str = ", ".join(existing_open_questions) if existing_open_questions else "(No open questions currently exist in the vault)"
-    existing_datasets = _get_existing_slugs(settings.vault_datasets_dir)
+    existing_datasets = get_existing_entities("datasets")
     existing_datasets_str = ", ".join(existing_datasets) if existing_datasets else "(No datasets currently exist in the vault)"
 
     prompt = f"""You are a highly selective research archivist protecting a long-lived ML knowledge vault.
@@ -167,9 +167,3 @@ def review_analysis(analysis: PaperAnalysis, paper: RawPaper, skill: SkillSpec)
 
     return analysis
 
-
-def _get_existing_slugs(directory: Path) -> list[str]:
-    """Return existing note slugs from a vault directory."""
-    if not directory.exists():
-        return []
-    return sorted(path.stem for path in directory.glob("*.md"))

{swarm_notes-0.2.0 → swarm_notes-0.3.0}/src/swarm_notes/vault_manager.py

@@ -12,6 +12,8 @@ import logging
 import shutil
 from pathlib import Path
 
+import frontmatter
+
 from swarm_notes.config import settings
 
 logger = logging.getLogger(__name__)
@@ -86,11 +88,42 @@ def _merge_directory(src: Path, dst: Path) -> None:
             logger.debug("Staged %s → %s", src_file, dst_file)
 
 
-def get_existing_concept_slugs() -> list[str]:
-    """Return a list of all concept slugs currently in the live vault."""
-    if not settings.vault_concepts_dir.exists():
+def get_existing_tags() -> list[str]:
+    """Return a deduplicated list of all tags currently found in live vault papers."""
+    if not settings.vault_papers_dir.exists():
+        return []
+    tags_set = set()
+    for f in settings.vault_papers_dir.glob("*.md"):
+        try:
+            post = frontmatter.load(f)
+            file_tags = post.metadata.get("tags", [])
+            if isinstance(file_tags, list):
+                for t in file_tags:
+                    if isinstance(t, str) and t.strip():
+                        tags_set.add(t.strip())
+            elif isinstance(file_tags, str) and file_tags.strip():
+                tags_set.add(file_tags.strip())
+        except Exception as exc:
+            logger.warning("VaultManager: failed to parse tags from %s: %s", f, exc)
+    return sorted(list(tags_set))
+
+
+def get_existing_entities(entity_type: str) -> list[str]:
+    """Return a deduplicated list of entity names (e.g. concepts, datasets, open_questions) from live vault papers."""
+    if not settings.vault_papers_dir.exists():
         return []
-    return [f.stem for f in settings.vault_concepts_dir.glob("*.md")]
+    entity_set = set()
+    for f in settings.vault_papers_dir.glob("*.md"):
+        try:
+            post = frontmatter.load(f)
+            entities = post.metadata.get(entity_type, [])
+            if isinstance(entities, list):
+                for e in entities:
+                    if isinstance(e, dict) and "name" in e:
+                        entity_set.add(e["name"])
+        except Exception as exc:
+            pass
+    return sorted(list(entity_set))
 
 
 def make_storage_id(source: str, paper_id: str) -> str:

{swarm_notes-0.2.0 → swarm_notes-0.3.0}/src/swarm_notes/vault_writer.py

@@ -23,29 +23,35 @@ logger = logging.getLogger(__name__)
 # YAML frontmatter template
 _FRONTMATTER_TEMPLATE = """\
 ---
-# CSL-compatible fields
+# OKF Required Field
+type: "paper"
+
+# OKF Recommended Core Fields
 title: "{title}"
+description: "{description}"
+resource: "{url}"
+tags:
+{tag_lines}
+timestamp: "{processed_at}"
+
+# CSL-compatible fields (Retained as custom)
 author:
 {author_lines}
 issued:
   date-parts:
     - [{date_parts}]
-url: "{url}"
 
-# Custom fields
+# Domain-specific custom fields
 paper_id: "{paper_id}"
 paper_source: "{paper_source}"
-domain: "{domain}"
-tags:
-{tag_lines}
 architectures:
 {architecture_lines}
+concepts:
+{concept_lines}
 datasets:
 {dataset_lines}
-concept_slugs:
-{concept_slug_lines}
-dataset_slugs:
-{dataset_slug_lines}
+open_questions:
+{open_question_lines}
 skill: "{skill}"
 processed_at: "{processed_at}"
 created_at: "{created_at}"
@@ -78,21 +84,6 @@ def write_paper(analysis: PaperAnalysis, skill_name: str) -> Path:
     out_path.write_text(content, encoding="utf-8")
     logger.info("VaultWriter: wrote paper → %s", out_path)
 
-    # Upsert concept notes and track this paper as a source mention.
-    for concept in analysis.concepts:
-        _write_concept_stub(
-            concept.slug,
-            concept.display_name,
-            concept.one_liner,
-            concept.importance_reason,
-            concept.evidence_excerpt,
-            slug,
-        )
-
-    # Ensure dataset notes exist for linked dataset slugs.
-    for dataset_name in analysis.datasets:
-        _write_dataset_stub(_slugify(dataset_name), dataset_name)
-
     return out_path
 
 
@@ -115,25 +106,41 @@ def _build_frontmatter(analysis: PaperAnalysis, skill_name: str) -> str:
     architecture_lines = (
         "\n".join(f'  - "{a}"' for a in analysis.architectures) or "  []"
     )
-    dataset_lines = "\n".join(f'  - "{d}"' for d in analysis.datasets) or "  []"
-    concept_slug_lines = "\n".join(f'  - "{c.slug}"' for c in analysis.concepts) or "  []"
-    dataset_slug_lines = "\n".join(f'  - "{_slugify(d)}"' for d in analysis.datasets) or "  []"
+    
+    def _render_okf_list(items) -> str:
+        if not items:
+            return "  []"
+        lines = []
+        for item in items:
+            # Handle ConceptLink (slug/one_liner), DatasetLink (name/description), OpenQuestion (slug/description)
+            name = getattr(item, "slug", getattr(item, "name", ""))
+            desc = getattr(item, "one_liner", getattr(item, "description", ""))
+            lines.append(f'  - name: "{_yaml_escape(name)}"\n    description: "{_yaml_escape(desc)}"')
+        return "\n".join(lines)
+
+    concept_lines = _render_okf_list(analysis.concepts)
+    dataset_lines = _render_okf_list(analysis.datasets)
+    open_question_lines = _render_okf_list(analysis.open_questions)
 
     now_utc = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    
+    # Escape summary for description field (strip newlines which might break YAML block if not multiline string)
+    safe_description = _yaml_escape(analysis.summary.replace('\n', ' '))
 
     return _FRONTMATTER_TEMPLATE.format(
         title=_yaml_escape(analysis.title),
+        description=safe_description,
+        url=analysis.url,
         author_lines=author_lines,
         date_parts=date_parts,
-        url=analysis.url,
         paper_id=analysis.arxiv_id,
         paper_source=analysis.source,
         domain=analysis.domain,
         tag_lines=tag_lines,
         architecture_lines=architecture_lines,
+        concept_lines=concept_lines,
         dataset_lines=dataset_lines,
-        concept_slug_lines=concept_slug_lines,
-        dataset_slug_lines=dataset_slug_lines,
+        open_question_lines=open_question_lines,
         skill=skill_name,
         processed_at=now_utc,
         created_at=now_utc,
@@ -165,7 +172,6 @@ def _build_body(analysis: PaperAnalysis) -> str:
         lines.append("## Open Questions & Future Work\n")
         for q in analysis.open_questions:
             lines.append(f"- [[{q.slug}]]")
-            _create_open_question(q, analysis)
         lines.append("")
 
     # Concepts
@@ -211,7 +217,7 @@ def _build_body(analysis: PaperAnalysis) -> str:
     if analysis.datasets:
         lines.append("## Datasets\n")
         for ds in analysis.datasets:
-            ds_slug = _slugify(ds)
+            ds_slug = getattr(ds, "name", "unknown")
             lines.append(f"- [[{ds_slug}]]")
         lines.append("")
 
@@ -224,175 +230,11 @@ def _build_body(analysis: PaperAnalysis) -> str:
     return "\n".join(lines)
 
 
-def _write_concept_stub(
-    slug: str,
-    display_name: str,
-    one_liner: str,
-    importance_reason: str = "",
-    evidence_excerpt: str = "",
-    paper_slug: str = "",
-) -> None:
-    """Create or update a concept note and track source papers in metadata/content."""
-    staging_path = settings.tmp_concepts_dir / f"{slug}.md"
-    vault_path = settings.vault_concepts_dir / f"{slug}.md"
-    paper_ref = f"[[{paper_slug}]]" if paper_slug else ""
-
-    if vault_path.exists() and not staging_path.exists():
-        shutil.copy2(vault_path, staging_path)
-
-    now_utc = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if staging_path.exists():
-        try:
-            post = frontmatter.load(staging_path)
-            source_papers = post.metadata.get("source_papers", [])
-            if not isinstance(source_papers, list):
-                source_papers = [source_papers]
-            if paper_ref and paper_ref not in source_papers:
-                source_papers.append(paper_ref)
-
-            post.metadata["source_papers"] = source_papers
-            post.metadata["modified_at"] = now_utc
-            post.content = _ensure_related_paper_link(post.content, paper_ref)
-
-            with staging_path.open("wb") as f:
-                frontmatter.dump(post, f)
-
-            logger.info("VaultWriter: updated concept note → %s", staging_path)
-            return
-        except Exception as exc:
-            logger.warning("VaultWriter: Failed to update concept frontmatter for %s: %s", slug, exc)
-
-    source_papers = [paper_ref] if paper_ref else []
-    content = f"""\
----
-title: "{_yaml_escape(display_name)}"
-slug: "{slug}"
-type: concept
-generated_stub: true
-source_papers:
-{_yaml_list(source_papers)}
-processed_at: "{now_utc}"
-created_at: "{now_utc}"
----
-
-# {display_name}
-
-> *Auto-generated stub. Edit this file to add more details.*
-
-{one_liner}
-
-"""
-    if importance_reason:
-        content += f"\n## Why It Matters\n\n{importance_reason}\n"
-    if evidence_excerpt:
-        content += f"\n## Evidence\n\n> {evidence_excerpt}\n"
-    content += "\n## Related Papers\n\n"
-    if paper_ref:
-        content += f"- {paper_ref}\n"
-
-    staging_path.write_text(content, encoding="utf-8")
-    logger.info("VaultWriter: created concept stub → %s", staging_path)
-
-
-def _ensure_related_paper_link(content: str, paper_ref: str) -> str:
-    """Ensure a concept body has a Related Papers section containing *paper_ref*."""
-    if not paper_ref:
-        return content
-    if paper_ref in content:
-        return content
-    trimmed = content.rstrip()
-    if "## Related Papers" in trimmed:
-        return f"{trimmed}\n- {paper_ref}\n"
-    return f"{trimmed}\n\n## Related Papers\n\n- {paper_ref}\n"
-
-
-def _write_dataset_stub(slug: str, display_name: str) -> None:
-    """Write a dataset stub note if one does not already exist."""
-    staging_path = settings.tmp_datasets_dir / f"{slug}.md"
-    vault_path = settings.vault_datasets_dir / f"{slug}.md"
-
-    if staging_path.exists() or vault_path.exists():
-        logger.debug("Dataset stub already exists for '%s' - skipping", slug)
-        return
-
-    now_utc = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-    content = f"""\
----
-title: "{_yaml_escape(display_name)}"
-slug: "{slug}"
-type: dataset
-generated_stub: true
-processed_at: "{now_utc}"
-created_at: "{now_utc}"
----
-
-# {display_name}
-
-> *Auto-generated dataset stub. Edit this file to add more details.*
-"""
-    staging_path.write_text(content, encoding="utf-8")
-    logger.info("VaultWriter: created dataset stub -> %s", staging_path)
-
-
 def _yaml_list(values: list[str]) -> str:
     """Render a YAML list with sane fallback for empty lists."""
     return "\n".join(f"  - \"{_yaml_escape(v)}\"" for v in values) if values else "  []"
 
 
-def _create_open_question(question: OpenQuestion, analysis: PaperAnalysis) -> None:
-    """Create or update an open question file with frontmatter tracking sources."""
-    slug = _slugify(question.slug)
-    filename = f"{slug}.md"
-    live_path = settings.vault_open_questions_dir / filename
-    staging_path = settings.tmp_open_questions_dir / filename
-
-    is_new = not staging_path.exists() and not live_path.exists()
-
-    if live_path.exists() and not staging_path.exists():
-        shutil.copy2(live_path, staging_path)
-
-    now_utc = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-    papers = []
-    content = f"**Background:** {question.background}\n\n**Question / Future Work:** {question.description}"
-    if question.importance_reason:
-        content += f"\n\n**Why It Matters:** {question.importance_reason}"
-    if question.evidence_excerpt:
-        content += f"\n\n**Evidence:** {question.evidence_excerpt}"
-    existing_created_at: str | None = None
-
-    if staging_path.exists():
-        try:
-            post = frontmatter.load(staging_path)
-            papers = post.metadata.get("source_papers", [])
-            if not isinstance(papers, list):
-                papers = [papers]
-            # Retain original deeper description instead of wiping it
-            content = post.content if post.content.strip() else content
-            existing_created_at = post.metadata.get("created_at")
-        except Exception as exc:
-            logger.warning("VaultWriter: Failed to parse existing frontmatter for %s: %s", slug, exc)
-
-    paper_slug = _make_slug(analysis.source, analysis.arxiv_id, analysis.title)
-    paper_id = f"[[{paper_slug}]]"
-    if paper_id not in papers:
-        papers.append(paper_id)
-
-    metadata: dict = {
-        "title": question.title,
-        "source_papers": papers,
-        "created_at": existing_created_at if existing_created_at else now_utc,
-    }
-    if not is_new:
-        metadata["modified_at"] = now_utc
-
-    post = frontmatter.Post(content, **metadata)
-    with staging_path.open("wb") as f:
-        frontmatter.dump(post, f)
-
-    logger.info("VaultWriter: tracked open question → %s", staging_path)
-
-
 # ---------------------------------------------------------------------------
 # Public feed maintenance
 # ---------------------------------------------------------------------------
@@ -501,7 +343,7 @@ def append_daily_discussion(content: str) -> None:
         try:
             post = frontmatter.load(staging_path)
             post.metadata["modified_at"] = now_utc
-            with staging_path.open("wb") as f:
+            with staging_path.open("w", encoding="utf-8") as f:
                 frontmatter.dump(post, f)
         except Exception as exc:
             logger.warning("VaultWriter: could not update modified_at for %s: %s", filename, exc)
@@ -724,6 +566,18 @@ def _analysis_to_feed_entry(analysis: PaperAnalysis) -> dict:
         "url": analysis.url,
         "domain": analysis.domain,
         "tags": analysis.tags,
+        "concepts": [
+            {"name": getattr(c, "slug", getattr(c, "name", "")), "description": getattr(c, "one_liner", getattr(c, "description", ""))}
+            for c in analysis.concepts
+        ],
+        "datasets": [
+            {"name": getattr(d, "name", ""), "description": getattr(d, "description", "")}
+            for d in analysis.datasets
+        ],
+        "open_questions": [
+            {"name": getattr(q, "slug", getattr(q, "name", "")), "description": getattr(q, "description", "")}
+            for q in analysis.open_questions
+        ],
         "summary": analysis.summary,
         "processed_at": datetime.now(tz=timezone.utc).isoformat(),
     }