feedsummary-core 2.2.0__tar.gz → 2.2.1__tar.gz

{feedsummary_core-2.2.0/src/feedsummary_core.egg-info → feedsummary_core-2.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feedsummary-core
-Version: 2.2.0
+Version: 2.2.1
 Summary: Core library for FeedSummary.
 Author: Martin Vesterlund
 License-Expression: BSD-3-Clause

{feedsummary_core-2.2.0 → feedsummary_core-2.2.1}/TAGGING_QUICKSTART.md

@@ -167,7 +167,7 @@ Edit `tagging.py` to modify patterns:
 
 ```python
 # Add CVE variants
-CVE_PATTERN = re.compile(r'CVE-\d{4}-\d{4,5}|GHSA-\w{4}-\w{4}-\w{4}')
+CVE_PATTERN = re.compile(r'CVE-[0-9]{4}-[0-9]{4,19}|GHSA-\w{4}-\w{4}-\w{4}')
 
 # Add threat actor keywords
 THREAT_ACTOR_KEYWORDS = {

{feedsummary_core-2.2.0 → feedsummary_core-2.2.1}/TAGGING_SYSTEM.md

@@ -86,28 +86,26 @@ CREATE TABLE article_tags (
 
 ## Taggkategorier
 
-### GENERAL (Standard)
-- Normala ämnestaggar
-- Prioriteras från befintliga taggar
-- Får inte skapas nya automatiskt (om de redan finns)
-
-### DOMAIN_ENTITY (Domänspecifik)
-Undantag där nya taggar ALLTID tillåts:
-- **CVE:er**: `CVE-2024-1234` mönster
-- **Hot-aktörer**: APT-grupper, kampanjnamn (t.ex. "APT28", "Lazarus Group")
-- **Geografiska regioner**: Länder, kontinenter (t.ex. "Russia", "North Korea")
-- **Sårbarheter**: Specifika sårbarhetsnamn och typer
+Kategorier hämtas dynamiskt från databasens `tag_categories`. LLM:en får
+kategoriernas namn, etikett och beskrivning och väljer den mest specifika
+lämpliga kategorin för varje ny tagg. Valet valideras case-insensitivt mot
+databasen och lagras med kategorins kanoniska namn.
+
+Om LLM:en inte anger en giltig kategori används en definierad standardkategori,
+normalt `GENERAL` eller `DOMAIN_ENTITY`. CVE-taggar är ett särskilt undantag och
+tvingas alltid till `VULNERABILITY`.
 
 ## Taggprioriteringsalgoritm
 
 ```
-1. Extrahera kandidattaggar från artikel (via LLM)
-2. För varje kandidattagg:
+1. Hämta tillgängliga kategorier från databasen
+2. Extrahera kandidattaggar och kategori från artikel (via LLM)
+3. För varje kandidattagg:
    a. Sök efter befintliga liknande taggar
    b. Om befintlig tagg finns → använd den
-   c. Om domänspecifik entitet → skapa ny DOMAIN_ENTITY tagg
-   d. Annars → hoppa över (föredra allmän framför specifik)
-3. Lagra valda taggar i databasen
+   c. Om en ny tagg ska skapas → validera LLM-kategorin mot databasen
+   d. Skapa taggen med databasens kanoniska kategorinamn
+4. Lagra valda taggar i databasen
 ```
 
 ### Likhetsmätning
@@ -215,23 +213,23 @@ print(f"Removed {removed} unused tags")
 Systemet använder följande prompt-mall för att få LLM att extrahera taggar:
 
 ```
-Analyze the following article and extract up to {max_tags} relevant tags.
+Analyze the following article and extract up to {max_tags} relevant non-CVE tags.
 
 IMPORTANT RULES:
-1. Prefer existing, general tags over creating new, specific tags
-2. CVE numbers (e.g., CVE-2024-1234), threat actors, regions, and 
-   vulnerability names are EXCEPTIONS - these should be new tags
-3. Return tags in lowercase
-4. Keep tags concise (1-3 words max)
-5. Focus on the main topics and entities mentioned
+1. Select "category" from the database categories included in the prompt
+2. Use the most specific suitable category and GENERAL as fallback
+3. Include every CVE; CVE tags do not count toward max_tags
+4. Return tags in lowercase and keep them concise
 
 Article:
 {article_text}
 
 Respond in JSON format:
 {
-    "tags": ["tag1", "tag2", "tag3"],
-    "reasoning": "Brief explanation of why these tags were chosen"
+    "tags": [
+        {"tag": "tag1", "type": "NAMED_ENTITY", "category": "ORGANIZATION",
+         "reasoning": "Why this tag and category are relevant"}
+    ]
 }
 ```
 
@@ -241,7 +239,7 @@ Systemet extraherar sedan JSON från svar och tillämpar prioriteringslogiken.
 
 Systemet identifierar domänspecifika entiteter genom:
 
-1. **Regex-mönster**: CVE format (`CVE-\d{4}-\d{4,5}`)
+1. **Regex-mönster**: CVE format (`CVE-[0-9]{4}-[0-9]{4,19}`)
 2. **Nyckelordsmatchning**:
    - Hot-aktörer: "APT", "group", "threat actor", "collective"
    - Regioner: Länder, kontinenter, geografiska termer
@@ -363,7 +361,7 @@ similar_tags = tag_manager._find_similar_existing_tags(
 Redigera regex-mönster och nyckelord i `tagging.py`:
 
 ```python
-CVE_PATTERN = re.compile(r'CVE-\d{4}-\d{4,5}', re.IGNORECASE)
+CVE_PATTERN = re.compile(r'CVE-[0-9]{4}-[0-9]{4,19}', re.IGNORECASE)
 THREAT_ACTOR_KEYWORDS = {'APT', 'group', ...}  # Lägg till fler ord
 ```
 

{feedsummary_core-2.2.0 → feedsummary_core-2.2.1}/src/feedsummary_core/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '2.2.0'
-__version_tuple__ = version_tuple = (2, 2, 0)
+__version__ = version = '2.2.1'
+__version_tuple__ = version_tuple = (2, 2, 1)
 
-__commit_id__ = commit_id = 'g03a56f8f4'
+__commit_id__ = commit_id = 'g4ab927c63'

{feedsummary_core-2.2.0 → feedsummary_core-2.2.1}/src/feedsummary_core/summarizer/tagging.py

@@ -64,6 +64,7 @@ from feedsummary_core.summarizer.batching import (
 from feedsummary_core.tagging_rules import (
     CVE_PATTERN,
     VULNERABILITY_TAG_CATEGORY,
+    extract_cve_ids,
     is_cve_tag,
 )
 
@@ -164,7 +165,7 @@ class TagManager:
 
         Args:
             name: Tag name (normalized to lowercase)
-            category: Tag category (GENERAL, DOMAIN_ENTITY, or VULNERABILITY)
+            category: Name of a database-defined tag category
             description: Optional description
 
         Returns:
@@ -182,13 +183,16 @@ class TagManager:
         if is_cve_tag(name):
             category = self.TAG_CATEGORY_VULNERABILITY
 
-        # Validate category
-        if category not in (
-            self.TAG_CATEGORY_GENERAL,
-            self.TAG_CATEGORY_DOMAIN_ENTITY,
-            self.TAG_CATEGORY_VULNERABILITY,
-        ):
-            category = self.TAG_CATEGORY_GENERAL
+        # Non-CVE categories must resolve to a category defined by the active
+        # database. Stores without category support retain the legacy defaults.
+        if not is_cve_tag(name):
+            resolved_category = self._resolve_tag_category(category)
+            if resolved_category is None:
+                logger.debug(
+                    f"[TagValidate] Rejected tag '{name}': unknown category {category!r}"
+                )
+                return None
+            category = resolved_category
 
         # Validate tag name length (max 2 words for GENERAL, unlimited otherwise)
         if not self._is_valid_tag_name(name, category):
@@ -236,6 +240,57 @@ class TagManager:
             logger.error(f"Error getting all tags: {e}")
         return []
 
+    def get_all_categories(self) -> List[Dict[str, Any]]:
+        """Get all tag categories defined by the active store."""
+        try:
+            fn = getattr(self.store, "get_all_categories", None)
+            if callable(fn):
+                categories = fn()
+                if isinstance(categories, list):
+                    return [category for category in categories if isinstance(category, dict)]
+        except Exception as e:
+            logger.error(f"Error getting tag categories: {e}")
+        return []
+
+    def _resolve_tag_category(
+        self,
+        proposed_category: Any,
+        fallback_category: str = TAG_CATEGORY_GENERAL,
+        categories: Optional[List[Dict[str, Any]]] = None,
+    ) -> Optional[str]:
+        """Resolve a proposed category to its canonical database-defined name."""
+        available = self.get_all_categories() if categories is None else categories
+        if not available:
+            legacy_categories = {
+                self.TAG_CATEGORY_GENERAL,
+                self.TAG_CATEGORY_DOMAIN_ENTITY,
+                self.TAG_CATEGORY_VULNERABILITY,
+            }
+            proposed = str(proposed_category or "").strip().upper()
+            if proposed in legacy_categories:
+                return proposed
+            if fallback_category in legacy_categories:
+                return fallback_category
+            return self.TAG_CATEGORY_GENERAL
+
+        category_names = {
+            str(category.get("name") or "").strip().casefold():
+            str(category.get("name") or "").strip()
+            for category in available
+            if str(category.get("name") or "").strip()
+        }
+        for candidate in (proposed_category, fallback_category, self.TAG_CATEGORY_GENERAL):
+            normalized = str(candidate or "").strip().casefold()
+            if normalized in category_names:
+                return category_names[normalized]
+
+        logger.warning(
+            "No database-defined tag category matches proposed=%r or fallback=%r",
+            proposed_category,
+            fallback_category,
+        )
+        return None
+
     def _ensure_cve_category(self, tag: Dict[str, Any]) -> Dict[str, Any]:
         """Persist and return the canonical category for an existing CVE tag."""
         name = str(tag.get("name") or "").strip()
@@ -883,6 +938,13 @@ class TagManager:
         """
         selected_tags: List[Dict[str, Any]] = []
         processed_tags: Set[str] = set()
+        available_categories = self.get_all_categories()
+        available_category_names = {
+            str(category.get("name") or "").strip().casefold():
+            str(category.get("name") or "").strip()
+            for category in available_categories
+            if str(category.get("name") or "").strip()
+        }
 
         for candidate in candidate_tags:
             if not candidate:
@@ -892,10 +954,12 @@ class TagManager:
             if isinstance(candidate, dict):
                 tag_name = candidate.get("name", "").strip().lower()
                 tag_type = candidate.get("type", "CATEGORY").upper()
+                proposed_category = candidate.get("category")
                 tag_reasoning = candidate.get("reasoning", "")
             else:
                 tag_name = str(candidate).strip().lower()
                 tag_type = "CATEGORY"  # Default for backward compatibility
+                proposed_category = None
                 tag_reasoning = ""
             
             if not tag_name or tag_name in processed_tags:
@@ -915,6 +979,19 @@ class TagManager:
             elif allow_new_tags:
                 # Decision logic based on tag type
                 should_create = False
+                default_category = (
+                    self.TAG_CATEGORY_DOMAIN_ENTITY
+                    if tag_type == "NAMED_ENTITY"
+                    else self.TAG_CATEGORY_GENERAL
+                )
+                explicit_category = available_category_names.get(
+                    str(proposed_category or "").strip().casefold()
+                )
+                resolved_category = self._resolve_tag_category(
+                    proposed_category,
+                    fallback_category=default_category,
+                    categories=available_categories,
+                )
                 
                 if tag_type == "NAMED_ENTITY":
                     # NAMED_ENTITY tags (organizations, groups, actors) are always created
@@ -923,17 +1000,24 @@ class TagManager:
                 elif tag_type == "CATEGORY":
                     # CATEGORY tags only created if they match our keyword heuristics
                     should_create = self._is_relevant_new_tag(tag_name)
+
+                # An explicit valid database category is enough evidence to create
+                # a new tag even if the legacy keyword heuristic does not know it.
+                if explicit_category:
+                    should_create = True
                 
                 if should_create:
                     # Validate tag name (max 2 words for GENERAL, unlimited otherwise)
                     if is_cve_tag(tag_name):
                         tag_category = self.TAG_CATEGORY_VULNERABILITY
                     else:
-                        tag_category = (
-                            self.TAG_CATEGORY_DOMAIN_ENTITY
-                            if tag_type == "NAMED_ENTITY"
-                            else self.TAG_CATEGORY_GENERAL
+                        tag_category = resolved_category
+
+                    if tag_category is None:
+                        logger.debug(
+                            f"[TagSelect] Rejected tag '{tag_name}': no valid database category"
                         )
+                        continue
                     
                     if not self._is_valid_tag_name(tag_name, tag_category):
                         logger.debug(f"[TagSelect] Rejected tag '{tag_name}': exceeds word limit for {tag_type}")
@@ -996,8 +1080,8 @@ class TagManager:
         Expects JSON format with per-tag reasoning:
         {
             "tags": [
-                {"tag": "tag1", "type": "NAMED_ENTITY", "reasoning": "Why this tag..."},
-                {"tag": "tag2", "type": "CATEGORY", "reasoning": "Why this tag..."},
+                {"tag": "tag1", "type": "NAMED_ENTITY", "category": "ORGANIZATION", "reasoning": "Why this tag..."},
+                {"tag": "tag2", "type": "CATEGORY", "category": "THREAT", "reasoning": "Why this tag..."},
                 ...
             ]
         }
@@ -1022,7 +1106,8 @@ class TagManager:
             response: LLM response text
 
         Returns:
-            List of tags as dicts with 'name', 'type', and optional 'reasoning' keys.
+            List of tags as dicts with 'name', 'type', 'category', and optional
+            'reasoning' keys.
             If reasoning is not provided, defaults to empty string.
         """
         if not response or not isinstance(response, str):
@@ -1053,11 +1138,13 @@ class TagManager:
                             if isinstance(obj, dict):
                                 tag_name = obj.get("tag") or obj.get("name")
                                 tag_type = obj.get("type", "CATEGORY")
+                                tag_category = obj.get("category")
                                 tag_reasoning = obj.get("reasoning", "")
                                 if tag_name:
                                     tags.append({
                                         "name": str(tag_name).strip().lower(),
                                         "type": tag_type.upper() if tag_type else "CATEGORY",
+                                        "category": str(tag_category).strip() if tag_category else "",
                                         "reasoning": str(tag_reasoning).strip() if tag_reasoning else ""
                                     })
                         except Exception:
@@ -1094,11 +1181,13 @@ class TagManager:
                                 if isinstance(t, dict):
                                     tag_name = t.get("tag") or t.get("name")
                                     tag_type = t.get("type", "CATEGORY")
+                                    tag_category = t.get("category")
                                     tag_reasoning = t.get("reasoning", "")
                                     if tag_name:
                                         tags.append({
                                             "name": str(tag_name).strip().lower(),
                                             "type": tag_type.upper() if tag_type else "CATEGORY",
+                                            "category": str(tag_category).strip() if tag_category else "",
                                             "reasoning": str(tag_reasoning).strip() if tag_reasoning else ""
                                         })
                                 elif isinstance(t, str):
@@ -1129,7 +1218,7 @@ class TagManager:
             llm_client: LLM client for generating tags
             article: Article dict with 'title', 'content', etc.
             config: Configuration dict
-            max_tags: Maximum number of tags to select
+            max_tags: Maximum number of non-CVE tags to select
 
         Returns:
             List of selected tags
@@ -1146,8 +1235,26 @@ class TagManager:
         if not text_context:
             return []
 
+        # CVE identifiers are deterministic domain entities. Extract them directly
+        # instead of relying on the LLM to include every identifier in its response.
+        cve_candidates = [
+            {
+                "name": cve_id,
+                "type": "NAMED_ENTITY",
+                "category": self.TAG_CATEGORY_VULNERABILITY,
+                "reasoning": "CVE identifier mentioned explicitly in the article.",
+            }
+            for cve_id in extract_cve_ids(
+                " ".join(part for part in (title, content, summary) if part)
+            )
+        ]
+
         # Prepare prompt for tag generation
-        prompt = self._build_tagging_prompt(text_context, max_tags)
+        prompt = self._build_tagging_prompt(
+            text_context,
+            max_tags,
+            categories=self.get_all_categories(),
+        )
 
         try:
             # Use chat() method instead of generate() for compatibility with FallbackLLMClient
@@ -1156,26 +1263,81 @@ class TagManager:
                 temperature=0.3
             )
             candidate_tags = self.extract_tags_from_llm_response(response)
+        except Exception as e:
+            logger.error(f"Error generating LLM tags: {e}")
+            candidate_tags = []
 
-            # Select best tags using priority logic
+        try:
+            # CVEs are prepended so all directly extracted identifiers are processed
+            # even when the LLM returns more candidates than the regular-tag budget.
+            regular_limit = max(0, max_tags)
+            regular_candidates = [
+                candidate
+                for candidate in candidate_tags
+                if not is_cve_tag(candidate.get("name"))
+            ]
             selected = await self.select_tags_for_article_async(
                 article_id=article.get("id", ""),
-                candidate_tags=candidate_tags[:max_tags * 2],  # Get more than needed
+                candidate_tags=cve_candidates + regular_candidates[:regular_limit * 2],
                 allow_new_tags=True,
             )
 
-            return selected[:max_tags]  # Limit to max_tags
+            # Preserve selection order while applying the maximum only to non-CVE
+            # tags. Every CVE mentioned in the article remains in the result.
+            limited: List[Dict[str, Any]] = []
+            regular_count = 0
+            for tag in selected:
+                if is_cve_tag(tag.get("name")):
+                    limited.append(tag)
+                elif regular_count < regular_limit:
+                    limited.append(tag)
+                    regular_count += 1
+            return limited
         except Exception as e:
             logger.error(f"Error generating tags: {e}")
             return []
 
-    def _build_tagging_prompt(self, article_text: str, max_tags: int = 5) -> str:
+    def _build_tagging_prompt(
+        self,
+        article_text: str,
+        max_tags: int = 5,
+        categories: Optional[List[Dict[str, Any]]] = None,
+    ) -> str:
         """Build a prompt for the LLM to generate tags."""
         # Truncate article text if too long
         if len(article_text) > 2000:
             article_text = article_text[:2000] + "..."
 
-        return f"""Analyze the following article and extract up to {max_tags} relevant tags.
+        category_lines = []
+        category_names = []
+        for category in categories or []:
+            name = str(category.get("name") or "").strip()
+            if not name:
+                continue
+            category_names.append(name)
+            label = str(category.get("label") or "").strip()
+            description = str(category.get("description") or "").strip()
+            details = ": ".join(part for part in (label, description) if part)
+            category_lines.append(f"- {name}" + (f": {details}" if details else ""))
+        if not category_names:
+            category_names = ["GENERAL", "DOMAIN_ENTITY", "VULNERABILITY"]
+            category_lines = [f"- {name}" for name in category_names]
+        category_list = "\n".join(category_lines)
+        category_lookup = {name.casefold(): name for name in category_names}
+
+        def example_category(*preferred: str) -> str:
+            for name in preferred:
+                if name.casefold() in category_lookup:
+                    return category_lookup[name.casefold()]
+            return category_names[0]
+
+        entity_category = example_category("ORGANIZATION", "DOMAIN_ENTITY", "GENERAL")
+        topic_category = example_category("THREAT", "GENERAL")
+        product_category = example_category("PRODUCT", "DOMAIN_ENTITY", "GENERAL")
+
+        return f"""Analyze the following article and extract up to {max_tags} relevant non-CVE tags.
+Also include every CVE identifier mentioned in the article.
+CVE identifiers do not count toward this limit.
 
 CLASSIFICATION RULES:
 Tags should be classified as one of two types:
@@ -1196,10 +1358,16 @@ Tags should be classified as one of two types:
    - Security concepts (zero-day, vulnerability, authentication)
    - Any conceptual or descriptive tags
 
+AVAILABLE DATABASE CATEGORIES:
+{category_list}
+
 IMPORTANT:
 - Return tags in lowercase
 - Keep tags concise (1-3 words max)
 - Focus on the main topics and entities mentioned
+- Include every CVE identifier, even when this makes the total exceed {max_tags} tags
+- Set "category" to exactly one name from AVAILABLE DATABASE CATEGORIES
+- Choose the most specific suitable database category; use GENERAL when none is suitable
 - For NAMED_ENTITY tags: include ANY organization/group/actor/product name mentioned, even if uncommon
 - For CATEGORY tags: only include if they are central to the article
 - Provide a brief explanation for each tag (1-2 sentences max)
@@ -1210,9 +1378,9 @@ Article:
 Respond in JSON format:
 {{
     "tags": [
-        {{"tag": "tag1", "type": "NAMED_ENTITY", "reasoning": "Brief explanation why this tag is relevant"}},
-        {{"tag": "tag2", "type": "CATEGORY", "reasoning": "Brief explanation why this tag is relevant"}},
-        {{"tag": "tag3", "type": "NAMED_ENTITY", "reasoning": "Brief explanation why this tag is relevant"}}
+        {{"tag": "tag1", "type": "NAMED_ENTITY", "category": "{entity_category}", "reasoning": "Brief explanation why this tag is relevant"}},
+        {{"tag": "tag2", "type": "CATEGORY", "category": "{topic_category}", "reasoning": "Brief explanation why this tag is relevant"}},
+        {{"tag": "tag3", "type": "NAMED_ENTITY", "category": "{product_category}", "reasoning": "Brief explanation why this tag is relevant"}}
     ]
 }}"""
 

{feedsummary_core-2.2.0 → feedsummary_core-2.2.1}/src/feedsummary_core/summarizer/tagging_integration.py

@@ -66,7 +66,7 @@ async def tag_articles(
         llm_client: LLM client for generating tags and embeddings
         article_ids: List of article IDs to tag
         config: Configuration dictionary
-        max_tags_per_article: Maximum tags per article
+        max_tags_per_article: Maximum non-CVE tags per article
         skip_if_already_tagged: If True, skip articles that already have tags
         enable_embedding_matching: If True, use embedding-based tag matching when available
 
@@ -187,7 +187,7 @@ async def tag_article(
         llm_client: LLM client
         article_id: Article ID to tag
         config: Configuration
-        max_tags: Maximum tags
+        max_tags: Maximum non-CVE tags
 
     Returns:
         List of assigned tags
@@ -320,7 +320,7 @@ async def tag_articles_safe(
         article_ids: List of article IDs to tag
         config: Configuration dictionary
         job_id: Optional job ID for logging
-        max_tags_per_article: Maximum tags per article
+        max_tags_per_article: Maximum non-CVE tags per article
 
     Returns:
         Number of articles successfully tagged

{feedsummary_core-2.2.0 → feedsummary_core-2.2.1}/src/feedsummary_core/tagging_rules.py

@@ -37,7 +37,25 @@ from typing import Any
 
 
 VULNERABILITY_TAG_CATEGORY = "VULNERABILITY"
-CVE_PATTERN = re.compile(r"CVE-\d{4}-\d{4,}", re.IGNORECASE)
+CVE_PATTERN = re.compile(
+    r"(?<!\w)CVE-[0-9]{4}-[0-9]{4,19}(?!\w)",
+    re.IGNORECASE,
+)
+
+
+def extract_cve_ids(text: Any) -> list[str]:
+    """Extract unique CVE identifiers in canonical uppercase form."""
+    if not isinstance(text, str):
+        return []
+
+    cve_ids: list[str] = []
+    seen: set[str] = set()
+    for match in CVE_PATTERN.finditer(text):
+        cve_id = match.group(0).upper()
+        if cve_id not in seen:
+            seen.add(cve_id)
+            cve_ids.append(cve_id)
+    return cve_ids
 
 
 def is_cve_tag(name: Any) -> bool:

{feedsummary_core-2.2.0 → feedsummary_core-2.2.1/src/feedsummary_core.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feedsummary-core
-Version: 2.2.0
+Version: 2.2.1
 Summary: Core library for FeedSummary.
 Author: Martin Vesterlund
 License-Expression: BSD-3-Clause

{feedsummary_core-2.2.0 → feedsummary_core-2.2.1}/src/feedsummary_core.egg-info/scm_version.json

@@ -1,7 +1,7 @@
 {
-  "tag": "2.2.0",
+  "tag": "2.2.1",
   "distance": 0,
-  "node": "g03a56f8f464f99830bdc5eb121fc20a457cc7b78",
+  "node": "g4ab927c63858ac0a23fc65c3ebd727d0cbe7884c",
   "dirty": false,
   "branch": "HEAD",
   "node_date": "2026-08-21"

feedsummary_core-2.2.1/tests/test_tagging_cve_category.py

@@ -0,0 +1,274 @@
+import asyncio
+import json
+import unittest
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from feedsummary_core.persistence.SqliteStore import SqliteStore
+from feedsummary_core.persistence.TinyDbStore import TinyDBStore
+from feedsummary_core.summarizer.tagging import TagManager
+from feedsummary_core.tagging_rules import extract_cve_ids
+
+
+class MemoryTagStore:
+    def __init__(self, categories=None):
+        self.tags = []
+        self.categories = categories or []
+
+    def add_tag(self, name, category="GENERAL", description=None):
+        existing = self.get_tag_by_name(name)
+        if existing:
+            return existing["id"]
+        tag = {
+            "id": len(self.tags) + 1,
+            "name": name,
+            "category": category,
+            "description": description,
+        }
+        self.tags.append(tag)
+        return tag["id"]
+
+    def get_tag_by_name(self, name):
+        normalized = name.strip().lower()
+        return next(
+            (tag.copy() for tag in self.tags if tag["name"].lower() == normalized),
+            None,
+        )
+
+    def get_all_tags(self):
+        return [tag.copy() for tag in self.tags]
+
+    def get_all_categories(self):
+        return [category.copy() for category in self.categories]
+
+    def update_tag(self, tag_id, category=None, **_kwargs):
+        tag = next(tag for tag in self.tags if tag["id"] == tag_id)
+        if category is not None:
+            tag["category"] = category
+        return tag.copy()
+
+
+class FakeTaggingClient:
+    def __init__(self, tags=None, error=None):
+        self.tags = tags
+        self.error = error
+        self.messages = []
+
+    async def chat(self, messages, temperature=0.3):
+        self.messages = messages
+        if self.error:
+            raise self.error
+        return json.dumps({"tags": self.tags})
+
+
+class CveCategoryTests(unittest.TestCase):
+    def setUp(self):
+        self.store = MemoryTagStore()
+        self.manager = TagManager(self.store)
+
+    def test_add_tag_assigns_vulnerability_to_uppercase_cve(self):
+        tag_id = self.manager.add_tag("CVE-2026-12345", category="DOMAIN_ENTITY")
+
+        tag = self.store.get_tag_by_name("CVE-2026-12345")
+        self.assertEqual(tag_id, tag["id"])
+        self.assertEqual("VULNERABILITY", tag["category"])
+
+    def test_select_tag_assigns_vulnerability_to_lowercase_six_digit_cve(self):
+        selected = self.manager.select_tags_for_article(
+            "article-1",
+            [{"name": "cve-2026-123456", "type": "NAMED_ENTITY"}],
+        )
+
+        self.assertEqual(1, len(selected))
+        self.assertEqual("cve-2026-123456", selected[0]["name"])
+        self.assertEqual("VULNERABILITY", selected[0]["category"])
+
+    def test_existing_cve_is_moved_to_vulnerability(self):
+        tag_id = self.store.add_tag("cve-2025-1234", category="DOMAIN_ENTITY")
+
+        selected = self.manager.select_tags_for_article(
+            "article-1",
+            ["CVE-2025-1234"],
+        )
+
+        self.assertEqual("VULNERABILITY", selected[0]["category"])
+        self.assertEqual(
+            "VULNERABILITY",
+            self.store.get_tag_by_name("cve-2025-1234")["category"],
+        )
+        self.assertEqual(tag_id, selected[0]["id"])
+
+    def test_distinct_cves_are_not_fuzzy_matched_to_each_other(self):
+        first = self.manager.select_tags_for_article(
+            "article-1",
+            ["CVE-2026-12345"],
+        )
+        second = self.manager.select_tags_for_article(
+            "article-2",
+            ["CVE-2026-12346"],
+        )
+
+        self.assertNotEqual(first[0]["id"], second[0]["id"])
+        self.assertEqual(2, len(self.store.tags))
+        self.assertTrue(
+            all(tag["category"] == "VULNERABILITY" for tag in self.store.tags)
+        )
+
+    def test_non_cve_tag_keeps_requested_category(self):
+        self.manager.add_tag("cve-2026-123", category="DOMAIN_ENTITY")
+
+        tag = self.store.get_tag_by_name("cve-2026-123")
+        self.assertEqual("DOMAIN_ENTITY", tag["category"])
+
+    def test_extracts_only_complete_cves_with_four_to_nineteen_digits(self):
+        nineteen_digits = "1234567890123456789"
+        text = (
+            "CVE-2024-1234, cve-2025-123456 and "
+            f"CVE-2026-{nineteen_digits}; repeated CVE-2024-1234. "
+            "Ignore CVE-2024-123, CVE-2024-12345678901234567890 and "
+            "prefixCVE-2024-9999suffix."
+        )
+
+        self.assertEqual(
+            [
+                "CVE-2024-1234",
+                "CVE-2025-123456",
+                f"CVE-2026-{nineteen_digits}",
+            ],
+            extract_cve_ids(text),
+        )
+
+    def test_generated_cves_do_not_count_toward_regular_tag_limit(self):
+        client = FakeTaggingClient(
+            [
+                {"tag": "CVE-2025-1234", "type": "NAMED_ENTITY"},
+                {"tag": "CVE-2026-123456", "type": "NAMED_ENTITY"},
+                {"tag": "CVE-2025-1234", "type": "NAMED_ENTITY"},
+                {"tag": "Acme", "type": "NAMED_ENTITY"},
+                {"tag": "Example Product", "type": "NAMED_ENTITY"},
+                {"tag": "Third Entity", "type": "NAMED_ENTITY"},
+            ]
+        )
+        article = {
+            "id": "article-1",
+            "title": "CVE-2025-1234 fixed by Acme",
+            "content": "The update also fixes cve-2026-123456.",
+            "summary": "CVE-2025-1234 is mentioned again.",
+        }
+
+        selected = asyncio.run(
+            self.manager.generate_tags_for_article(
+                llm_client=client,
+                article=article,
+                config={},
+                max_tags=2,
+            )
+        )
+
+        cve_tags = [tag for tag in selected if tag["category"] == "VULNERABILITY"]
+        regular_tags = [tag for tag in selected if tag["category"] != "VULNERABILITY"]
+        self.assertEqual(
+            {"cve-2025-1234", "cve-2026-123456"},
+            {tag["name"] for tag in cve_tags},
+        )
+        self.assertEqual(2, len(regular_tags))
+        self.assertEqual(4, len(selected))
+
+    def test_direct_cve_extraction_survives_llm_failure_and_zero_limit(self):
+        selected = asyncio.run(
+            self.manager.generate_tags_for_article(
+                llm_client=FakeTaggingClient(error=RuntimeError("LLM unavailable")),
+                article={
+                    "id": "article-2",
+                    "title": "Advisory for CVE-2026-9876",
+                    "content": "",
+                },
+                config={},
+                max_tags=0,
+            )
+        )
+
+        self.assertEqual(["cve-2026-9876"], [tag["name"] for tag in selected])
+        self.assertEqual("VULNERABILITY", selected[0]["category"])
+
+    def test_llm_uses_database_defined_category_for_new_tag(self):
+        store = MemoryTagStore(
+            categories=[
+                {"name": "GENERAL", "label": "Allmän", "description": "Fallback"},
+                {
+                    "name": "RISK_THEME",
+                    "label": "Risktema",
+                    "description": "Strategic and emerging risks",
+                },
+            ]
+        )
+        manager = TagManager(store)
+        client = FakeTaggingClient(
+            [
+                {"tag": "quux", "type": "CATEGORY", "category": "risk_theme"},
+                {"tag": "ransomware", "type": "CATEGORY", "category": "NOT_REAL"},
+            ]
+        )
+
+        selected = asyncio.run(
+            manager.generate_tags_for_article(
+                llm_client=client,
+                article={"id": "article-3", "title": "Quux ransomware risk"},
+                config={},
+                max_tags=2,
+            )
+        )
+
+        self.assertEqual(
+            {"quux": "RISK_THEME", "ransomware": "GENERAL"},
+            {tag["name"]: tag["category"] for tag in selected},
+        )
+        prompt = client.messages[0]["content"]
+        self.assertIn("RISK_THEME: Risktema: Strategic and emerging risks", prompt)
+        self.assertIn('Set "category" to exactly one name', prompt)
+
+
+class VulnerabilityCategoryInitializationTests(unittest.TestCase):
+    def test_sqlite_custom_category_is_used_for_new_tag(self):
+        with TemporaryDirectory() as directory:
+            store = SqliteStore(str(Path(directory) / "tags.sqlite"))
+            store.initialize_default_categories()
+            store.create_category(
+                "RISK_THEME",
+                "Risktema",
+                description="Strategic and emerging risks",
+            )
+
+            tag_id = TagManager(store).add_tag("quantum risk", category="risk_theme")
+
+            stored_tag = store.get_tag_by_name("quantum risk")
+            self.assertEqual(tag_id, stored_tag["id"])
+            self.assertEqual("RISK_THEME", stored_tag["category"])
+
+    def test_tinydb_initializes_vulnerability_category(self):
+        with TemporaryDirectory() as directory:
+            store = TinyDBStore(str(Path(directory) / "tags.json"))
+            tag_id = store.add_tag("cve-2024-12345", category="DOMAIN_ENTITY")
+            store.initialize_default_categories()
+
+            categories = {category["name"] for category in store.get_all_categories()}
+            self.assertIn("VULNERABILITY", categories)
+            stored_tag = store.get_tag_by_name("cve-2024-12345")
+            self.assertEqual("VULNERABILITY", stored_tag["category"])
+            self.assertEqual(tag_id, stored_tag["id"])
+
+    def test_sqlite_initializes_vulnerability_category(self):
+        with TemporaryDirectory() as directory:
+            store = SqliteStore(str(Path(directory) / "tags.sqlite"))
+            tag_id = store.add_tag("CVE-2024-123456", category="DOMAIN_ENTITY")
+            store.initialize_default_categories()
+
+            categories = {category["name"] for category in store.get_all_categories()}
+            self.assertIn("VULNERABILITY", categories)
+            stored_tag = store.get_tag_by_name("CVE-2024-123456")
+            self.assertEqual("VULNERABILITY", stored_tag["category"])
+            self.assertEqual(tag_id, stored_tag["id"])
+
+
+if __name__ == "__main__":
+    unittest.main()

feedsummary_core-2.2.0/tests/test_tagging_cve_category.py

@@ -1,131 +0,0 @@
-import unittest
-from pathlib import Path
-from tempfile import TemporaryDirectory
-
-from feedsummary_core.persistence.SqliteStore import SqliteStore
-from feedsummary_core.persistence.TinyDbStore import TinyDBStore
-from feedsummary_core.summarizer.tagging import TagManager
-
-
-class MemoryTagStore:
-    def __init__(self):
-        self.tags = []
-
-    def add_tag(self, name, category="GENERAL", description=None):
-        existing = self.get_tag_by_name(name)
-        if existing:
-            return existing["id"]
-        tag = {
-            "id": len(self.tags) + 1,
-            "name": name,
-            "category": category,
-            "description": description,
-        }
-        self.tags.append(tag)
-        return tag["id"]
-
-    def get_tag_by_name(self, name):
-        normalized = name.strip().lower()
-        return next(
-            (tag.copy() for tag in self.tags if tag["name"].lower() == normalized),
-            None,
-        )
-
-    def get_all_tags(self):
-        return [tag.copy() for tag in self.tags]
-
-    def update_tag(self, tag_id, category=None, **_kwargs):
-        tag = next(tag for tag in self.tags if tag["id"] == tag_id)
-        if category is not None:
-            tag["category"] = category
-        return tag.copy()
-
-
-class CveCategoryTests(unittest.TestCase):
-    def setUp(self):
-        self.store = MemoryTagStore()
-        self.manager = TagManager(self.store)
-
-    def test_add_tag_assigns_vulnerability_to_uppercase_cve(self):
-        tag_id = self.manager.add_tag("CVE-2026-12345", category="DOMAIN_ENTITY")
-
-        tag = self.store.get_tag_by_name("CVE-2026-12345")
-        self.assertEqual(tag_id, tag["id"])
-        self.assertEqual("VULNERABILITY", tag["category"])
-
-    def test_select_tag_assigns_vulnerability_to_lowercase_six_digit_cve(self):
-        selected = self.manager.select_tags_for_article(
-            "article-1",
-            [{"name": "cve-2026-123456", "type": "NAMED_ENTITY"}],
-        )
-
-        self.assertEqual(1, len(selected))
-        self.assertEqual("cve-2026-123456", selected[0]["name"])
-        self.assertEqual("VULNERABILITY", selected[0]["category"])
-
-    def test_existing_cve_is_moved_to_vulnerability(self):
-        tag_id = self.store.add_tag("cve-2025-1234", category="DOMAIN_ENTITY")
-
-        selected = self.manager.select_tags_for_article(
-            "article-1",
-            ["CVE-2025-1234"],
-        )
-
-        self.assertEqual("VULNERABILITY", selected[0]["category"])
-        self.assertEqual(
-            "VULNERABILITY",
-            self.store.get_tag_by_name("cve-2025-1234")["category"],
-        )
-        self.assertEqual(tag_id, selected[0]["id"])
-
-    def test_distinct_cves_are_not_fuzzy_matched_to_each_other(self):
-        first = self.manager.select_tags_for_article(
-            "article-1",
-            ["CVE-2026-12345"],
-        )
-        second = self.manager.select_tags_for_article(
-            "article-2",
-            ["CVE-2026-12346"],
-        )
-
-        self.assertNotEqual(first[0]["id"], second[0]["id"])
-        self.assertEqual(2, len(self.store.tags))
-        self.assertTrue(
-            all(tag["category"] == "VULNERABILITY" for tag in self.store.tags)
-        )
-
-    def test_non_cve_tag_keeps_requested_category(self):
-        self.manager.add_tag("cve-2026-123", category="DOMAIN_ENTITY")
-
-        tag = self.store.get_tag_by_name("cve-2026-123")
-        self.assertEqual("DOMAIN_ENTITY", tag["category"])
-
-
-class VulnerabilityCategoryInitializationTests(unittest.TestCase):
-    def test_tinydb_initializes_vulnerability_category(self):
-        with TemporaryDirectory() as directory:
-            store = TinyDBStore(str(Path(directory) / "tags.json"))
-            tag_id = store.add_tag("cve-2024-12345", category="DOMAIN_ENTITY")
-            store.initialize_default_categories()
-
-            categories = {category["name"] for category in store.get_all_categories()}
-            self.assertIn("VULNERABILITY", categories)
-            stored_tag = store.get_tag_by_name("cve-2024-12345")
-            self.assertEqual("VULNERABILITY", stored_tag["category"])
-            self.assertEqual(tag_id, stored_tag["id"])
-
-    def test_sqlite_initializes_vulnerability_category(self):
-        with TemporaryDirectory() as directory:
-            store = SqliteStore(str(Path(directory) / "tags.sqlite"))
-            tag_id = store.add_tag("CVE-2024-123456", category="DOMAIN_ENTITY")
-            store.initialize_default_categories()
-
-            categories = {category["name"] for category in store.get_all_categories()}
-            self.assertIn("VULNERABILITY", categories)
-            stored_tag = store.get_tag_by_name("CVE-2024-123456")
-            self.assertEqual("VULNERABILITY", stored_tag["category"])
-            self.assertEqual(tag_id, stored_tag["id"])
-
-
-if __name__ == "__main__":
-    unittest.main()