feedsummary-core 2.2.0__tar.gz → 2.3.0__tar.gz

{feedsummary_core-2.2.0 → feedsummary_core-2.3.0}/.gitignore

@@ -154,6 +154,9 @@ venv.bak/
 # mkdocs documentation
 /site
 
+# Local machine-learning benchmark outputs
+/artifacts/tag-benchmark/
+
 # mypy
 .mypy_cache/
 .dmypy.json

{feedsummary_core-2.2.0/src/feedsummary_core.egg-info → feedsummary_core-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feedsummary-core
-Version: 2.2.0
+Version: 2.3.0
 Summary: Core library for FeedSummary.
 Author: Martin Vesterlund
 License-Expression: BSD-3-Clause
@@ -22,6 +22,11 @@ Requires-Dist: ollama>=0.6.1
 Requires-Dist: lxml_html_clean>=0.4.3
 Provides-Extra: mongodb
 Requires-Dist: pymongo>=4.10; extra == "mongodb"
+Provides-Extra: ml
+Requires-Dist: pymongo>=4.10; extra == "ml"
+Requires-Dist: joblib>=1.2; extra == "ml"
+Requires-Dist: scikit-learn<1.8,>=1.7; python_version < "3.11" and extra == "ml"
+Requires-Dist: scikit-learn<2,>=1.7; python_version >= "3.11" and extra == "ml"
 Dynamic: license-file
 
 # feedSummaryBase
@@ -188,9 +193,15 @@ Create a `config.yaml` file:
 
 ```yaml
 feeds:
-  - url: "https://example.com/feed.xml"
-    lookback: "48h"
-    enabled: true
+  - name: "Example feed"
+    url: "https://example.com/feed.xml"
+    # Optional per-feed minimum TLS version, applied to the feed and its articles.
+    # Supported values: "1.2" and "1.3".
+    tls_min_version: "1.3"
+    # Use native curl for servers that reject aiohttp's TLS fingerprint.
+    http_client: "curl"
+    # Optional: use only text embedded in the RSS entry and never fetch its link.
+    fetch_article: false
 
 store:
   provider: "sqlite"  # sqlite, tinydb, or mongodb
@@ -219,6 +230,104 @@ tagging:
     max_shared_tags: 1
 ```
 
+### Embedding-based ML tagging
+
+The production ML path uses a classifier selected in configuration with
+persisted article embeddings. Both the classifier and the category scope are
+configuration-driven; `categories` accepts one or more database category names.
+On each tagging run, a corpus fingerprint detects changed article-tag
+associations, classifier settings, or category scope and atomically retrains the
+artifact when needed. Full retraining correctly handles tags that were removed
+or corrected.
+
+```yaml
+tagging:
+  # Existing tags in these categories may be selected by the LLM, but the LLM
+  # may not create new tags in them.
+  llm_new_tag_excluded_categories: [DOMAIN_ENTITY]
+  ml:
+    enabled: false
+    classifier: logistic_regression  # logistic_regression or sgd
+    representation: embedding
+    categories: [DOMAIN_ENTITY]
+    embedding_model: ""  # inherit from the local embedding provider
+    embedding_text_chars: 2000
+    model_path: data/tagging_ml/embedding_classifier.joblib
+    min_label_support: 3
+    min_training_articles: 30
+    max_tags_per_article: 5
+    threshold: 0.8014432738078289
+    regularization_c: 1.0  # logistic_regression only
+    alpha: 0.0001          # sgd only
+    max_iter: 2000
+    tolerance: 0.0001
+    random_state: 42
+    auto_retrain: true
+```
+
+To change or extend the scope, edit only the category list, for example
+`categories: [DOMAIN_ENTITY, LOCATION]`. The artifact is rejected and retrained
+when its configured classifier or category list no longer matches.
+
+When a compatible embedding or model is unavailable, tagging safely falls back
+to the existing LLM path. `llm_new_tag_excluded_categories` still applies to
+that fallback, so it may reuse an existing `DOMAIN_ENTITY` tag but cannot add a
+new one. Joblib artifacts must only be loaded from trusted local paths.
+
+ML tagging emits searchable JSON payloads after stable event names:
+
+- `ml_tagging.model_ready` records the classifier, categories, threshold, model
+  version, label count, and training corpus metadata.
+- `ml_tagging.predictions` records the article ID and every accepted ML tag with
+  category and probability. An empty `suggestions` list means that the model ran
+  but no score met the configured threshold.
+- `ml_tagging.skipped` records incompatible or missing article embeddings, while
+  `ml_tagging.model_unavailable` records initialization and training failures.
+- `ml_tagging.below_threshold` records the five strongest rejected candidates
+  at DEBUG level.
+
+No article text is included in these events. Accepted ML assignments also keep
+the classifier and rounded probability in the persisted `motivering` field.
+
+### Benchmark classical ML tagging
+
+The first ML tagging stage is an offline, read-only benchmark against historical
+MongoDB tag assignments. It does not change the production LLM tagging flow.
+Install the optional dependencies and add an ML section to the normal config:
+
+```bash
+pip install "feedsummary-core[ml]"
+feedsummary-benchmark-tags --config config.yaml --output-dir artifacts/tag-benchmark
+```
+
+```yaml
+store:
+  provider: mongodb
+  uri: mongodb://localhost:27017
+  database: feedsummary
+
+tagging:
+  ml:
+    categories: [GENERAL]
+    min_label_support: 10
+    max_tags_per_article: 5
+    max_text_chars: 20000
+    random_seed: 42
+    n_jobs: 1
+    max_category_combinations: 63
+```
+
+Each run writes a JSON report, a compact Markdown report, and the winning
+scikit-learn pipeline. Historical tags are treated as labels and may contain
+LLM-generated noise. Only load the generated Joblib model from a trusted source.
+When multiple categories are configured, every non-empty category combination
+is evaluated. Ranking uses chronological validation micro-F1 adjusted for tag
+assignment coverage: tags excluded by `min_label_support` count as unpredicted.
+The report shows raw F1 alongside label and assignment coverage, so a category
+with one easy, well-supported tag cannot win merely because its many rare tags
+were filtered out. The limit above prevents an accidental exponential benchmark
+when too many categories are supplied.
+
 After all articles in a run have been tagged, similarity consistency checks
 embedding-based article groups for tag overlap. If a group has no tag shared by
 all members, the most common existing general tag is added to the missing

{feedsummary_core-2.2.0 → feedsummary_core-2.3.0}/README.md

@@ -162,9 +162,15 @@ Create a `config.yaml` file:
 
 ```yaml
 feeds:
-  - url: "https://example.com/feed.xml"
-    lookback: "48h"
-    enabled: true
+  - name: "Example feed"
+    url: "https://example.com/feed.xml"
+    # Optional per-feed minimum TLS version, applied to the feed and its articles.
+    # Supported values: "1.2" and "1.3".
+    tls_min_version: "1.3"
+    # Use native curl for servers that reject aiohttp's TLS fingerprint.
+    http_client: "curl"
+    # Optional: use only text embedded in the RSS entry and never fetch its link.
+    fetch_article: false
 
 store:
   provider: "sqlite"  # sqlite, tinydb, or mongodb
@@ -193,6 +199,104 @@ tagging:
     max_shared_tags: 1
 ```
 
+### Embedding-based ML tagging
+
+The production ML path uses a classifier selected in configuration with
+persisted article embeddings. Both the classifier and the category scope are
+configuration-driven; `categories` accepts one or more database category names.
+On each tagging run, a corpus fingerprint detects changed article-tag
+associations, classifier settings, or category scope and atomically retrains the
+artifact when needed. Full retraining correctly handles tags that were removed
+or corrected.
+
+```yaml
+tagging:
+  # Existing tags in these categories may be selected by the LLM, but the LLM
+  # may not create new tags in them.
+  llm_new_tag_excluded_categories: [DOMAIN_ENTITY]
+  ml:
+    enabled: false
+    classifier: logistic_regression  # logistic_regression or sgd
+    representation: embedding
+    categories: [DOMAIN_ENTITY]
+    embedding_model: ""  # inherit from the local embedding provider
+    embedding_text_chars: 2000
+    model_path: data/tagging_ml/embedding_classifier.joblib
+    min_label_support: 3
+    min_training_articles: 30
+    max_tags_per_article: 5
+    threshold: 0.8014432738078289
+    regularization_c: 1.0  # logistic_regression only
+    alpha: 0.0001          # sgd only
+    max_iter: 2000
+    tolerance: 0.0001
+    random_state: 42
+    auto_retrain: true
+```
+
+To change or extend the scope, edit only the category list, for example
+`categories: [DOMAIN_ENTITY, LOCATION]`. The artifact is rejected and retrained
+when its configured classifier or category list no longer matches.
+
+When a compatible embedding or model is unavailable, tagging safely falls back
+to the existing LLM path. `llm_new_tag_excluded_categories` still applies to
+that fallback, so it may reuse an existing `DOMAIN_ENTITY` tag but cannot add a
+new one. Joblib artifacts must only be loaded from trusted local paths.
+
+ML tagging emits searchable JSON payloads after stable event names:
+
+- `ml_tagging.model_ready` records the classifier, categories, threshold, model
+  version, label count, and training corpus metadata.
+- `ml_tagging.predictions` records the article ID and every accepted ML tag with
+  category and probability. An empty `suggestions` list means that the model ran
+  but no score met the configured threshold.
+- `ml_tagging.skipped` records incompatible or missing article embeddings, while
+  `ml_tagging.model_unavailable` records initialization and training failures.
+- `ml_tagging.below_threshold` records the five strongest rejected candidates
+  at DEBUG level.
+
+No article text is included in these events. Accepted ML assignments also keep
+the classifier and rounded probability in the persisted `motivering` field.
+
+### Benchmark classical ML tagging
+
+The first ML tagging stage is an offline, read-only benchmark against historical
+MongoDB tag assignments. It does not change the production LLM tagging flow.
+Install the optional dependencies and add an ML section to the normal config:
+
+```bash
+pip install "feedsummary-core[ml]"
+feedsummary-benchmark-tags --config config.yaml --output-dir artifacts/tag-benchmark
+```
+
+```yaml
+store:
+  provider: mongodb
+  uri: mongodb://localhost:27017
+  database: feedsummary
+
+tagging:
+  ml:
+    categories: [GENERAL]
+    min_label_support: 10
+    max_tags_per_article: 5
+    max_text_chars: 20000
+    random_seed: 42
+    n_jobs: 1
+    max_category_combinations: 63
+```
+
+Each run writes a JSON report, a compact Markdown report, and the winning
+scikit-learn pipeline. Historical tags are treated as labels and may contain
+LLM-generated noise. Only load the generated Joblib model from a trusted source.
+When multiple categories are configured, every non-empty category combination
+is evaluated. Ranking uses chronological validation micro-F1 adjusted for tag
+assignment coverage: tags excluded by `min_label_support` count as unpredicted.
+The report shows raw F1 alongside label and assignment coverage, so a category
+with one easy, well-supported tag cannot win merely because its many rare tags
+were filtered out. The limit above prevents an accidental exponential benchmark
+when too many categories are supplied.
+
 After all articles in a run have been tagged, similarity consistency checks
 embedding-based article groups for tag overlap. If a group has no tag shared by
 all members, the most common existing general tag is added to the missing

{feedsummary_core-2.2.0 → feedsummary_core-2.3.0}/TAGGING_QUICKSTART.md

@@ -167,7 +167,7 @@ Edit `tagging.py` to modify patterns:
 
 ```python
 # Add CVE variants
-CVE_PATTERN = re.compile(r'CVE-\d{4}-\d{4,5}|GHSA-\w{4}-\w{4}-\w{4}')
+CVE_PATTERN = re.compile(r'CVE-[0-9]{4}-[0-9]{4,19}|GHSA-\w{4}-\w{4}-\w{4}')
 
 # Add threat actor keywords
 THREAT_ACTOR_KEYWORDS = {

{feedsummary_core-2.2.0 → feedsummary_core-2.3.0}/TAGGING_SYSTEM.md

@@ -86,28 +86,26 @@ CREATE TABLE article_tags (
 
 ## Taggkategorier
 
-### GENERAL (Standard)
-- Normala ämnestaggar
-- Prioriteras från befintliga taggar
-- Får inte skapas nya automatiskt (om de redan finns)
-
-### DOMAIN_ENTITY (Domänspecifik)
-Undantag där nya taggar ALLTID tillåts:
-- **CVE:er**: `CVE-2024-1234` mönster
-- **Hot-aktörer**: APT-grupper, kampanjnamn (t.ex. "APT28", "Lazarus Group")
-- **Geografiska regioner**: Länder, kontinenter (t.ex. "Russia", "North Korea")
-- **Sårbarheter**: Specifika sårbarhetsnamn och typer
+Kategorier hämtas dynamiskt från databasens `tag_categories`. LLM:en får
+kategoriernas namn, etikett och beskrivning och väljer den mest specifika
+lämpliga kategorin för varje ny tagg. Valet valideras case-insensitivt mot
+databasen och lagras med kategorins kanoniska namn.
+
+Om LLM:en inte anger en giltig kategori används en definierad standardkategori,
+normalt `GENERAL` eller `DOMAIN_ENTITY`. CVE-taggar är ett särskilt undantag och
+tvingas alltid till `VULNERABILITY`.
 
 ## Taggprioriteringsalgoritm
 
 ```
-1. Extrahera kandidattaggar från artikel (via LLM)
-2. För varje kandidattagg:
+1. Hämta tillgängliga kategorier från databasen
+2. Extrahera kandidattaggar och kategori från artikel (via LLM)
+3. För varje kandidattagg:
    a. Sök efter befintliga liknande taggar
    b. Om befintlig tagg finns → använd den
-   c. Om domänspecifik entitet → skapa ny DOMAIN_ENTITY tagg
-   d. Annars → hoppa över (föredra allmän framför specifik)
-3. Lagra valda taggar i databasen
+   c. Om en ny tagg ska skapas → validera LLM-kategorin mot databasen
+   d. Skapa taggen med databasens kanoniska kategorinamn
+4. Lagra valda taggar i databasen
 ```
 
 ### Likhetsmätning
@@ -215,23 +213,23 @@ print(f"Removed {removed} unused tags")
 Systemet använder följande prompt-mall för att få LLM att extrahera taggar:
 
 ```
-Analyze the following article and extract up to {max_tags} relevant tags.
+Analyze the following article and extract up to {max_tags} relevant non-CVE tags.
 
 IMPORTANT RULES:
-1. Prefer existing, general tags over creating new, specific tags
-2. CVE numbers (e.g., CVE-2024-1234), threat actors, regions, and 
-   vulnerability names are EXCEPTIONS - these should be new tags
-3. Return tags in lowercase
-4. Keep tags concise (1-3 words max)
-5. Focus on the main topics and entities mentioned
+1. Select "category" from the database categories included in the prompt
+2. Use the most specific suitable category and GENERAL as fallback
+3. Include every CVE; CVE tags do not count toward max_tags
+4. Return tags in lowercase and keep them concise
 
 Article:
 {article_text}
 
 Respond in JSON format:
 {
-    "tags": ["tag1", "tag2", "tag3"],
-    "reasoning": "Brief explanation of why these tags were chosen"
+    "tags": [
+        {"tag": "tag1", "type": "NAMED_ENTITY", "category": "ORGANIZATION",
+         "reasoning": "Why this tag and category are relevant"}
+    ]
 }
 ```
 
@@ -241,7 +239,7 @@ Systemet extraherar sedan JSON från svar och tillämpar prioriteringslogiken.
 
 Systemet identifierar domänspecifika entiteter genom:
 
-1. **Regex-mönster**: CVE format (`CVE-\d{4}-\d{4,5}`)
+1. **Regex-mönster**: CVE format (`CVE-[0-9]{4}-[0-9]{4,19}`)
 2. **Nyckelordsmatchning**:
    - Hot-aktörer: "APT", "group", "threat actor", "collective"
    - Regioner: Länder, kontinenter, geografiska termer
@@ -363,7 +361,7 @@ similar_tags = tag_manager._find_similar_existing_tags(
 Redigera regex-mönster och nyckelord i `tagging.py`:
 
 ```python
-CVE_PATTERN = re.compile(r'CVE-\d{4}-\d{4,5}', re.IGNORECASE)
+CVE_PATTERN = re.compile(r'CVE-[0-9]{4}-[0-9]{4,19}', re.IGNORECASE)
 THREAT_ACTOR_KEYWORDS = {'APT', 'group', ...}  # Lägg till fler ord
 ```
 

{feedsummary_core-2.2.0 → feedsummary_core-2.3.0}/pyproject.toml

@@ -32,9 +32,16 @@ dependencies = [
 
 [project.optional-dependencies]
 mongodb = ["pymongo>=4.10"]
+ml = [
+  "pymongo>=4.10",
+  "joblib>=1.2",
+  "scikit-learn>=1.7,<1.8; python_version < '3.11'",
+  "scikit-learn>=1.7,<2; python_version >= '3.11'",
+]
 
 [project.scripts]
 feedsummary-migrate-tinydb-mongodb = "feedsummary_core.persistence.migrate_tinydb_to_mongodb:main"
+feedsummary-benchmark-tags = "feedsummary_core.tagging_ml.cli:main"
 
 [tool.setuptools]
 package-dir = {"" = "src"}

{feedsummary_core-2.2.0 → feedsummary_core-2.3.0}/src/feedsummary_core/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '2.2.0'
-__version_tuple__ = version_tuple = (2, 2, 0)
+__version__ = version = '2.3.0'
+__version_tuple__ = version_tuple = (2, 3, 0)
 
-__commit_id__ = commit_id = 'g03a56f8f4'
+__commit_id__ = commit_id = 'g6b2e06899'

{feedsummary_core-2.2.0 → feedsummary_core-2.3.0}/src/feedsummary_core/persistence/MongoDBStore.py

@@ -34,6 +34,7 @@ from __future__ import annotations
 import logging
 import math
 import time
+from collections.abc import Iterator
 from typing import Any, Dict, List, Optional, Tuple
 
 from feedsummary_core.persistence.CleanUpPolicy import CleanupPolicy
@@ -106,6 +107,7 @@ class MongoDBStore:
         *,
         client: Any = None,
         connect_timeout_ms: int = 5000,
+        initialize_schema: bool = True,
     ):
         if not database or not str(database).strip():
             raise ValueError("database must be a non-empty string")
@@ -122,7 +124,8 @@ class MongoDBStore:
 
         self.client = client
         self.db = client[self.database_name]
-        self._init_db()
+        if initialize_schema:
+            self._init_db()
 
     def close(self) -> None:
         if self._owns_client and self.client is not None:
@@ -247,6 +250,16 @@ class MongoDBStore:
         cursor = self.db.articles.find().sort("_mongo_sort_ts", ASCENDING).limit(max(0, int(limit)))
         return [_public_doc(doc) for doc in cursor]  # type: ignore[misc]
 
+    def iter_articles(self, limit: Optional[int] = None) -> Iterator[Dict[str, Any]]:
+        """Yield every article oldest-first without the list API's default cap."""
+        cursor = self.db.articles.find().sort("_mongo_sort_ts", ASCENDING)
+        if limit is not None and int(limit) > 0:
+            cursor = cursor.limit(int(limit))
+        for doc in cursor:
+            article = _public_doc(doc)
+            if isinstance(article, dict):
+                yield article
+
     def list_articles_by_filter(
         self,
         *,
@@ -466,6 +479,71 @@ class MongoDBStore:
     def get_all_tags(self) -> List[Dict[str, Any]]:
         return [self._tag_doc(doc) for doc in self.db.tags.find().sort("name", ASCENDING)]  # type: ignore[misc]
 
+    def iter_articles_with_tags(
+        self,
+        *,
+        categories: Optional[List[str]] = None,
+        limit: Optional[int] = None,
+    ) -> Iterator[Dict[str, Any]]:
+        """Yield canonically tagged articles for read-only exports.
+
+        The export starts from ``article_tags`` so an article is included only
+        when a tagging pass produced at least one persisted association.  The
+        optional category filter applies to the returned labels, not to article
+        eligibility; an article tagged only with excluded entity categories is
+        therefore returned with an empty ``tags`` list and can act as a negative
+        training example.
+        """
+        pipeline: List[Dict[str, Any]] = [
+            {
+                "$group": {
+                    "_id": "$article_id",
+                    "tag_ids": {"$addToSet": "$tag_id"},
+                }
+            },
+            {
+                "$lookup": {
+                    "from": "articles",
+                    "localField": "_id",
+                    "foreignField": "_id",
+                    "as": "article_docs",
+                }
+            },
+            {"$unwind": "$article_docs"},
+            {
+                "$lookup": {
+                    "from": "tags",
+                    "localField": "tag_ids",
+                    "foreignField": "_id",
+                    "as": "tag_docs",
+                }
+            },
+            {"$sort": {"article_docs._mongo_sort_ts": ASCENDING, "_id": ASCENDING}},
+        ]
+        if limit is not None and int(limit) > 0:
+            pipeline.append({"$limit": int(limit)})
+
+        allowed = {
+            str(category).strip().casefold()
+            for category in categories or []
+            if str(category).strip()
+        }
+        for row in self.db.article_tags.aggregate(pipeline):
+            article = _public_doc(row.get("article_docs"))
+            if not isinstance(article, dict):
+                continue
+            tags = []
+            for raw_tag in row.get("tag_docs") or []:
+                tag = self._tag_doc(raw_tag)
+                if not isinstance(tag, dict):
+                    continue
+                category = str(tag.get("category") or "GENERAL").casefold()
+                if allowed and category not in allowed:
+                    continue
+                tags.append(tag)
+            tags.sort(key=lambda tag: str(tag.get("name") or "").casefold())
+            yield {"article": article, "tags": tags}
+
     def add_article_tags(self, article_id: str, tag_ids: List) -> None:
         if not article_id or not tag_ids:
             return

{feedsummary_core-2.2.0 → feedsummary_core-2.3.0}/src/feedsummary_core/persistence/SqliteStore.py

@@ -36,6 +36,7 @@ import json
 import logging
 import sqlite3
 import time
+from collections.abc import Iterator
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -434,6 +435,26 @@ class SqliteStore:
         finally:
             con.close()
 
+    def iter_articles(self, limit: Optional[int] = None) -> Iterator[Dict[str, Any]]:
+        """Yield every article oldest-first without the list API's default cap."""
+        con = self._connect()
+        try:
+            query = """
+                SELECT doc_json
+                FROM articles
+                ORDER BY COALESCE(published_ts, fetched_at, 0) ASC
+            """
+            params: Tuple[Any, ...] = ()
+            if limit is not None and int(limit) > 0:
+                query += " LIMIT ?"
+                params = (int(limit),)
+            for row in con.execute(query, params):
+                article = _json_loads(row["doc_json"]) or {}
+                if isinstance(article, dict):
+                    yield article
+        finally:
+            con.close()
+
     def list_articles_by_filter(
         self,
         *,

{feedsummary_core-2.2.0 → feedsummary_core-2.3.0}/src/feedsummary_core/persistence/TinyDbStore.py

@@ -34,6 +34,7 @@ from __future__ import annotations
 
 import logging
 import time
+from collections.abc import Iterator
 from typing import Any, Dict, List, Optional, Set, Tuple
 
 from tinydb import Query, TinyDB
@@ -118,6 +119,19 @@ class TinyDBStore:
         out.sort(key=lambda r: int(r.get("published_ts") or r.get("fetched_at") or 0))
         return out[:limit]
 
+    def iter_articles(self, limit: Optional[int] = None) -> Iterator[Dict[str, Any]]:
+        """Yield every article oldest-first without the list API's default cap."""
+        db = self._db()
+        try:
+            articles = [dict(row) for row in db.table("articles")]
+        finally:
+            db.close()
+        articles.sort(
+            key=lambda row: int(row.get("published_ts") or row.get("fetched_at") or 0)
+        )
+        maximum = int(limit) if limit is not None and int(limit) > 0 else None
+        yield from articles[:maximum]
+
     def list_articles_by_filter(
         self,
         *,

{feedsummary_core-2.2.0 → feedsummary_core-2.3.0}/src/feedsummary_core/persistence/__init__.py

@@ -33,6 +33,7 @@
 from __future__ import annotations
 
 import os
+from collections.abc import Iterator
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Protocol
 from feedsummary_core.persistence.CleanUpPolicy import CleanupPolicy
@@ -67,6 +68,8 @@ class NewsStore(Protocol):
 
     def list_articles(self, limit: int = 2000) -> List[Dict[str, Any]]: ...
 
+    def iter_articles(self, limit: Optional[int] = None) -> Iterator[Dict[str, Any]]: ...
+
     def list_articles_by_filter(
         self,
         *,
@@ -115,6 +118,13 @@ class NewsStore(Protocol):
 
     def get_all_tags(self) -> List[Dict[str, Any]]: ...
 
+    def iter_articles_with_tags(
+        self,
+        *,
+        categories: Optional[List[str]] = None,
+        limit: Optional[int] = None,
+    ): ...
+
     def add_article_tags(self, article_id: str, tag_ids: List) -> None: ...
 
     def add_tag_to_article(self, article_id: str, tag_id: int) -> bool: ...
@@ -220,6 +230,7 @@ def create_store(cfg: Dict[str, Any]) -> NewsStore:
             database=cfg.get("database") or cfg.get("database_name") or "feedsummary",
             client=cfg.get("client"),
             connect_timeout_ms=int(cfg.get("connect_timeout_ms", 5000)),
+            initialize_schema=bool(cfg.get("initialize_schema", True)),
         )  # type: ignore
 
     raise ValueError(f"Unsupported store provider: {provider}")