bertrend 0.4.1__tar.gz → 0.4.3__tar.gz

{bertrend-0.4.1 → bertrend-0.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bertrend
-Version: 0.4.1
+Version: 0.4.3
 Summary: BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM
 License: LICENSE.md
 License-File: AUTHORS.txt
@@ -12,74 +12,68 @@ Maintainer: Jérôme Picault
 Maintainer-email: jerome.picault@rte-france.com
 Requires-Python: >=3.12,<4.0
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
-Requires-Dist: APScheduler (>=3.11.0)
-Requires-Dist: PyJWT (>=2.10.1)
-Requires-Dist: Pyqtree (==1.0.0)
-Requires-Dist: SQLAlchemy (>=2.0.40)
+Requires-Dist: APScheduler (>=3.11.2)
+Requires-Dist: PyJWT (>=2.11.0)
+Requires-Dist: SQLAlchemy (>=2.0.46)
 Requires-Dist: aio-pika (>=9.5.5)
-Requires-Dist: arxiv (>=2.2.0)
-Requires-Dist: bertopic (==0.16.2)
-Requires-Dist: black (>=24.10.0)
-Requires-Dist: coverage (>=7.6.9)
-Requires-Dist: cron-descriptor (>=1.4.5)
-Requires-Dist: dask (==2024.12.0)
-Requires-Dist: datamapplot (==0.3.0)
-Requires-Dist: dateparser (>=1.2.0)
-Requires-Dist: dill (>=0.4.0)
-Requires-Dist: fastapi (>=0.120.0)
-Requires-Dist: feedparser (>=6.0.11)
+Requires-Dist: arxiv (>=2.4.0)
+Requires-Dist: bertopic (>=0.16.2)
+Requires-Dist: coverage (>=7.13.2)
+Requires-Dist: cron-descriptor (>=2.0.6)
+Requires-Dist: datamapplot (>=0.6.4)
+Requires-Dist: dateparser (>=1.2.2)
+Requires-Dist: ddgs (>=9.10.0)
+Requires-Dist: dill (>=0.4.1)
+Requires-Dist: fastapi (>=0.128.0)
+Requires-Dist: feedparser (>=6.0.12)
 Requires-Dist: folium (>=0.20.0)
-Requires-Dist: gensim (>=4.3.3)
+Requires-Dist: gensim (>=4.4.0)
 Requires-Dist: geopy (>=2.4.1)
-Requires-Dist: google-api-python-client (>=2.148.0)
-Requires-Dist: google-auth-oauthlib (>=1.2.1)
+Requires-Dist: google-api-python-client (>=2.189.0)
+Requires-Dist: google-auth-oauthlib (>=1.2.4)
 Requires-Dist: googlenewsdecoder (>=0.1.7)
-Requires-Dist: goose3 (>=3.1.19)
-Requires-Dist: hdbscan (>=0.8.40)
+Requires-Dist: goose3 (>=3.1.21)
+Requires-Dist: hdbscan (>=0.8.41)
 Requires-Dist: ipython (>=8.28.0)
-Requires-Dist: joblib (>=1.4.2)
+Requires-Dist: joblib (>=1.5.3)
 Requires-Dist: jsonlines (>=4.0.0)
-Requires-Dist: langchain-core (>=0.3.59)
+Requires-Dist: langchain-core (>=1.2.9)
 Requires-Dist: langdetect (>=1.0.9)
 Requires-Dist: loguru (>=0.7.3)
-Requires-Dist: lxml-html-clean (>=0.2.2)
-Requires-Dist: markdown (>=3.7)
+Requires-Dist: lxml-html-clean (>=0.4.3)
+Requires-Dist: markdown (>=3.10.1)
 Requires-Dist: newscatcher-sdk (>=1.4.0)
 Requires-Dist: newspaper4k (>=0.9.3.1)
-Requires-Dist: nltk (>=3.9.1)
-Requires-Dist: openai (>=1.77.0)
-Requires-Dist: openai-agents[litellm] (>=0.4.0)
+Requires-Dist: nltk (>=3.9.2)
+Requires-Dist: openai (>=2.18.0)
+Requires-Dist: openai-agents[litellm] (>=0.8.3)
 Requires-Dist: openpyxl (>=3.1.5)
-Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (==1.25.0)
-Requires-Dist: pandas (>=2.2.2)
+Requires-Dist: pandas (<3.0.0)
 Requires-Dist: pika (>=1.3.2)
-Requires-Dist: plotly (>=5.24.1)
-Requires-Dist: plotly-resampler (>=0.10.0)
-Requires-Dist: psutil (>=7.2.1)
-Requires-Dist: pyarrow (>=21.0.0)
+Requires-Dist: plotly (>=6.5.2)
+Requires-Dist: plotly-resampler (>=0.11.0)
+Requires-Dist: psutil (>=7.2.2)
 Requires-Dist: pygooglenews (>=0.1.3)
-Requires-Dist: pylabeladjust (==0.1.13)
-Requires-Dist: pytest (>=8.4.0)
+Requires-Dist: pytest
 Requires-Dist: pytest-asyncio
-Requires-Dist: python-dotenv (>=1.1.1)
+Requires-Dist: python-dotenv (>=1.2.1)
 Requires-Dist: requests (>=2.32.3)
-Requires-Dist: ruff (>=0.14.14)
-Requires-Dist: scikit-learn (>=1.5.2)
+Requires-Dist: ruff (>=0.15.0)
+Requires-Dist: scikit-learn (>=1.8.0)
 Requires-Dist: seaborn (>=0.13.2)
-Requires-Dist: sentence-transformers (==3.0.1)
-Requires-Dist: sentencepiece (>=0.2.0)
-Requires-Dist: starlette (>=0.46.2)
-Requires-Dist: streamlit (>=1.51.0)
+Requires-Dist: sentence-transformers (>=5.2.2)
+Requires-Dist: starlette (>=0.52.1)
+Requires-Dist: streamlit (>=1.54.0)
 Requires-Dist: streamlit-code-editor (>=0.1.22)
 Requires-Dist: streamlit-folium (>=0.26.1)
 Requires-Dist: supervisor (>=4.3.0)
 Requires-Dist: thefuzz (>=0.22.1)
-Requires-Dist: tiktoken (>=0.8.0)
-Requires-Dist: tldextract (>=2.0.1)
-Requires-Dist: torch (>=2.7.0)
-Requires-Dist: tqdm (>=4.67.1)
-Requires-Dist: typer (>=0.16.0)
-Requires-Dist: umap-learn (>=0.5.7)
+Requires-Dist: tiktoken (>=0.12.0)
+Requires-Dist: tldextract (>=5.3.1)
+Requires-Dist: torch (>=2.10.0)
+Requires-Dist: tqdm (>=4.67.3)
+Requires-Dist: typer (>=0.21.1)
+Requires-Dist: umap-learn (>=0.5.11)
 Project-URL: Bug Tracker, https://github.com/rte-france/BERTrend/issues
 Project-URL: Documentation, https://github.com/rte-france/BERTrend
 Project-URL: Homepage, https://github.com/rte-france/BERTrend

bertrend-0.4.3/bertrend/bertrend_apps/config/feeds/feed_deep_research_example.toml

@@ -0,0 +1,9 @@
+[data-feed]
+id = "feed_deep_research_example"
+query = "artificial intelligence energy grid"
+provider = "deep_research"
+max_results = 3
+number_of_days = 7
+feed_dir_path = "feed_deep_research"
+language = "en"
+update_frequency = "0 6 * * 1"

{bertrend-0.4.1 → bertrend-0.4.3}/bertrend/bertrend_apps/data_provider/arxiv_provider.py

@@ -27,8 +27,6 @@ NUM_RETRIES = 10
 query = "cat:cs.AI"
 max_results = float("inf")
 
-### Parameters Semantic Scholar ###
-SEMANTIC_SCHOLAR_API_KEY = os.getenv("SEMANTIC_SCHOLAR_API_KEY")
 
 PAPERS_PER_REQUEST = 500  # should not be more than 500
 SEMANTIC_SCHOLAR_FIELDS = "title,abstract,citationCount,publicationDate,url"
@@ -130,14 +128,15 @@ class ArxivProvider(DataProvider):
     @wait(1)
     def _request_semantic_scholar_chunk(self, chunk: list[dict]):
         """Get information from semantic scholar API per batch of articles IDs"""
-        if not SEMANTIC_SCHOLAR_API_KEY:
+        api_key = os.getenv("SEMANTIC_SCHOLAR_API_KEY")
+        if not api_key:
             raise ValueError("SEMANTIC_SCHOLAR_API_KEY environment variable is not set")
         ids_list = ["URL:" + entry["id"] for entry in chunk]
         with requests.post(
             "https://api.semanticscholar.org/graph/v1/paper/batch",
             params={"fields": SEMANTIC_SCHOLAR_FIELDS},
             json={"ids": ids_list},
-            headers={"x-api-key": SEMANTIC_SCHOLAR_API_KEY},
+            headers={"x-api-key": api_key},
         ) as response:
             return [item for item in response.json() if item is not None]
 

{bertrend-0.4.1 → bertrend-0.4.3}/bertrend/bertrend_apps/data_provider/data_provider_utils.py

@@ -14,6 +14,9 @@ from bertrend.bertrend_apps.common.date_utils import daterange
 from bertrend.bertrend_apps.data_provider.arxiv_provider import ArxivProvider
 from bertrend.bertrend_apps.data_provider.atom_feed_provider import ATOMFeedProvider
 from bertrend.bertrend_apps.data_provider.bing_news_provider import BingNewsProvider
+from bertrend.bertrend_apps.data_provider.deep_research_provider import (
+    DeepResearchProvider,
+)
 from bertrend.bertrend_apps.data_provider.google_news_provider import GoogleNewsProvider
 from bertrend.bertrend_apps.data_provider.newscatcher_provider import (
     NewsCatcherProvider,
@@ -27,6 +30,7 @@ PROVIDERS = {
     "google": GoogleNewsProvider,
     "bing": BingNewsProvider,
     "newscatcher": NewsCatcherProvider,
+    "deep_research": DeepResearchProvider,
 }
 
 
@@ -145,8 +149,11 @@ def scrape_feed_from_config(
 
     # Generate a query file
     with tempfile.NamedTemporaryFile() as query_file:
-        if (
-            provider == "arxiv" or provider == "atom" or provider == "rss"
+        if provider in (
+            "arxiv",
+            "atom",
+            "rss",
+            "deep_research",
         ):  # already returns batches
             scrape(
                 keywords=keywords,