bertrend 0.3.8__tar.gz → 0.3.9__tar.gz

{bertrend-0.3.8 → bertrend-0.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bertrend
-Version: 0.3.8
+Version: 0.3.9
 Summary: BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM
 License: LICENSE.md
 Keywords: topics,BERTopic,transformers,weak signals,LLM
@@ -65,6 +65,7 @@ Requires-Dist: tldextract (>=2.0.1)
 Requires-Dist: tokenizers (<0.20.0)
 Requires-Dist: torch (>=2.7.0)
 Requires-Dist: tqdm (>=4.67.1)
+Requires-Dist: typer (>=0.16.0)
 Requires-Dist: umap-learn (>=0.5.7)
 Project-URL: Bug Tracker, https://github.com/rte-france/BERTrend/issues
 Project-URL: Documentation, https://github.com/rte-france/BERTrend

{bertrend-0.3.8 → bertrend-0.3.9}/bertrend_apps/config/feeds/arxiv_feed.toml

@@ -1,6 +1,6 @@
 [data-feed]
 id = "feed_arxiv"
-query = "cat:cs.CL"
+query = "cat:cs.AI"
 provider = "arxiv"
 # maximum number of results *per day*
 max_results = 1000

{bertrend-0.3.8 → bertrend-0.3.9}/bertrend_apps/data_provider/arxiv_provider.py

@@ -22,7 +22,7 @@ DELAY_SECONDS = 3
 NUM_RETRIES = 10
 
 ### Request parameters ###
-query = "cat:cs.CL"
+query = "cat:cs.AI"
 max_results = float("inf")
 
 ### Parameters Semantic Scholar ###
@@ -54,6 +54,7 @@ class ArxivProvider(DataProvider):
         before: str,
         max_results: int,
         language: str = None,
+        add_citations_count: bool = False,
     ) -> list[dict]:
         """Requests the news data provider, collects a set of URLs to be parsed, return results as json lines.
 
@@ -97,8 +98,12 @@ class ArxivProvider(DataProvider):
             <= datetime.strptime(res["timestamp"], DATE_FORMAT_YYYYMMDD_TIME)
             <= end
         ]
-        # add citations count
-        return self.add_citations_count(results)
+
+        if add_citations_count:
+            # add citations count
+            return self.add_citations_count(results)
+        else:
+            return results
 
     def _parse_entry(self, entry: arxiv.Result) -> dict | None:
         """Parses a Arxiv entry"""
@@ -115,6 +120,8 @@ class ArxivProvider(DataProvider):
                 "summary": entry.summary,
                 "text": entry.summary,
                 "timestamp": published,
+                "url": entry.entry_id,
+                "link": entry.entry_id,
             }
         except Exception as e:
             logger.error(str(e) + f"\nError occurred with parsing of: {entry}")

{bertrend-0.3.8 → bertrend-0.3.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "bertrend"
-version = "0.3.8"
+version = "0.3.9"
 description = "BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM"
 license = "LICENSE.md"
 readme = "README.md"
@@ -62,6 +62,7 @@ dependencies = [
     "torch (>=2.7.0)",
     "tqdm (>=4.67.1)",
     "tiktoken (>=0.8.0)",
+    "typer (>=0.16.0)",
     "tokenizers (<0.20.0)",
     "umap-learn (>=0.5.7)"
 ]

bertrend-0.3.8/bertrend_apps/config/newsletters/arxiv_newsletter_from_scratch.toml

@@ -1,39 +0,0 @@
-######## NEWSLETTER CONFIG ########
-[newsletter]
-id = "newsletter_arxiv"
-title = "Arxiv Newsletter"
-top_n_topics = 6
-top_n_docs = 4
-output_format = "html"
-# output_directory is a relative path - base_output_path/output_directory; the base_output_path is controlled by the application
-output_directory = "newsletter_arxiv"
-summarizer_class = "bertrend.services.summary.chatgpt_summarizer.GPTSummarizer"
-prompt_language = "en"
-# To automate the newsletters generation, e.g. “At 22:00 on every day-of-week from Monday through Friday.”
-update_frequency = "30 1 * * 5"
-recipients = "$NEWSLETTER_DEFAULT_RECIPIENTS"
-improve_topic_description = true
-# 'document' to summarize top n documents independently or 'topic' to generate a single topic summary
-summary_mode = "document"
-#openai_model_name =
-
-######## LEARNING STRATEGY ########
-[learning_strategy]
-# Choose among:
-# - learn_from_scratch : uses all available data from feed to create the model
-# - learn_from_last: uses only the last feed data to create the model
-# - inference_only: do not retrain model; reuse existing bertopic model if available,
-#   otherwise, fallback to learn_from_scratch for the first run
-learning_strategy = "learn_from_scratch"
-split_data_by_paragraphs = "no"
-# model saving path (empty = do not save model)
-bertopic_model_path = "models/arxiv"
-
-######## TOPIC MODEL CONFIG ########
-[embedding_service]
-# local embedding model to be used
-model_name = "OrdalieTech/Solon-embeddings-base-0.1"
-
-[bertopic_parameters]
-language = "English"
-