bertrend 0.3.4__tar.gz → 0.3.6__tar.gz

{bertrend-0.3.4 → bertrend-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bertrend
-Version: 0.3.4
+Version: 0.3.6
 Summary: BERTrend analyses topic evolution over time using state-of-the-art transformer models: topic modelling and LLM
 License: LICENSE.md
 Keywords: topics,BERTopic,transformers,weak signals,LLM
@@ -11,6 +11,7 @@ Maintainer-email: jerome.picault@rte-france.com
 Requires-Python: >=3.12,<4.0
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Provides-Extra: apps
+Requires-Dist: PyJWT (>=2.10.1)
 Requires-Dist: Pyqtree (==1.0.0)
 Requires-Dist: arxiv (>=2.2.0) ; extra == "apps"
 Requires-Dist: bertopic (==0.16.2)
@@ -48,7 +49,7 @@ Requires-Dist: plotly (>=5.24.1)
 Requires-Dist: plotly-resampler (>=0.10.0)
 Requires-Dist: pygooglenews (>=0.1.3) ; extra == "apps"
 Requires-Dist: pylabeladjust (==0.1.13)
-Requires-Dist: pytest (>=8.3.3)
+Requires-Dist: pytest (>=8.4.0)
 Requires-Dist: requests (>=2.32.3)
 Requires-Dist: scikit-learn (>=1.5.2)
 Requires-Dist: scipy (==1.12)
@@ -56,7 +57,7 @@ Requires-Dist: seaborn (>=0.13.2)
 Requires-Dist: sentence-transformers (==3.0.1)
 Requires-Dist: sentencepiece (>=0.2.0)
 Requires-Dist: starlette (>=0.46.2)
-Requires-Dist: streamlit (>=1.45.0)
+Requires-Dist: streamlit (>=1.45.1)
 Requires-Dist: streamlit-code-editor (>=0.1.22)
 Requires-Dist: thefuzz (>=0.22.1)
 Requires-Dist: tiktoken (>=0.8.0)
@@ -259,7 +260,7 @@ We recommend using a remote embedding service (this is now the default config) a
 
 The change of configuration can be done in: [services_default_config.toml](bertrend/config/services_default_config.toml).
 
-An example of a very simple implementation to deploy a separate embedding service can be found here: https://github.com/rte-france/wattelse/tree/main/wattelse/api/embedding. (this implementation is compatible with our code)
+An example of a very simple implementation to deploy a separate embedding service can be found here: `bertrend/services/embedding_server`
 
 
 ![parameters](docs/images/parameters.png)

{bertrend-0.3.4 → bertrend-0.3.6}/README.md

@@ -186,7 +186,7 @@ We recommend using a remote embedding service (this is now the default config) a
 
 The change of configuration can be done in: [services_default_config.toml](bertrend/config/services_default_config.toml).
 
-An example of a very simple implementation to deploy a separate embedding service can be found here: https://github.com/rte-france/wattelse/tree/main/wattelse/api/embedding. (this implementation is compatible with our code)
+An example of a very simple implementation to deploy a separate embedding service can be found here: `bertrend/services/embedding_server`
 
 
 ![parameters](docs/images/parameters.png)

{bertrend-0.3.4 → bertrend-0.3.6}/bertrend/config/bertopic_default_config.toml

@@ -30,7 +30,7 @@ prediction_data = true
 
 # CountVectorizer: https://scikit-learn.org/1.5/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html
 [vectorizer_model]
-ngram_range = [1, 1]
+ngram_range = [1, 2]
 stop_words = true # If true, will check `language` parameter and load associated stopwords file
 min_df = 2
 

bertrend-0.3.6/bertrend/demos/start_topic_analysis_app.sh

@@ -0,0 +1,14 @@
+#
+# Copyright (c) 2024, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of BERTrend.
+#
+
+
+# Set logs directory and create if not exists
+export BERTREND_LOGS_DIR=$BERTREND_BASE_DIR/logs/bertrend
+mkdir -p $BERTREND_LOGS_DIR
+
+echo "Starting BERTrend Topic Analysis demo"
+cd `pwd`/topic_analysis && CUDA_VISIBLE_DEVICES=0 streamlit run app.py 2>&1 | tee -a $BERTREND_LOGS_DIR/topic_analysis_demo.log

bertrend-0.3.6/bertrend/demos/start_weak_signals_app.sh

@@ -0,0 +1,14 @@
+#
+# Copyright (c) 2024, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of BERTrend.
+#
+
+
+# Set logs directory and create if not exists
+export BERTREND_LOGS_DIR=$BERTREND_BASE_DIR/logs/bertrend
+mkdir -p $BERTREND_LOGS_DIR
+
+echo "Starting BERTrend Weak Signal Analysis demo"
+cd `pwd`/weak_signals && CUDA_VISIBLE_DEVICES=0 streamlit run app.py 2>&1 | tee -a $BERTREND_LOGS_DIR/weak_signals_demo.log

bertrend-0.3.6/bertrend/demos/topic_analysis/.streamlit/config.toml

@@ -0,0 +1,6 @@
+[theme]
+primaryColor = "royalblue"
+baseFontSize = 13
+
+[server]
+port = 8083

{bertrend-0.3.4 → bertrend-0.3.6}/bertrend/demos/topic_analysis/demo_pages/explore_topics.py

@@ -291,7 +291,9 @@ def _display_topic_description(filtered_df: pd.DataFrame):
     ):
         with st.spinner(translate("generating_description")):
             language_code = (
-                "fr" if SessionStateManager.get("language") == "French" else "en"
+                "fr"
+                if SessionStateManager.get("internationalization_language") == "French"
+                else "en"
             )
             gpt_description = generate_topic_description(
                 st.session_state["topic_model"],

{bertrend-0.3.4 → bertrend-0.3.6}/bertrend/demos/topic_analysis/demo_pages/newsletters_generation.py

@@ -14,6 +14,7 @@ from bertrend.demos.demos_utils.state_utils import (
     save_widget_state,
     restore_widget_state,
     register_multiple_widget,
+    SessionStateManager,
 )
 from bertrend.services.summary.abstractive_summarizer import AbstractiveSummarizer
 from bertrend.services.summary.chatgpt_summarizer import GPTSummarizer
@@ -48,6 +49,12 @@ def generate_newsletter_wrapper(
         else st.session_state["newsletter_nb_docs"]
     )
 
+    language_code = (
+        "fr"
+        if SessionStateManager.get("internationalization_language") == "French"
+        else "en"
+    )
+
     return generate_newsletter(
         topic_model=st.session_state["topic_model"],
         df=df,
@@ -60,6 +67,7 @@ def generate_newsletter_wrapper(
             st.session_state["summarizer_classname"]
         ],
         summary_mode=st.session_state["summary_mode"],
+        prompt_language=language_code,
     )
 
 

bertrend-0.3.6/bertrend/demos/weak_signals/.streamlit/config.toml

@@ -0,0 +1,6 @@
+[theme]
+primaryColor = "royalblue"
+baseFontSize = 13
+
+[server]
+port = 8084

bertrend-0.3.6/bertrend/services/embedding_server/README.md

@@ -0,0 +1,31 @@
+# How to use the embedding server
+
+## Launch the embedding server
+
+- check the file `config/default_config.toml`
+
+- run the embedding server
+
+```
+python start.py
+```
+
+## Check the embedding server
+Ensure the API is launched.
+
+```python
+from bertrend.services.embedding_client import EmbeddingAPIClient
+
+embedding_service_endpoint = "https://yourservice:1234"
+
+# Initialize the API
+api = EmbeddingAPIClient(embedding_service_endpoint)
+
+# Get embedding model name
+print(api.get_api_model_name())
+
+# Transform text to embedding
+text = "Hello, world!"
+embedding = api.embed_query(text)
+print(embedding)
+```

bertrend-0.3.6/bertrend/services/embedding_server/config/default_config.toml

@@ -0,0 +1,7 @@
+host = "0.0.0.0"
+port = 6464
+model_name = "OrdalieTech/Solon-embeddings-large-0.1"
+#model_name=dangvantuan/sentence-camembert-large
+#model_name=antoinelouis/biencoder-camembert-base-mmarcoFR
+number_workers = 2
+cuda_visible_devices = "0"

bertrend-0.3.6/bertrend/services/embedding_server/config/settings.py

@@ -0,0 +1,34 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+import os
+from pathlib import Path
+import tomllib
+from pydantic_settings import BaseSettings
+
+DEFAULT_CONFIG_FILE = Path(__file__).parent / "default_config.toml"
+
+
+class EmbeddingAPIConfig(BaseSettings):
+    host: str
+    port: int
+    model_name: str
+    number_workers: int
+    cuda_visible_devices: str
+
+
+# Use environment variable to specify which config to load
+def get_config() -> EmbeddingAPIConfig:
+    """
+    Return the configuration for the embedding API.
+    The configuration is loaded from a toml file.
+    The path to the toml file can be specified using the environment variable `EMBEDDING_API_CONFIG_FILE`.
+    If the environment variable is not set, the default configuration file is used.
+    """
+    config_file = os.environ.get("EMBEDDING_API_CONFIG_FILE", None)
+    if config_file is None:
+        config_file = DEFAULT_CONFIG_FILE
+    with open(config_file, "rb") as f:
+        return EmbeddingAPIConfig(**tomllib.load(f))

bertrend-0.3.6/bertrend/services/embedding_server/main.py

@@ -0,0 +1,14 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+from fastapi import FastAPI
+
+from bertrend.services.embedding_server.routers import embeddings, info, authentication
+
+app = FastAPI()
+
+app.include_router(info.router, tags=["Info"])
+app.include_router(embeddings.router, tags=["Embedding"])
+app.include_router(authentication.router, tags=["Authentication"])

bertrend-0.3.6/bertrend/services/embedding_server/models.py

@@ -0,0 +1,12 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+# Pydantic class for FastAPI typing control
+from pydantic import BaseModel
+
+
+class InputText(BaseModel):
+    text: str | list[str]
+    show_progress_bar: bool

bertrend-0.3.6/bertrend/services/embedding_server/routers/authentication.py

@@ -0,0 +1,45 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+from typing import Annotated
+
+from fastapi import Depends, APIRouter, Security
+from fastapi.security import OAuth2PasswordRequestForm
+
+from bertrend.services.embedding_server.security import (
+    ADMIN,
+    Token,
+    TokenData,
+    get_token,
+    list_registered_clients,
+    get_current_client,
+    view_rate_limits,
+)
+
+router = APIRouter()
+
+
+@router.post("/token")
+async def login_for_access_token(
+    form_data: Annotated[OAuth2PasswordRequestForm, Depends()],
+) -> Token:
+    return get_token(form_data)
+
+
+@router.get(
+    "/list_registered_clients", summary="List registered clients (requires admin scope)"
+)
+async def list_clients(
+    current_client: TokenData = Security(get_current_client, scopes=[ADMIN])
+):
+    return list_registered_clients()
+
+
+@router.get(
+    "/rate-limits", summary="Check current rate limit usage (requires admin scope)"
+)
+async def get_rate_limits(
+    current_client: TokenData = Security(get_current_client, scopes=[ADMIN]),
+):
+    return view_rate_limits()

bertrend-0.3.6/bertrend/services/embedding_server/routers/embeddings.py

@@ -0,0 +1,39 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+from fastapi import APIRouter, Security
+from loguru import logger
+from sentence_transformers import SentenceTransformer
+
+from bertrend.services.embedding_server.security import (
+    TokenData,
+    get_current_client,
+    FULL_ACCESS,
+)
+
+from bertrend.services.embedding_server.config.settings import get_config
+from bertrend.services.embedding_server.models import InputText
+
+# Load the configuration
+CONFIG = get_config()
+
+# Load embedding model
+logger.debug(f"Loading embedding model : {CONFIG.model_name}")
+EMBEDDING_MODEL = SentenceTransformer(CONFIG.model_name, trust_remote_code=True)
+
+# Fix max model length error
+if EMBEDDING_MODEL.max_seq_length == 514:
+    EMBEDDING_MODEL.max_seq_length = 512
+router = APIRouter()
+
+
+@router.post("/encode", summary="Embed data (requires full_access scope)")
+def embed(
+    input: InputText,
+    current_client: TokenData = Security(get_current_client, scopes=[FULL_ACCESS]),
+):
+    logger.debug(f"Request by: {current_client.client_id}")
+    emb = EMBEDDING_MODEL.encode(input.text, show_progress_bar=input.show_progress_bar)
+    return {"embeddings": emb.tolist()}

bertrend-0.3.6/bertrend/services/embedding_server/routers/info.py

@@ -0,0 +1,29 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+from fastapi import APIRouter
+
+from bertrend.services.embedding_server.config.settings import get_config
+
+# Load the configuration
+CONFIG = get_config()
+
+
+router = APIRouter()
+
+
+@router.get("/health")
+def health():
+    return {"status": "ok"}
+
+
+@router.get("/model_name")
+def get_model_name():
+    return CONFIG.model_name
+
+
+@router.get("/num_workers")
+def get_num_workers():
+    return CONFIG.number_workers