bertrend 0.3.1__py3-none-any.whl

bertrend_apps/exploration/dataviz/Main_page.py

@@ -0,0 +1,10 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of Wattelse, a NLP application suite.
+
+import streamlit as st
+
+st.set_page_config(layout="wide")
+
+st.write("Select a page")

bertrend_apps/exploration/dataviz/__init__.py

@@ -0,0 +1,4 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of Wattelse, a NLP application suite.

bertrend_apps/exploration/dataviz/pages/1_Data_feeds.py

@@ -0,0 +1,105 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of Wattelse, a NLP application suite.
+import math
+
+import pandas as pd
+import streamlit as st
+import pydeck as pdk
+
+from bertrend.utils.data_loading import TIMESTAMP_COLUMN
+from bertrend_apps.exploration.geolocalization.entity_utils import (
+    geojson_to_dataframe,
+    geolocalize_data,
+)
+from bertrend_apps.exploration.geolocalization.spacy.utils import load_nlp
+
+GEODATA_COLUM = "geodata"
+
+
+def get_coords(geojson_data):
+    geo_df = geojson_to_dataframe(geojson_data)
+    # selection of overall coordinates for the whole doc
+    # TODO: heuristics to be defined, basic example below (doc location represented by the one of its first entity)
+    if len(geo_df) > 0:
+        return (
+            geo_df.iloc[0]["latitude"],
+            geo_df.iloc[0]["longitude"],
+            geo_df.iloc[0]["label"],
+        )
+    else:
+        return math.nan, math.nan, math.nan
+
+
+def plot_geojson_on_map(df: pd.DataFrame):
+    # Use st.map to plot GeoJSON data
+    # Convert the GeoDataFrame to a DataFrame
+    df[["latitude", "longitude", "label"]] = (
+        df[GEODATA_COLUM].apply(get_coords).apply(pd.Series)
+    )
+    print(df)
+    st.map(df.dropna())
+    # st.pydeck_chart(create_pydeck_chart(df.dropna()))
+
+
+def create_pydeck_chart(geo_dataframe: pd.DataFrame):
+    # Create a PyDeck layer with lat,lon  data
+    layer = pdk.Layer(
+        "ScatterplotLayer",
+        data=geo_dataframe,
+        get_position="[longitute, latitude]",
+        get_color="[255, 0, 0]",
+        get_radius=1000,
+        pickable=True,
+    )
+
+    # Create a PyDeck view
+    view_state = pdk.ViewState(
+        longitude=geo_dataframe["longitude"].mean(),
+        latitude=geo_dataframe["latitude"].mean(),
+        zoom=5,
+    )
+
+    # Create a PyDeck deck
+    deck = pdk.Deck(
+        map_style="mapbox://styles/mapbox/light-v9",
+        layers=[layer],
+        initial_view_state=view_state,
+    )
+
+    return deck
+
+
+@st.cache_resource
+def load_nlp_wrapper():
+    return load_nlp()
+
+
+nlp = load_nlp_wrapper()
+
+st.title("Browse data")
+
+# Load selected DataFrame
+choose_data(DATA_DIR, ["*.csv", "*.jsonl*"])
+
+df = (
+    load_data_wrapper(
+        f"{st.session_state['data_folder']}/{st.session_state['data_name']}"
+    )
+    .sort_values(by=TIMESTAMP_COLUMN, ascending=False)
+    .reset_index(drop=True)
+).reset_index()
+
+with st.expander("Data content"):
+    st.dataframe(df, hide_index=True)
+
+# data overview
+data_overview(df)
+
+# Géolocalization of data
+if st.button("Geolocalize data"):
+    with st.expander("Data geolocalization"):
+        with st.spinner("Extraction of locations from text"):
+            df[GEODATA_COLUM] = geolocalize_data(nlp, df)
+            plot_geojson_on_map(df)

bertrend_apps/exploration/geolocalization/__init__.py

@@ -0,0 +1,4 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of Wattelse, a NLP application suite.

bertrend_apps/exploration/geolocalization/entity_utils.py

@@ -0,0 +1,118 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of Wattelse, a NLP application suite.
+
+import pandas as pd
+
+from SPARQLWrapper import SPARQLWrapper, JSON
+import json
+import os
+from loguru import logger
+from tqdm import tqdm
+
+TEXT_COLUMN = "text"
+
+
+def geolocalize_data(nlp, df: pd.DataFrame) -> pd.DataFrame:
+    """Creates geojson data associated to text entries of the dataframe"""
+    text_col = "processed_text" if "processed_text" in df.columns else TEXT_COLUMN
+    logger.info("Geolocalizing data...")
+    geojson_data_list = []
+
+    for doc in tqdm(nlp.pipe(df[text_col], n_process=os.cpu_count() - 1)):
+        # DBPedia resources
+        resources = doc._.dbpedia_raw_result.get("Resources")
+        features = []
+        if resources:
+            for r in resources:
+                if "DBpedia:Location" in r["@types"].split(","):
+                    features += get_geojson_features_from_dbpedia_resource(r["@URI"])
+        geojson_data = {"type": "FeatureCollection", "features": features}
+        geojson_data_list.append(geojson_data)
+        logger.debug(geojson_data)
+    return geojson_data_list
+
+
+def geojson_to_dataframe(geojson_data) -> pd.DataFrame:
+    features = geojson_data["features"]
+
+    # Initialize lists to store latitude, longitude, and label data
+    latitudes = []
+    longitudes = []
+    labels = []
+
+    # Extract data from GeoJSON features
+    for feature in features:
+        geometry = feature["geometry"]
+        properties = feature["properties"]
+
+        # Extract latitude and longitude
+        latitude, longitude = geometry["coordinates"]
+        latitudes.append(latitude)
+        longitudes.append(longitude)
+
+        # Extract label (if available)
+        label = properties.get("label", None)
+        labels.append(label)
+
+    # Create a DataFrame
+    df = pd.DataFrame({"latitude": latitudes, "longitude": longitudes, "label": labels})
+
+    return df
+
+
+def get_geojson_features_from_dbpedia_resource(resource_uri):
+    # Define the DBpedia SPARQL endpoint
+    sparql_endpoint = "http://fr.dbpedia.org/sparql"
+
+    # Set up SPARQLWrapper
+    sparql = SPARQLWrapper(sparql_endpoint)
+
+    # Define the SPARQL query to retrieve GeoJSON data
+    sparql_query = f"""
+        PREFIX geo: <http://www.w3.org/2003/01/geo/wgs84_pos#>
+        SELECT DISTINCT ?subject ?lat ?long ?label
+        WHERE {{
+            <{resource_uri}> geo:lat ?lat ;
+                            geo:long ?long ;
+                            rdfs:label ?label .
+        }}
+    """
+
+    # Set the SPARQL query and request JSON format
+    sparql.setQuery(sparql_query)
+    sparql.setReturnFormat(JSON)
+
+    # Execute the SPARQL query
+    results = sparql.query().convert()
+
+    # Process the results and create GeoJSON
+    features = []
+    for result in results["results"]["bindings"]:
+        feature = {
+            "type": "Feature",
+            "geometry": {
+                "type": "Point",
+                "coordinates": [
+                    float(result["long"]["value"]),
+                    float(result["lat"]["value"]),
+                ],
+            },
+            "properties": {"label": result["label"]["value"]},
+        }
+        features.append(feature)
+        break  # to avoid similar entries (ex. "Berlin" in different languages)
+
+    return features
+
+
+if __name__ == "__main__":
+    # Specify the DBpedia resource URI
+    dbpedia_resource_uri = "http://fr.dbpedia.org/resource/Berlin"
+
+    # Get GeoJSON data for the specified resource
+    geojson_data = get_geojson_features_from_dbpedia_resource(dbpedia_resource_uri)
+
+    # Print the GeoJSON data
+    print(json.dumps(geojson_data, indent=2))

bertrend_apps/exploration/geolocalization/spacy/__init__.py

@@ -0,0 +1,4 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of Wattelse, a NLP application suite.

bertrend_apps/exploration/geolocalization/spacy/common_factory.py

@@ -0,0 +1,142 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of Wattelse, a NLP application suite.
+
+import re
+import unicodedata
+from typing import List, Union
+
+from loguru import logger
+from spacy import Language
+from spacy.tokens import Token
+
+from bertrend_apps.exploration.geolocalization.spacy.dbpedia import DBPediaEntityLinker
+
+RE_NORM_APOSTROPHE_DASH = re.compile(r"(-|l'|d')")
+
+
+@Language.factory("normalizer_component")
+def create_normalizer_component(nlp, name):
+    return NormalizerComponent()
+
+
+@Language.factory("tokenizer_component")
+def create_tokenizer_component(nlp, name):
+    return TokenizerComponent()
+
+
+@Language.factory(
+    "DBpedia_spotlight",
+    default_config={
+        "language_code": "fr",
+        "dbpedia_rest_endpoint": None,
+        "process": "annotate",
+        "confidence": 0.5,
+        "support": "",
+        "types": None,
+        "sparql": "",
+        "policy": "",
+        "span_group": "dbpedia_spotlight",
+        "overwrite_ents": True,
+    },
+)
+def dbpedia_spotlight_factory(
+    nlp: Language,
+    name: str,
+    language_code: str,
+    dbpedia_rest_endpoint: str,
+    process: str,
+    confidence: float,
+    support: str,
+    types: List[str],
+    sparql: str,
+    policy: str,
+    span_group: str,
+    overwrite_ents: bool,
+):
+    """Factory of the pipeline stage `dbpedia_spotlight`.
+    Parameters:
+    - `language_code`: which language to use for entity linking. Possible values are listed in EntityLinker.supported_languages. If the parameter is left as None, the language code is matched with the nlp object currently used.
+    - `dbpedia_rest_endpoint`: this needs to be configured if you want to use a different REST endpoint from the default `EntityLinker.base_url`. Example: `http://localhost:2222/rest` for a localhost server
+    - `process`: (REST API path) which of the processes to use from DBpedia Spotlight (see https://www.dbpedia-spotlight.org/api). The value can be 'annotate', 'spot' or 'candidates'
+    - `confidence`: (REST API parameter) confidence score for disambiguation / linking
+    - `support`: (REST API parameter) how prominent is this entity in Lucene Model, i.e. number of inlinks in Wikipedia
+    - `types`: (REST API parameter) types filter (Eg.DBpedia:Place)
+    - `sparql`: (REST API parameter) SPARQL filtering
+    - `policy`: (REST API parameter) (whitelist) select all entities that have the same type; (blacklist) - select all entities that have not the same type.
+    - `span_group`: which span group to write the entities to. By default the value is `dbpedia_spotlight` which writes to `doc.spans['dbpedia_spotlight']`
+    - `overwrite_ents`: if set to False, it won't overwrite `doc.ents` in cases of overlapping spans with current entities, and only produce the results in `doc.spans[span_group]. If it is True, it will move the entities from doc.ents into `doc.spans['ents_original']`
+    """
+    logger.debug(
+        f"dbpedia_spotlight_factory: {nlp}, language_code {language_code}, "
+        f"dbpedia_rest_endpoint {dbpedia_rest_endpoint}, process {process}, "
+        f"confidence {confidence}, support {support}, types {types}, "
+        f"sparql {sparql}, policy {policy}, overwrite_ents {overwrite_ents}"
+    )
+    # take the language code from the nlp object
+    nlp_lang_code = nlp.meta["lang"]
+    # language_code can override the language code from the nlp object
+    if not language_code:
+        language_code = nlp_lang_code
+    return DBPediaEntityLinker(
+        language_code,
+        dbpedia_rest_endpoint,
+        process,
+        confidence,
+        support,
+        types,
+        sparql,
+        policy,
+        span_group,
+        overwrite_ents,
+    )
+
+
+class NormalizerComponent:
+    """
+    Do some custom normalization steps
+    WARNING: This DOES NOT modify utils length (applied to .norm_ token attribute)
+    """
+
+    def __call__(self, doc):
+        for token in doc:
+            norm = normalize(token)
+            token.norm_ = norm
+        return doc
+
+
+class TokenizerComponent:
+    """Split tokens by apostrophes and dashes"""
+
+    def __call__(self, doc):
+        with doc.retokenize() as retokenizer:
+            for token in doc:
+                text = token.text
+                if len(text) > 1 and RE_NORM_APOSTROPHE_DASH.search(text):
+                    elts = RE_NORM_APOSTROPHE_DASH.split(text)
+                    elts = list(filter(bool, elts))
+                    heads = [(token, i) for i in range(len(elts))]
+                    retokenizer.split(token, elts, heads=heads)
+        return doc
+
+
+def strip_accents(s: str) -> str:
+    # From https://stackoverflow.com/questions/517923/what-is-the-best-way-to-remove-accents-normalize-in-a-python-unicode-string
+    return "".join(
+        c for c in unicodedata.normalize("NFD", s) if unicodedata.category(c) != "Mn"
+    )
+
+
+def normalize(s: Union[str, Token]) -> str:
+    if isinstance(s, Token):
+        norm = s.lower_
+    elif isinstance(s, str):
+        norm = s.lower()
+    norm = strip_accents(norm)
+    norm = norm.strip()
+    if norm == "saint":
+        norm = "st"
+    if norm == "sainte":
+        norm = "ste"
+    return norm

bertrend_apps/exploration/geolocalization/spacy/dbpedia.py

@@ -0,0 +1,206 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of Wattelse, a NLP application suite.
+
+# NB. Largely inspired from https://github.com/MartinoMensio/spacy-dbpedia-spotlight (MIT license)
+
+import spacy
+import requests
+from loguru import logger
+from requests import HTTPError
+
+from spacy.tokens import Span, Doc
+
+DBPEDIA_SPOTLIGHT_DEFAULT_ENDPOINT = "https://api.dbpedia-spotlight.org"
+DBPEDIA_ENT = "DBPedia"
+
+# span extension attribute for raw json
+Span.set_extension("dbpedia_raw_result", default=None, force=True)
+Doc.set_extension("dbpedia_raw_result", default=None, force=True)
+
+
+class DBPediaEntityLinker(object):
+    """This class manages the querying of DBpedia and attaches the found entities to the document"""
+
+    # default location of the service
+    base_url = DBPEDIA_SPOTLIGHT_DEFAULT_ENDPOINT
+    # list of supported languages
+    supported_languages = ["en", "de", "es", "fr", "it", "nl", "pt", "ru"]
+    # list of supported processes
+    supported_processes = ["annotate", "spot", "candidates"]
+
+    def __init__(
+        self,
+        language_code="en",
+        dbpedia_rest_endpoint=None,
+        process="annotate",
+        confidence=None,
+        support=None,
+        types=None,
+        sparql=None,
+        policy=None,
+        span_group=DBPEDIA_ENT,
+        overwrite_ents=True,
+    ):
+        # constructor of the pipeline stage
+        if language_code not in self.supported_languages:
+            raise ValueError(
+                f"Linker not available in {language_code}. Choose one of {self.supported_languages}"
+            )
+        self.language_code = language_code
+        if process not in self.supported_processes:
+            raise ValueError(
+                f"The process {process} is not supported. Choose one of {self.supported_processes}"
+            )
+        self.process = process
+        self.confidence = confidence
+        self.support = support
+        self.types = types
+        self.sparql = sparql
+        self.policy = policy
+        self.span_group = span_group
+        self.overwrite_ents = overwrite_ents
+        self.dbpedia_rest_endpoint = dbpedia_rest_endpoint
+
+    def get_uri(self, el):
+        # fields have different names depending on the process
+        if self.process == "annotate":
+            return el["@URI"]
+        elif self.process == "candidates":
+            return f"http://dbpedia.org/resource/{el['resource']['@uri']}"
+        return None
+
+    def get_ents_list(self, json):
+        # fields have different names depending on the process
+        if self.process == "annotate":
+            return json.get("Resources", [])
+        return json.get("annotation", {}).get("surfaceForm", [])
+
+    def __call__(self, doc):
+        # called in the pipeline
+        if self.dbpedia_rest_endpoint:
+            # override the default endpoint, e.g., 'http://localhost:2222/rest'
+            endpoint = self.dbpedia_rest_endpoint
+        else:
+            # use the default endpoint for the language selected
+            endpoint = f"{self.base_url}/{self.language_code}"
+
+        params = {"text": doc.text}
+        if self.confidence:
+            params["confidence"] = self.confidence
+        if self.support:
+            params["support"] = self.support
+        if self.types:
+            params["types"] = ",".join(
+                self.types
+            )  # JP: for proper formatting in the API
+        if self.sparql:
+            params["sparql"] = self.sparql
+        if self.policy:
+            params["policy"] = self.policy
+
+        # TODO: application/ld+json would be more detailed? https://github.com/digitalbazaar/pyld
+        try:
+            response = requests.post(
+                f"{endpoint}/{self.process}",
+                headers={"accept": "application/json"},
+                data=params,
+            )
+        except HTTPError as e:
+            # due to too many requests to the endpoint - this happens sometimes with the default public endpoint
+            logger.warn(
+                f"Bad response from server {endpoint}, probably too many requests. Consider using your own endpoint. Document not updated."
+            )
+            logger.trace(str(e))
+            return doc
+        except Exception as e:  # other erros
+            logger.error(
+                f"Endpoint {endpoint} unreachable, please check your connection. Document not updated."
+            )
+            logger.trace(str(e))
+            return doc
+
+        response.raise_for_status()
+        data = response.json()
+        logger.trace(f"Received data: {data}")
+
+        doc._.dbpedia_raw_result = data
+
+        ents_data = []
+
+        # fields have different names depending on the process
+        text_key = "@name"
+        # get_offset
+        if self.process == "annotate":
+            text_key = "@surfaceForm"
+
+        for ent in self.get_ents_list(data):
+            start_ch = int(ent["@offset"])
+            end_ch = int(start_ch + len(ent[text_key]))
+            ent_kb_id = self.get_uri(ent)
+            # TODO look at '@types' and choose most relevant?
+            # JP: adaptation to indicate the type instead of a generic span type when possible.
+            # In case of a span matching several types, we retain the first one (the assumption is that types
+            # are ordered by decreasing preference)
+            if not self.types:
+                # set a default span type
+                span_type = DBPEDIA_ENT
+            else:
+                # takes the first one that matches
+                for t in self.types:
+                    if t in ent["@types"]:
+                        span_type = t
+                        break
+
+            # if ent_kb_id:
+            #    span = doc.char_span(start_ch, end_ch, span_type, ent_kb_id)
+            span = doc.char_span(start_ch, end_ch, span_type)
+
+            if not span:
+                # something strange like "something@bbc.co.uk" where the match is only part of a SpaCy token
+                # 1. find the token to split
+                # tokens also wider than start_ch, end_ch
+                tokens_to_split = [
+                    t for t in doc if t.idx >= start_ch or t.idx + len(t) <= end_ch
+                ]
+                span = doc.char_span(
+                    min(t.idx for t in tokens_to_split),
+                    max(t.idx + len(t) for t in tokens_to_split),
+                )
+
+            span._.dbpedia_raw_result = ent
+
+            ents_data.append(span)
+
+        # try to add results to doc.ents
+        try:
+            # NB. Order is important here for precedence of infra entities over other entities
+            doc.ents = list(doc.ents) + ents_data
+        except Exception as e:
+            logger.trace(str(e))
+            if self.overwrite_ents:
+                # overwrite ok
+                doc.spans["ents_original"] = doc.ents
+                try:
+                    doc.ents = ents_data
+                except (
+                    ValueError
+                ):  # if there are overlapping spans in the dbpedia_spotlight entities
+                    doc.ents = spacy.util.filter_spans(ents_data)
+            # else don't overwrite
+
+        doc.spans[self.span_group] = ents_data
+
+        return doc
+
+
+def create(language_code, nlp=None):
+    """Creates an instance of a Language with the DBpedia EntityLinker pipeline stage.
+    If the parameter `nlp` is None, it will return a blank language with the EntityLinker.
+    If the parameter `nlp` is an existing Language, it simply adds the EntityLinker pipeline stage (equivalent to `nlp.add`)
+    """
+    if not nlp:
+        nlp = spacy.blank(language_code)
+    nlp.add_pipe("dbpedia_spotlight")
+    return nlp

bertrend_apps/exploration/geolocalization/spacy/nlp.config

@@ -0,0 +1,37 @@
+[paths]
+train = null
+dev = null
+vectors = null
+init_tok2vec = null
+
+[system]
+gpu_allocator = "pytorch"
+seed = 0
+
+[nlp]
+lang = "fr"
+batch_size = 128
+disabled = []
+before_creation = null
+after_creation = null
+after_pipeline_creation = null
+tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
+
+pipeline = ["tokenizer_component", "normalizer_component",
+            "dbpedia"]
+
+[components]
+
+[components.tokenizer_component]
+factory = "tokenizer_component"
+
+[components.normalizer_component]
+factory = "normalizer_component"
+
+[components.dbpedia]
+factory = "DBpedia_spotlight"
+overwrite_ents = false
+language_code= "fr"
+confidence = 0.3
+types =  ["DBpedia:Place", "DBpedia:Organisation"]
+

bertrend_apps/exploration/geolocalization/spacy/utils.py

@@ -0,0 +1,39 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of Wattelse, a NLP application suite.
+
+import os
+import time
+from pathlib import Path
+from typing import Union
+
+from loguru import logger
+from spacy.lang.fr import French
+from thinc.config import Config
+
+from bertrend_apps.exploration.geolocalization.spacy import common_factory
+
+# Those imports are not explicitly used, but mandatory to have a working nlp pipeline... - DO NOT REMOVE!
+
+ASSETS_PATH = Path(__file__).parent.absolute()
+RTE_NLP_CONFIG_PATH = ASSETS_PATH / "nlp.config"
+DBPEDIA_REST_API_URL = os.getenv("DBPEDIA_REST_API_URL", "http://localhost:2222/rest")
+
+# These logs are just there to force the imports of the factories - DO NOT REMOVE!
+logger.debug(f"{common_factory.__name__} loaded.")
+
+
+def load_nlp(config_path: Union[str, Path] = RTE_NLP_CONFIG_PATH, **kwargs):
+    t1 = time.time()
+    logger.info("Building nlp...")
+    config = Config().from_disk(
+        config_path,
+        overrides={"components.dbpedia.dbpedia_rest_endpoint": DBPEDIA_REST_API_URL},
+    )
+
+    nlp = French.from_config(config, **kwargs)
+    t2 = time.time()
+
+    logger.success(f"NLP pipeline built in {t2 - t1:3f} seconds!")
+    return nlp