bertrend 0.3.1__py3-none-any.whl

bertrend_apps/data_provider/__init__.py

@@ -0,0 +1,9 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+# Define a pattern for a basic URL validation
+URL_PATTERN = (
+    r"^(https?://)?([a-z0-9-]+\.)+[a-z]{2,6}(:\d+)?(/[\w.-]*)*$|"
+    r"^(https?://)?(localhost|(\d{1,3}\.){3}\d{1,3})(:\d+)?(/[\w.-]*)*$"
+)

bertrend_apps/data_provider/__main__.py

@@ -0,0 +1,237 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+import os
+import tempfile
+from datetime import timedelta, datetime
+
+import typer
+from loguru import logger
+from pathlib import Path
+
+from bertrend import FEED_BASE_PATH, load_toml_config
+from bertrend_apps.common.crontab_utils import schedule_scrapping
+from bertrend_apps.data_provider.arxiv_provider import ArxivProvider
+from bertrend_apps.data_provider.bing_news_provider import BingNewsProvider
+from bertrend_apps.data_provider.curebot_provider import CurebotProvider
+from bertrend_apps.data_provider.google_news_provider import GoogleNewsProvider
+from bertrend_apps.data_provider.newscatcher_provider import NewsCatcherProvider
+
+# Ensures to write with +rw for both user and groups
+os.umask(0o002)
+
+PROVIDERS = {
+    "arxiv": ArxivProvider,
+    "curebot": CurebotProvider,
+    "google": GoogleNewsProvider,
+    "bing": BingNewsProvider,
+    "newscatcher": NewsCatcherProvider,
+}
+
+if __name__ == "__main__":
+    app = typer.Typer()
+
+    @app.command("scrape")
+    def scrape(
+        keywords: str = typer.Argument(help="keywords for data search engine."),
+        provider: str = typer.Option(
+            "google", help="source for data [arxiv, google, bing, newscatcher]"
+        ),
+        after: str = typer.Option(
+            None, help="date after which to consider news [format YYYY-MM-DD]"
+        ),
+        before: str = typer.Option(
+            None, help="date before which to consider news [format YYYY-MM-DD]"
+        ),
+        max_results: int = typer.Option(
+            50, help="maximum number of results per request"
+        ),
+        save_path: Path = typer.Option(
+            None, help="Path for writing results. File is in jsonl format."
+        ),
+        language: str = typer.Option(None, help="Language filter"),
+    ):
+        """Scrape data from Arxiv, Google, Bing or NewsCatcher news (single request).
+
+        Parameters
+        ----------
+        keywords: str
+            query described as keywords
+        provider: str
+            News data provider. Current authorized values [google, bing, newscatcher]
+        after: str
+            "from" date, formatted as YYYY-MM-DD
+        before: str
+            "to" date, formatted as YYYY-MM-DD
+        max_results: int
+            Maximum number of results per request
+        save_path: Path
+            Path to the output file (jsonl format)
+        language: str
+            Language filter
+
+        Returns
+        -------
+
+        """
+        provider_class = PROVIDERS.get(provider)
+        provider = provider_class()
+        results = provider.get_articles(keywords, after, before, max_results, language)
+        provider.store_articles(results, save_path)
+
+    @app.command("auto-scrape")
+    def auto_scrape(
+        requests_file: str = typer.Argument(
+            help="path of jsonlines input file containing the expected queries."
+        ),
+        max_results: int = typer.Option(
+            50, help="maximum number of results per request"
+        ),
+        provider: str = typer.Option(
+            "google", help="source for news [google, bing, newscatcher]"
+        ),
+        save_path: Path = typer.Option(None, help="Path for writing results."),
+        language: str = typer.Option(None, help="Language filter"),
+    ):
+        """Scrape data from Arxiv, Google, Bing news or NewsCatcher (multiple requests from a configuration file: each line of the file shall be compliant with the following format:
+        <keyword list>;<after_date, format YYYY-MM-DD>;<before_date, format YYYY-MM-DD>)
+
+        Parameters
+        ----------
+        requests_file: str
+            Text file containing the list of requests to be processed
+        max_results: int
+            Maximum number of results per request
+        provider: str
+            News data provider. Current authorized values [google, bing, newscatcher]
+        save_path: Path
+            Path to the output file (jsonl format)
+        language: str
+            Language filter
+
+        Returns
+        -------
+
+        """
+        provider_class = PROVIDERS.get(provider)
+        provider = provider_class()
+        logger.info(f"Opening query file: {requests_file}")
+        with open(requests_file) as file:
+            try:
+                requests = [line.rstrip().split(";") for line in file]
+            except:
+                logger.error("Bad file format")
+                return -1
+            results = provider.get_articles_batch(requests, max_results, language)
+            logger.info(f"Storing {len(results)} articles")
+            provider.store_articles(results, save_path)
+
+    @app.command("generate-query-file")
+    def generate_query_file(
+        keywords: str = typer.Argument(help="keywords for news search engine."),
+        after: str = typer.Option(
+            None, help="date after which to consider news [format YYYY-MM-DD]"
+        ),
+        before: str = typer.Option(
+            None, help="date before which to consider news [format YYYY-MM-DD]"
+        ),
+        save_path: str = typer.Option(
+            None, help="Path for writing results. File is in jsonl format."
+        ),
+        interval: int = typer.Option(30, help="Range of days of atomic requests"),
+    ):
+        """Generates a query file to be used with the auto-scrape command. This is useful for queries generating many results.
+        This will split the broad query into many ones, each one covering an 'interval' (range) in days covered by each atomic
+        request.
+        If you want to cover several keywords, run the command several times with the same output file.
+
+        Parameters
+        ----------
+        keywords: str
+            query described as keywords
+        after: str
+            "from" date, formatted as YYYY-MM-DD
+        before: str
+            "to" date, formatted as YYYY-MM-DD
+        save_path: str
+            Path to the output file (jsonl format)
+
+        Returns
+        -------
+
+        """
+        date_format = "%Y-%m-%d"
+        start_date = datetime.strptime(after, date_format)
+        end_date = datetime.strptime(before, date_format)
+        dates_l = list(_daterange(start_date, end_date, interval))
+
+        with open(save_path, "a") as query_file:
+            for elem in dates_l:
+                query_file.write(
+                    f"{keywords};{elem[0].strftime(date_format)};{elem[1].strftime(date_format)}\n"
+                )
+
+    def _daterange(start_date, end_date, ndays):
+        for n in range(int((end_date - start_date).days / ndays)):
+            yield (
+                start_date + timedelta(ndays * n),
+                start_date + timedelta(ndays * (n + 1)),
+            )
+
+    @app.command("scrape-feed")
+    def scrape_from_feed(
+        feed_cfg: Path = typer.Argument(help="Path of the data feed config file"),
+    ):
+        """Scrape data from Arxiv, Google, Bing news or NewsCatcher on the basis of a feed configuration file"""
+        data_feed_cfg = load_toml_config(feed_cfg)
+        current_date = datetime.today()
+        current_date_str = current_date.strftime("%Y-%m-%d")
+        days_to_subtract = data_feed_cfg["data-feed"].get("number_of_days")
+        provider = data_feed_cfg["data-feed"].get("provider")
+        keywords = data_feed_cfg["data-feed"].get("query")
+        max_results = data_feed_cfg["data-feed"].get("max_results")
+        before = current_date_str
+        after = (current_date - timedelta(days=days_to_subtract)).strftime("%Y-%m-%d")
+        language = data_feed_cfg["data-feed"].get("language")
+        save_path = (
+            FEED_BASE_PATH
+            / data_feed_cfg["data-feed"].get("feed_dir_path")
+            / f"{current_date_str}_{data_feed_cfg['data-feed'].get('id')}.jsonl"
+        )
+        save_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Generate a query file
+        with tempfile.NamedTemporaryFile() as query_file:
+            if provider == "arxiv" or provider == "curebot":  # already returns batches
+                scrape(
+                    keywords=keywords,
+                    provider=provider,
+                    after=after,
+                    before=before,
+                    max_results=max_results,
+                    save_path=save_path,
+                    language=language,
+                )
+            else:
+                generate_query_file(
+                    keywords, after, before, interval=1, save_path=query_file.name
+                )
+                auto_scrape(
+                    requests_file=query_file.name,
+                    max_results=max_results,
+                    provider=provider,
+                    save_path=save_path,
+                    language=language,
+                )
+
+    @app.command("schedule-scrapping")
+    def automate_scrapping(
+        feed_cfg: Path = typer.Argument(help="Path of the data feed config file"),
+    ):
+        """Schedule data scrapping on the basis of a feed configuration file"""
+        schedule_scrapping(feed_cfg)
+
+    ##################
+    app()

bertrend_apps/data_provider/arxiv_provider.py

@@ -0,0 +1,161 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+import itertools
+import os
+from datetime import datetime
+from collections import defaultdict
+
+import arxiv
+
+from loguru import logger
+import requests
+
+from bertrend.utils.data_loading import TEXT_COLUMN
+from bertrend_apps.data_provider.data_provider import DataProvider
+from bertrend_apps.data_provider.utils import wait
+
+PAGE_SIZE = 2000
+DELAY_SECONDS = 3
+NUM_RETRIES = 10
+
+### Request parameters ###
+query = "cat:cs.CL"
+max_results = float("inf")
+
+### Parameters Semantic Scholar ###
+SEMANTIC_SCHOLAR_API_KEY = os.getenv("SEMANTIC_SCHOLAR_API_KEY")
+if not SEMANTIC_SCHOLAR_API_KEY:
+    raise ValueError("SEMANTIC_SCHOLAR_API_KEY environment variable is not set")
+
+PAPERS_PER_REQUEST = 500  # should not be more than 500
+SEMANTIC_SCHOLAR_FIELDS = "title,abstract,citationCount,publicationDate,url"
+
+DATE_FORMAT_YYYYMMDD = "%Y-%m-%d"
+DATE_FORMAT_YYYYMMDD_TIME = "%Y-%m-%d %H:%M:%S"
+
+
+class ArxivProvider(DataProvider):
+    """Scientific articles providers for Arxiv"""
+
+    def __init__(self):
+        self.client = arxiv.Client(
+            page_size=PAGE_SIZE,
+            delay_seconds=DELAY_SECONDS,
+            num_retries=NUM_RETRIES,
+        )
+
+    def get_articles(
+        self,
+        query: str,
+        after: str,
+        before: str,
+        max_results: int,
+        language: str = None,
+    ) -> list[dict]:
+        """Requests the news data provider, collects a set of URLs to be parsed, return results as json lines.
+
+        Parameters
+        ----------
+        query: str
+            keywords describing the request
+        after: str
+            date after which to consider articles, formatted as YYYY-MM-DD, currently ignored
+        before: str
+            date before which to consider articles, formatted as YYYY-MM-DD, currently ignored
+        max_results: int
+            Maximum number of results per request
+        language: str
+            Language filter
+
+        Returns
+        -------
+        A list of dict entries, each one describing an article
+        """
+        begin = datetime.strptime(after, DATE_FORMAT_YYYYMMDD)
+        end = datetime.strptime(before, DATE_FORMAT_YYYYMMDD)
+
+        logger.info(f"Querying Arxiv: {query}")
+        entries = list(
+            self.client.results(
+                arxiv.Search(
+                    query=query,
+                    max_results=max_results,
+                    sort_by=arxiv.SortCriterion.SubmittedDate,
+                    sort_order=arxiv.SortOrder.Descending,
+                )
+            )
+        )
+        results = self.process_entries(entries, language)
+        # post-filtering by date
+        results = [
+            res
+            for res in results
+            if begin
+            <= datetime.strptime(res["timestamp"], DATE_FORMAT_YYYYMMDD_TIME)
+            <= end
+        ]
+        # add citations count
+        return self.add_citations_count(results)
+
+    def _parse_entry(self, entry: arxiv.Result) -> dict | None:
+        """Parses a Arxiv entry"""
+        try:
+            id = entry.entry_id
+            title = entry.title
+            published = entry.published.strftime(
+                "%Y-%m-%d %H:%M:%S"
+            )  # dateparser.parse(entry.published).strftime("%Y-%m-%d %H:%M:%S")
+            logger.debug(f"----- Title: {title},\tDate: {published}")
+            return {
+                "id": id,
+                "title": title,
+                "summary": entry.summary,
+                "text": entry.summary,
+                "timestamp": published,
+            }
+        except Exception as e:
+            logger.error(str(e) + f"\nError occurred with parsing of: {entry}")
+            return None
+
+    @wait(1)
+    def _request_semantic_scholar_chunk(self, chunk: list[dict]):
+        """Get information from semantic scholar API per batch of articles IDs"""
+        ids_list = ["URL:" + entry["id"] for entry in chunk]
+        response = requests.post(
+            "https://api.semanticscholar.org/graph/v1/paper/batch",
+            params={"fields": SEMANTIC_SCHOLAR_FIELDS},
+            json={"ids": ids_list},
+            headers={"x-api-key": SEMANTIC_SCHOLAR_API_KEY},
+        )
+        return [item for item in response.json() if item is not None]
+
+    def add_citations_count(self, entries: list[dict]):
+        """Uses the semantic_scholar API to get the number of counts per paper"""
+        # split list into chunks and request semantic scholar
+        chunks = [
+            entries[i : i + PAPERS_PER_REQUEST]
+            for i in range(0, len(entries), PAPERS_PER_REQUEST)
+        ]
+        # request API
+        semantic_scholar_items_list = list(
+            itertools.chain(
+                [self._request_semantic_scholar_chunk(chunk) for chunk in chunks]
+            )
+        )[0]
+
+        # merge semantic scholar entries with arxiv entries
+
+        # update values of entries
+        d = defaultdict(dict)
+
+        for item in semantic_scholar_items_list + entries:
+            # careful, order is important, semantic_scholar_items may contain fewer items than entries
+            d[item["title"]].update(item)
+
+        # filter out possible missing values
+        new_entries = [item for item in list(d.values()) if TEXT_COLUMN in item.keys()]
+
+        return new_entries

bertrend_apps/data_provider/bing_news_provider.py

@@ -0,0 +1,82 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+
+import urllib.parse
+
+import dateparser
+import feedparser
+from loguru import logger
+
+from bertrend_apps.data_provider.data_provider import DataProvider
+from bertrend_apps.data_provider.utils import wait
+
+PATTERN = "{QUERY}"
+
+
+class BingNewsProvider(DataProvider):
+    """News provider for Bing News.
+    Limitations:
+        - since of results limited to 12
+        - hard to request specific dates
+    """
+
+    URL_ENDPOINT = f"https://www.bing.com/news/search?q={PATTERN}&format=rss&setLang=fr&sortBy=Date"
+
+    def __init__(self):
+        super().__init__()
+
+    @wait(0.2)
+    def get_articles(
+        self,
+        query: str,
+        after: str,
+        before: str,
+        max_results: int,
+        language: str = None,
+    ) -> list[dict]:
+        """Requests the news data provider, collects a set of URLs to be parsed, return results as json lines"""
+        q = self._build_query(query, after, before)
+        logger.info(f"Querying Bing: {q}")
+        result = feedparser.parse(q)
+        entries = result["entries"][:max_results]
+        return self.process_entries(entries, language)
+
+    def _build_query(self, keywords: str, after: str = None, before: str = None) -> str:
+        # FIXME: don't know how to use after/before parameters with Bing news queries
+        query = self.URL_ENDPOINT.replace(PATTERN, f"{urllib.parse.quote(keywords)}")
+        return query
+
+    def _clean_url(self, bing_url) -> str:
+        """Clean encoded URLs returned by Bing news such as "http://www.bing.com/news/apiclick.aspx?ref=FexRss&aid=&tid=649475a6257945d6900378c8310bcfde&url=https%3a%2f%2fwww.lemondeinformatique.fr%2factualites%2flire-avec-schema-gpt-translator-datastax-automatise-la-creation-de-pipelines-de-donnees-90737.html&c=15009376565431680830&mkt=fr-fr" """
+        try:
+            clean_url = bing_url.split("url=")[1].split("&  ")[0]
+            return urllib.parse.unquote(clean_url)
+        except IndexError:
+            # fallback (the URL does not match the expected pattern)
+            return bing_url
+
+    def _parse_entry(self, entry: dict) -> dict | None:
+        """Parses a Bing news entry, uses wait decorator to force delay between 2 successive calls"""
+        try:
+            link = entry["link"]
+            url = self._clean_url(link)
+            summary = entry["summary"]
+            published = dateparser.parse(entry["published"]).strftime(
+                "%Y-%m-%d %H:%M:%S"
+            )
+            text, title = self._get_text(url=link)
+            return {
+                "title": title,
+                "summary": summary,
+                "link": link,
+                "url": url,
+                "text": text,
+                "timestamp": published,
+            }
+        except Exception as e:
+            logger.error(
+                str(e) + f"\nError occurred with text parsing of url in : {entry}"
+            )
+            return None

bertrend_apps/data_provider/curebot_provider.py

@@ -0,0 +1,105 @@
+#  Copyright (c) 2024, RTE (https://www.rte-france.com)
+#  See AUTHORS.txt
+#  SPDX-License-Identifier: MPL-2.0
+#  This file is part of BERTrend.
+import re
+from pathlib import Path
+
+import pandas as pd
+from loguru import logger
+
+from bertrend_apps.data_provider import URL_PATTERN
+from bertrend_apps.data_provider.data_provider import DataProvider
+import feedparser
+
+
+class CurebotProvider(DataProvider):
+    """Class used to process exports from Curebot tool"""
+
+    def __init__(self, curebot_export_file: Path = None, feed_url: str = None):
+        super().__init__()
+        self.data_file = curebot_export_file
+        if self.data_file:
+            self.df_dict = pd.read_excel(self.data_file, sheet_name=None, dtype=str)
+        else:
+            self.df_dict = None
+        self.feed_url = feed_url
+
+    def get_articles(
+        self,
+        query: str = None,
+        after: str = None,
+        before: str = None,
+        max_results: int = None,
+        language: str = "fr",
+    ) -> list[dict]:
+        """Requests the news data provider, collects a set of URLs to be parsed, return results as json lines"""
+        if query and re.match(URL_PATTERN, query):
+            # if using a config file, the "query" field may contain the feed url
+            self.feed_url = query
+        if self.feed_url:
+            return self.parse_ATOM_feed()
+
+        if self.df_dict:
+            entries = []
+            for k in self.df_dict.keys():
+                entries += self.df_dict[k].to_dict(orient="records")
+            results = [self._parse_entry(res) for res in entries]
+            return [
+                res for res in results if res is not None
+            ]  # sanity check to remove errors
+
+        return []
+
+    def parse_ATOM_feed(self) -> list[dict]:
+        feed = feedparser.parse(self.feed_url)
+        # Initialize an empty list to store the entries
+        entries = []
+        # Extract information for each entry
+        for entry in feed.entries:
+            link = entry.get("link", "")
+            summary = entry.get("summary", "")
+            published = entry.get("published", "")
+            formatted_date = self.parse_date(published)
+            text, title = self._get_text(url=link)
+            if text is None or text == "":
+                continue
+            # Store the extracted information in a dictionary
+            entry_info = {
+                "title": title,
+                "link": link,
+                "url": link,
+                "summary": summary,
+                "text": text,
+                "timestamp": formatted_date,
+            }
+            # Add the dictionary to the list
+            entries.append(entry_info)
+
+        return entries
+
+    def _parse_entry(self, entry: dict) -> dict | None:
+        """Parses a Curebot news entry"""
+        try:
+            # NB. we do not use the title from Gnews as it is sometimes truncated
+            link = entry["URL de la ressource"]
+            url = link
+            summary = entry["Contenu de la ressource"]
+            published = self.parse_date(entry["Date de trouvaille"])
+            text, title = self._get_text(url=url)
+            if text is None or text == "":
+                return None
+            logger.debug(f"----- Title: {title},\tDate: {published}")
+            return {
+                "title": title,
+                "summary": summary,
+                "link": link,
+                "url": url,
+                "text": text,
+                "timestamp": published,
+            }
+        except Exception as e:
+            logger.error(
+                str(e) + f"\nError occurred with text parsing of url in : {entry}"
+            )
+            return None