gismap 0.1.0__py3-none-any.whl

gismap/sources/dblp.py

@@ -0,0 +1,162 @@
+from typing import ClassVar
+from dataclasses import dataclass, field
+from urllib.parse import quote_plus
+from bs4 import BeautifulSoup as Soup
+from time import sleep
+
+from gismap.sources.models import DB, Author, Publication
+from gismap.utils.text import clean_aliases
+from gismap.utils.requests import get
+
+
+@dataclass(repr=False)
+class DBLP(DB):
+    db_name: ClassVar[str] = "dblp"
+    author_backoff: ClassVar[float] = 7.0
+    publi_backoff: ClassVar[float] = 2.0
+
+    @classmethod
+    def search_author(cls, name, wait=True):
+        """
+        Parameters
+        ----------
+        name: :class:`str`
+            People to find.
+        wait: :class:`bool`
+            Wait a bit to avoid 429.
+
+        Returns
+        -------
+        :class:`list`
+            Potential matches.
+
+        Examples
+        --------
+
+        >>> fabien = DBLP.search_author("Fabien Mathieu")
+        >>> fabien
+        [DBLPAuthor(name='Fabien Mathieu', key='66/2077')]
+        >>> fabien[0].url
+        'https://dblp.org/pid/66/2077.html'
+        >>> manu = DBLP.search_author("Manuel Barragan")
+        >>> manu # doctest:  +NORMALIZE_WHITESPACE
+        [DBLPAuthor(name='Manuel Barragan', key='07/10587'),
+        DBLPAuthor(name='Manuel Barragan', key='83/3865'),
+        DBLPAuthor(name='Manuel Barragan', key='188/0198')]
+        >>> DBLP.search_author("NotaSearcherName", wait=False)
+        []
+        """
+        dblp_api = "https://dblp.org/search/author/api"
+        dblp_args = {"q": name}
+        r = get(dblp_api, params=dblp_args)
+        soup = Soup(r, features="xml")
+        if wait:
+            sleep(cls.author_backoff)
+        return [
+            DBLPAuthor(
+                name=name,
+                key=hit.url.text.split("pid/")[1],
+                aliases=clean_aliases(
+                    name, [hit.author.text] + [alia.text for alia in hit("alias")]
+                ),
+            )
+            for hit in soup("hit")
+        ]
+
+    @classmethod
+    def from_author(cls, a, wait=True):
+        """
+        Returns
+        -------
+        :class:`list`
+            Papers available in DBLP.
+        wait: :class:`bool`
+            Wait a bit to avoid 429.
+
+        Examples
+        --------
+
+        >>> fabien = DBLPAuthor('Fabien Mathieu', key='66/2077')
+        >>> publications = sorted(DBLP.from_author(fabien),
+        ...                 key=lambda p: p.title)
+        >>> publications[0] # doctest:  +NORMALIZE_WHITESPACE
+        DBLPPublication(title='Achievable catalog size in peer-to-peer video-on-demand systems.',
+        authors=[DBLPAuthor(name='Yacine Boufkhad', key='75/5742'), DBLPAuthor(name='Fabien Mathieu', key='66/2077'),
+        DBLPAuthor(name='Fabien de Montgolfier', key='57/6313'), DBLPAuthor(name='Diego Perino', key='03/3645'),
+        DBLPAuthor(name='Laurent Viennot', key='v/LaurentViennot')],
+        venue='IPTPS', type='conference', year=2008, key='conf/iptps/BoufkhadMMPV08',
+        url='https://dblp.org/rec/conf/iptps/BoufkhadMMPV08.html', pages=4)
+        >>> publications[-1] # doctest:  +NORMALIZE_WHITESPACE
+        DBLPPublication(title='Upper Bounds for Stabilization in Acyclic Preference-Based Systems.',
+        authors=[DBLPAuthor(name='Fabien Mathieu', key='66/2077')], venue='SSS', type='conference', year=2007,
+        key='conf/sss/Mathieu07', url='https://dblp.org/rec/conf/sss/Mathieu07.html', pages='372-382')
+        """
+        r = get(f"https://dblp.org/pid/{a.key}.xml")
+        soup = Soup(r, features="xml")
+        if wait:
+            sleep(cls.author_backoff)
+        res = [DBLPPublication.from_soup(r) for r in soup("r")]
+        return [p for p in res if p.authors]
+
+
+@dataclass(repr=False)
+class DBLPAuthor(Author, DBLP):
+    key: str
+    aliases: list = field(default_factory=list)
+
+    @property
+    def url(self):
+        if self.key:
+            return f"https://dblp.org/pid/{self.key}.html"
+        return f"https://dblp.org/search?q={quote_plus(self.name)}"
+
+    def get_publications(self, wait=False):
+        return DBLP.from_author(self, wait=wait)
+
+
+DBLP_TYPES = {
+    "article": "journal",
+    "inproceedings": "conference",
+    "proceedings": "book",
+    "informal": "report",
+    "phdthesis": "thesis",
+    "habil": "hdr",
+    "software": "software",
+}
+
+
+@dataclass(repr=False)
+class DBLPPublication(Publication, DBLP):
+    key: str
+    url: str = None
+    pages: str = None
+    volume: int = None
+    number: int = None
+
+    @classmethod
+    def from_soup(cls, soup):
+        p = soup.find()
+        typ = p.get("publtype", p.name)
+        typ = DBLP_TYPES.get(typ, typ)
+        res = {
+            "type": typ,
+            "key": p["key"],
+            "url": f"https://dblp.org/rec/{p['key']}.html",
+        }
+        keys = ["title", "booktitle", "pages", "journal", "year", "volume", "number"]
+        for tag in keys:
+            t = p.find(tag)
+            if t:
+                try:
+                    res[tag] = int(t.text)
+                except ValueError:
+                    res[tag] = t.text
+        for tag in ["booktitle", "journal"]:
+            t = p.find(tag)
+            if t:
+                res["venue"] = t.text
+                break
+        else:
+            res["venue"] = "unpublished"
+        res["authors"] = [DBLPAuthor(key=a["pid"], name=a.text) for a in p("author")]
+        return cls(**{k: v for k, v in res.items() if k in cls.__match_args__})

gismap/sources/hal.py

@@ -0,0 +1,272 @@
+from typing import ClassVar
+from dataclasses import dataclass, field
+from collections import defaultdict
+from urllib.parse import quote_plus
+import json
+
+from gismap.sources.models import DB, Publication, Author  #  DBAuthor, DBPublication
+from gismap.utils.text import clean_aliases
+from gismap.utils.requests import get
+from gismap.utils.common import unlist
+
+
+@dataclass(repr=False)
+class HAL(DB):
+    db_name: ClassVar[str] = "hal"
+
+    @classmethod
+    def search_author(cls, name):
+        """
+        Parameters
+        ----------
+        name: :class:`str`
+            People to find.
+
+        Returns
+        -------
+        :class:`list`
+            Potential matches.
+
+        Examples
+        --------
+
+        >>> fabien = HAL.search_author("Fabien Mathieu")
+        >>> fabien
+        [HALAuthor(name='Fabien Mathieu', key='fabien-mathieu')]
+        >>> fabien = fabien[0]
+        >>> fabien.url
+        'https://hal.science/search/index/?q=*&authIdHal_s=fabien-mathieu'
+        >>> HAL.search_author("Laurent Viennot")[0]
+        HALAuthor(name='Laurent Viennot', key='laurentviennot')
+        >>> HAL.search_author("NotaSearcherName")
+        []
+        >>> HAL.search_author("Ana Busic")
+        [HALAuthor(name='Ana Busic', key='anabusic')]
+        >>> HAL.search_author("Potop-Butucaru Maria")  # doctest:  +NORMALIZE_WHITESPACE
+        [HALAuthor(name='Potop-Butucaru Maria', key='858256', key_type='pid'),
+        HALAuthor(name='Potop-Butucaru Maria', key='841868', key_type='pid')]
+        >>> diego = HAL.search_author("Diego Perino")
+        >>> diego  # doctest:  +NORMALIZE_WHITESPACE
+        [HALAuthor(name='Diego Perino', key='847558', key_type='pid'),
+        HALAuthor(name='Diego Perino', key='978810', key_type='pid')]
+        >>> diego[1].url
+        'https://hal.science/search/index/?q=*&authIdPerson_i=978810'
+        """
+        hal_api = "https://api.archives-ouvertes.fr/ref/author/"
+        fields = ",".join(["label_s", "idHal_s", "person_i", "fullName_s"])
+        hal_args = {"q": name, "fl": fields, "wt": "json"}
+        r = get(hal_api, params=hal_args)
+        response = json.loads(r)["response"]
+        hids = defaultdict(set)
+        pids = defaultdict(set)
+        names = set()
+        for a in response.get("docs", []):
+            if "label_s" in a:
+                if "idHal_s" in a:
+                    hids[a["idHal_s"]].add(a.get("label_s"))
+                elif "person_i" in a:
+                    pids[a["person_i"]].add(a.get("label_s"))
+            elif "fullName_s" in a:
+                    names.add(a["fullName_s"])
+        res = [
+            HALAuthor(name=name, key=k, aliases=clean_aliases(name, v))
+            for k, v in hids.items()
+        ] + [
+            HALAuthor(
+                name=name, key=str(k), aliases=clean_aliases(name, v), key_type="pid"
+            )
+            for k, v in pids.items()
+        ]
+        return (
+            res
+            if res
+            else [
+                HALAuthor(
+                    name=name,
+                    key=fullname,
+                    aliases=clean_aliases(name, fullname),
+                    key_type="fullname",
+                )
+                for fullname in names
+            ]
+        )
+
+    @classmethod
+    def from_author(cls, a):
+        """
+        Parameters
+        ----------
+        a: :class:`~gismap.sources.hal.HALAuthor`
+            Hal researcher.
+
+        Returns
+        -------
+        :class:`list`
+            Papers available in HAL.
+
+        Examples
+        --------
+
+        >>> fabien = HAL.search_author("Fabien Mathieu")[0]
+        >>> publications = sorted(fabien.get_publications(), key=lambda p: p.title)
+        >>> publications[2] # doctest:  +NORMALIZE_WHITESPACE
+        HALPublication(title='Achievable Catalog Size in Peer-to-Peer Video-on-Demand Systems',
+        authors=[HALAuthor(name='Yacine Boufkhad', key='yacine-boufkhad'),
+        HALAuthor(name='Fabien Mathieu', key='fabien-mathieu'),
+        HALAuthor(name='Fabien de Montgolfier', key='949013', key_type='pid'),
+        HALAuthor(name='Diego Perino', key='Diego Perino', key_type='fullname'),
+        HALAuthor(name='Laurent Viennot', key='laurentviennot')],
+        venue='Proceedings of the 7th Internnational Workshop on Peer-to-Peer Systems (IPTPS)', type='conference',
+        year=2008, key='471724', url='https://inria.hal.science/inria-00471724v1')
+        >>> diego = publications[2].authors[3]
+        >>> diego
+        HALAuthor(name='Diego Perino', key='Diego Perino', key_type='fullname')
+        >>> len(diego.get_publications())
+        28
+        >>> publications[-7] # doctest:  +NORMALIZE_WHITESPACE
+        HALPublication(title='Upper bounds for stabilization in acyclic preference-based systems',
+        authors=[HALAuthor(name='Fabien Mathieu', key='fabien-mathieu')],
+        venue="SSS'07 - 9th international conference on Stabilization, Safety, and Security of Distributed Systems",
+        type='conference', year=2007, key='668356', url='https://inria.hal.science/hal-00668356v1')
+
+        Case of someone with multiple ids one want to cumulate:
+
+        >>> maria = HAL.search_author('Maria Potop-Butucaru')
+        >>> maria  # doctest: +NORMALIZE_WHITESPACE
+        [HALAuthor(name='Maria Potop-Butucaru', key='858256', key_type='pid'),
+        HALAuthor(name='Maria Potop-Butucaru', key='841868', key_type='pid')]
+        >>> len(HAL.from_author(maria[0]))
+        26
+        >>> len(maria[1].get_publications())
+        123
+
+        Note: an error is raised if not enough data is provided
+
+        >>> HAL.from_author(HALAuthor('Fabien Mathieu'))
+        Traceback (most recent call last):
+        ...
+        ValueError: HALAuthor(name='Fabien Mathieu') must have a key for publications to be fetched.
+        """
+        api = "https://api.archives-ouvertes.fr/search/"
+        fields = [
+            "docid",
+            "abstract_s",
+            "label_s",
+            "uri_s",
+            "*Title_s",
+            "title_s",
+            "producedDateY_i",
+            "auth_s",
+            "authFullNamePersonIDIDHal_fs",
+            "docType_s",
+        ]
+        params = {"fl": fields, "rows": 2000, "wt": "json"}
+        if a.key is None:
+            raise ValueError(f"{a} must have a key for publications to be fetched.")
+        if a.key_type == "pid":
+            params["q"] = f"authIdPerson_i:{a.key}"
+        elif a.key_type == "fullname":
+            params["q"] = f'authFullName_s:"{a.key}"'
+        else:
+            params["q"] = f"authIdHal_s:{a.key}"
+        r = get(api, params=params)
+        response = json.loads(r)["response"]
+        res = [HALPublication.from_json(r) for r in response.get("docs", [])]
+        return res
+
+
+@dataclass(repr=False)
+class HALAuthor(Author, HAL):
+    key: str | int = None
+    key_type: str = None
+    aliases: list = field(default_factory=list)
+
+    @property
+    def url(self):
+        if self.key_type == "pid":
+            return f"https://hal.science/search/index/?q=*&authIdPerson_i={self.key}"
+        elif self.key_type == "fullname":
+            return f"https://hal.science/search/index?q={quote_plus(self.name)}"
+        else:
+            return f"https://hal.science/search/index/?q=*&authIdHal_s={self.key}"
+
+    def get_publications(self):
+        return HAL.from_author(self)
+
+
+def parse_facet_author(a):
+    """
+
+    Parameters
+    ----------
+    a: :class:`str`
+        Hal facet of author
+
+    Returns
+    -------
+    :class:`~gismap.sources.hal.HALAuthor`
+
+    """
+    name, pid, hid = a.split("_FacetSep_")
+    if hid:
+        return HALAuthor(name=name, key=hid)
+    elif pid and int(pid):
+        return HALAuthor(name=name, key=pid, key_type="pid")
+    else:
+        return HALAuthor(name=name, key=name, key_type="fullname")
+
+
+HAL_TYPES = {
+    "ART": "journal",
+    "COMM": "conference",
+    "OUV": "book",
+    "COUV": "chapter",
+    "THESE": "thesis",
+    "UNDEFINED": "report",
+}
+
+HAL_KEYS = {
+    "title_s": "title",
+    "abstract_s": "abstract",
+    "docid": "key",
+    "bookTitle_s": "booktitle",
+    "conferenceTitle_s": "conference",
+    "journalTitle_s": "journal",
+    "docType_s": "type",
+    "producedDateY_i": "year",
+    "uri_s": "url",
+}
+
+
+@dataclass(repr=False)
+class HALPublication(Publication, HAL):
+    key: str
+    abstract: str = None
+    url: str = None
+
+    @classmethod
+    def from_json(cls, r):
+        """
+
+        Parameters
+        ----------
+        r: :class:`dict`
+            De-serialized JSON.
+
+        Returns
+        -------
+        :class:`~gismap.sources.hal.HALPublication`
+
+        """
+        res = {v: unlist(r[k]) for k, v in HAL_KEYS.items() if k in r}
+        res["authors"] = [
+            parse_facet_author(a) for a in r.get("authFullNamePersonIDIDHal_fs", [])
+        ]
+        for tag in ["booktitle", "journal", "conference"]:
+            if tag in res:
+                res["venue"] = res[tag]
+                break
+        else:
+            res["venue"] = "unpublished"
+        res["type"] = HAL_TYPES.get(res["type"], res["type"].lower())
+        return cls(**{k: v for k, v in res.items() if k in cls.__match_args__})

gismap/sources/models.py

@@ -0,0 +1,31 @@
+from dataclasses import dataclass
+from typing import ClassVar
+
+from gismap.utils.common import LazyRepr
+
+
+@dataclass(repr=False)
+class Author(LazyRepr):
+    name: str
+
+
+@dataclass(repr=False)
+class Publication(LazyRepr):
+    title: str
+    authors: list
+    venue: str
+    type: str
+    year: int
+
+
+@dataclass(repr=False)
+class DB(LazyRepr):
+    db_name: ClassVar[str] = None
+
+    @classmethod
+    def search_author(cls, name):
+        raise NotImplementedError
+
+    @classmethod
+    def from_author(cls, a):
+        raise NotImplementedError

gismap/sources/multi.py

@@ -0,0 +1,135 @@
+from dataclasses import dataclass, field
+from bof.fuzz import Process
+import numpy as np
+
+from gismap.sources.models import Publication, Author
+from gismap.utils.text import clean_aliases
+
+
+score_rosetta = {
+    "db_name": {"dblp": 1, "hal": 2},
+    "venue": {"CoRR": -1, "unpublished": -2},
+    "type": {"conference": 1, "journal": 2},
+}
+
+
+@dataclass(repr=False)
+class SourcedAuthor(Author):
+    sources: list = field(default_factory=list)
+
+    @property
+    def key(self):
+        if self.sources:
+            return self.sources[0].key
+        else:
+            return None
+
+    @property
+    def aliases(self):
+        if self.sources:
+            return clean_aliases(
+                self.name, [n for a in self.sources for n in [a.name] + a.aliases]
+            )
+        else:
+            return []
+
+    @classmethod
+    def from_sources(cls, sources):
+        return cls(name=sources[0].name, sources=sources)
+
+    def get_publications(self, clean=True):
+        res = {p.key: p for a in self.sources for p in a.get_publications()}
+        if clean:
+            regroup_authors({self.key: self}, res)
+            return regroup_publications(res)
+        else:
+            return res
+
+
+@dataclass(repr=False)
+class SourcedPublication(Publication):
+    key: str
+    sources: list = field(default_factory=list)
+
+    @classmethod
+    def from_sources(cls, sources):
+        sources = sorted(sources, key=cls.score_source, reverse=True)
+        main = sources[0]
+        res = cls(
+            **{k: getattr(main, k) for k in main.__dict__ if k in cls.__match_args__},
+            sources=sources,
+        )
+        for k, v in main.__dict__.items():
+            if k not in cls.__match_args__:
+                setattr(res, k, v)
+        return res
+
+    @staticmethod
+    def score_source(source):
+        scores = [v.get(getattr(source, k, None), 0) for k, v in score_rosetta.items()]
+        scores.append(source.year)
+        return tuple(scores)
+
+
+def regroup_authors(auth_dict, pub_dict):
+    """
+    Replace authors of publications with matching authors.
+    Typical use: upgrade DB-specific authors to multisource authors.
+
+    Replacement is in place.
+
+    Parameters
+    ----------
+    auth_dict: :class:`dict`
+        Authors to unify.
+    pub_dict: :class:`dict`
+        Publications to unify.
+
+    Returns
+    -------
+    None
+    """
+    redirection = {
+        k: a
+        for a in auth_dict.values()
+        for s in a.sources
+        for k in [s.key, s.name, *s.aliases]
+    }
+
+    for pub in pub_dict.values():
+        pub.authors = [redirection.get(a.key, a) for a in pub.authors]
+
+
+def regroup_publications(pub_dict, threshold=90, length_impact=0.2):
+    """
+    Puts together copies of the same publication.
+
+    Parameters
+    ----------
+    pub_dict: :class:`dict`
+        Publications to unify.
+    threshold: float
+        Similarity parameter.
+    length_impact: float
+        Length impact parameter.
+
+    Returns
+    -------
+    :class:`dict`
+        Unified publications.
+    """
+    pub_list = [p for p in pub_dict.values()]
+
+    p = Process(length_impact=length_impact)
+    p.fit([paper.title for paper in pub_list])
+
+    res = dict()
+    done = np.zeros(len(pub_list), dtype=bool)
+    for i, paper in enumerate(pub_list):
+        if done[i]:
+            continue
+        locs = np.where(p.transform([paper.title])[0, :] > threshold)[0]
+        pub = SourcedPublication.from_sources([pub_list[i] for i in locs])
+        res[pub.key] = pub
+        done[locs] = True
+    return res

gismap/utils/common.py

@@ -0,0 +1,60 @@
+HIDDEN_KEYS = {"sources", "aliases", "abstract"}
+
+
+class LazyRepr:
+    """
+    MixIn that hides empty fields in dataclasses repr's.
+    """
+
+    def __repr__(self):
+        kws = [
+            f"{key}={value!r}"
+            for key, value in self.__dict__.items()
+            if value and key not in HIDDEN_KEYS
+        ]
+        return f"{type(self).__name__}({', '.join(kws)})"
+
+
+def unlist(x):
+    """
+    Parameters
+    ----------
+    x: :class:`str` or :class:`list` or :class:`int`
+        Something.
+
+    Returns
+    -------
+    x: :class:`str` or :class:`int`
+        If it's a list, make it flat.
+    """
+    return x[0] if (isinstance(x, list) and x) else x
+
+
+def get_classes(root, key="name"):
+    """
+    Parameters
+    ----------
+    root: :class:`class`
+        Starting class (can be abstract).
+    key: :class:`str`, default='name'
+        Attribute to look-up
+
+    Returns
+    -------
+    :class:`dict`
+        Dictionaries of all subclasses that have a key attribute (as in class attribute `key`).
+
+    Examples
+    --------
+
+    >>> from gismap.sources.models import DB
+    >>> subclasses = get_classes(DB, key='db_name')
+    >>> dict(sorted(subclasses.items())) # doctest: +NORMALIZE_WHITESPACE
+    {'dblp': <class 'gismap.sources.dblp.DBLP'>, 'hal': <class 'gismap.sources.hal.HAL'>}
+    """
+    result = {
+        getattr(c, key): c for c in root.__subclasses__() if getattr(c, key, None)
+    }
+    for c in root.__subclasses__():
+        result.update(get_classes(c))
+    return result

gismap/utils/logger.py

@@ -0,0 +1,4 @@
+import logging
+
+logger = logging.getLogger("GisMap")
+"""Default logging interface."""

gismap/utils/requests.py

@@ -0,0 +1,33 @@
+from time import sleep
+import requests
+from gismap.utils.logger import logger
+
+
+session = requests.Session()
+
+
+def get(url, params=None):
+    """
+    Parameters
+    ----------
+    url: :class:`str`
+        Entry point to fetch.
+    params: :class:`dict`, optional
+        Get arguments (appended to URL).
+
+    Returns
+    -------
+    :class:`str`
+        Result.
+    """
+    while True:
+        r = session.get(url, params=params)
+        if r.status_code == 429:
+            try:
+                t = int(r.headers["Retry-After"])
+            except KeyError:
+                t = 60
+            logger.warning(f"Too many requests. Auto-retry in {t} seconds.")
+            sleep(t)
+        else:
+            return r.text