gismap 0.2.2__py3-none-any.whl → 0.4.0__py3-none-any.whl

gismap/sources/dblp.py

@@ -13,7 +13,7 @@ from gismap.utils.requests import get
 class DBLP(DB):
     db_name: ClassVar[str] = "dblp"
     author_backoff: ClassVar[float] = 5.0
-    publi_backoff: ClassVar[float] = 1.0
+    publi_backoff: ClassVar[float] = 5.0
 
     @classmethod
     def search_author(cls, name, wait=True):
@@ -72,23 +72,6 @@ class DBLP(DB):
             Papers available in DBLP.
         wait: :class:`bool`
             Wait a bit to avoid 429.
-
-        Examples
-        --------
-
-        >>> fabien = DBLPAuthor('Fabien Mathieu', key='66/2077')
-        >>> publications = sorted(DBLP.from_author(fabien),
-        ...                 key=lambda p: p.title)
-        >>> publications[0] # doctest:  +NORMALIZE_WHITESPACE
-        DBLPPublication(title='Achievable catalog size in peer-to-peer video-on-demand systems.',
-        authors=[DBLPAuthor(name='Yacine Boufkhad', key='75/5742'), DBLPAuthor(name='Fabien Mathieu', key='66/2077'),
-        DBLPAuthor(name='Fabien de Montgolfier', key='57/6313'), DBLPAuthor(name='Diego Perino', key='03/3645'),
-        DBLPAuthor(name='Laurent Viennot', key='v/LaurentViennot')],
-        venue='IPTPS', type='conference', year=2008, key='conf/iptps/BoufkhadMMPV08')
-        >>> publications[-1] # doctest:  +NORMALIZE_WHITESPACE
-        DBLPPublication(title='Upper Bounds for Stabilization in Acyclic Preference-Based Systems.',
-        authors=[DBLPAuthor(name='Fabien Mathieu', key='66/2077')], venue='SSS', type='conference', year=2007,
-        key='conf/sss/Mathieu07')
         """
         r = get(f"https://dblp.org/pid/{a.key}.xml")
         soup = Soup(r, features="xml")
@@ -100,6 +83,21 @@ class DBLP(DB):
 
 @dataclass(repr=False)
 class DBLPAuthor(Author, DBLP):
+    """
+    Examples
+    --------
+
+    >>> fabien = DBLPAuthor('Fabien Mathieu', key='66/2077')
+    >>> publications = sorted(fabien.get_publications(),
+    ...                 key=lambda p: p.title)
+    >>> publications[0].url # doctest:  +NORMALIZE_WHITESPACE
+     'https://dblp.org/rec/conf/iptps/BoufkhadMMPV08.html'
+    >>> publications[-1] # doctest:  +NORMALIZE_WHITESPACE
+    DBLPPublication(title='Upper Bounds for Stabilization in Acyclic Preference-Based Systems.',
+    authors=[DBLPAuthor(name='Fabien Mathieu', key='66/2077')], venue='SSS', type='conference', year=2007,
+    key='conf/sss/Mathieu07')
+
+    """
     key: str
     aliases: list = field(default_factory=list)
 

gismap/sources/dblp_ttl.py

@@ -0,0 +1,168 @@
+import re
+import zlib
+from contextlib import contextmanager
+from pathlib import Path
+
+from tqdm.auto import tqdm
+
+from gismap.utils.requests import session
+from gismap.sources.dblp import DBLP_TYPES
+
+key_re = r'<https://dblp.org/rec/([^>]+)>'
+title_re = r'.*?dblp:title\s+"([^"]+)"'
+type_re = r'.*?dblp:bibtexType\s+bibtex:(\w+)'
+authors_re = r'.*?dblp:hasSignature\s+(\[.*\])\s*;'
+url_re = r'(?:.*?dblp:primaryDocumentPage <([^>]+)>)?'
+stream_re = r'(?:.*?dblp:publishedInStream ([^;]+) ;)?'
+pages_re = r'(?:.*?dblp:pagination "([^"]+)")?'
+venue_re = r'(?:.*?dblp:publishedIn\s+"([^"]+?)")?'
+year_re = r'.*?"(\d{4})"\^\^<http://www.w3.org/2001/XMLSchema#gYear>'
+
+pub_re = re.compile("".join([key_re, title_re, type_re, authors_re,
+                             url_re, stream_re, pages_re, venue_re, year_re]), flags=re.S)
+
+streams_re = re.compile(r'<https://dblp.org/streams/((?:conf|journals)/[^>]+)>')
+
+authid_re = re.compile(
+    r'\[.*?signatureDblpName\s*?"([^"]+?)(?:\s+\d+)?".*?signatureCreator\s*<https://dblp.org/pid/([^>]+?)>.*?]',
+    flags=re.S)
+
+
+def parse_block(dblp_block):
+    """
+    Parameters
+    ----------
+    dblp_block: :class:`str`
+        A DBLP publication, turtle format.
+
+    Returns
+    -------
+    key: :class:`str`
+        DBLP key.
+    title: :class:`str`
+        Publication title.
+    type: :class:`str`
+        Type of publication.
+    authors: :class:`dict`
+        Publication authors (key -> name)
+    url: :class:`str` or :class:`NoneType`
+        Publication URL.
+    stream: :class:`list` or :class:`NoneType`
+        Publication streams (normalized journal/conf).
+    pages: :class:`str` or :class:`NoneType`
+        Publication pages.
+    venue: :class:`str` or :class:`NoneType`
+        Publication venue (conf/journal).
+    year: :class:`int`
+        Year of publication.
+    """
+    items = pub_re.search(dblp_block)
+    if items is None:
+        return None
+    key, title, typ, authors, url, stream, pages, venue, year = items.groups()
+    typ = typ.lower()
+    typ = DBLP_TYPES.get(typ, typ)
+    if stream:
+        stream = streams_re.findall(stream)
+    authors = {i: n for n, i in authid_re.findall(authors)}
+    if authors:
+        return key, title, typ, authors, url, stream, pages, venue, int(year)
+    return None
+
+
+@contextmanager
+def get_stream(source, chunk_size=1024 * 64):
+    """
+    Parameters
+    ----------
+    source: :class:`str` or :class:`~pathlib.Path`
+        Where the content. Can be on a local file or on the Internet.
+    chunk_size: :class:`int`, optional
+        Desired chunk size. For streaming gz content, must be a multiple of 32kB.
+
+    Yields
+    -------
+    iterable
+        Chunk iterator that streams the content.
+    :class:`int`
+        Source size (used later to compute ETA).
+    """
+    if isinstance(source, str) and source.startswith("https://"):
+        # URL HTTP
+        with session.get(source, stream=True) as r:
+            r.raise_for_status()
+            total = int(r.headers.get("content-length", 0)) or None
+            yield r.iter_content(chunk_size=chunk_size), total
+    else:
+        source = Path(source)
+        if not source.exists():
+            yield [], 0
+            return None
+        total = source.stat().st_size
+        with source.open("rb") as file_handle:
+            def read_chunks():
+                while True:
+                    chunk = file_handle.read(chunk_size)
+                    if not chunk:
+                        break
+                    yield chunk
+            yield read_chunks(), total
+
+
+def publis_streamer(source, chunk_size=1024 * 64, encoding="unicode_escape"):
+    """
+    Parameters
+    ----------
+    source: :class:`str` or :class:`~pathlib.Path`
+        Where the DBLP turtle content is. Can be on a local file or on the Internet.
+    chunk_size: :class:`int`, optional
+        Desired chunk size. Must be a multiple of 32kB.
+    encoding: :class:`str`, default=unicode_escape
+        Encoding of stream.
+
+    Yields
+    -------
+    key: :class:`str`
+        DBLP key.
+    title: :class:`str`
+        Publication title.
+    type: :class:`str`
+        Type of publication.
+    authors: :class:`dict`
+        Publication authors (key -> name).
+    venue: :class:`str`
+        Publication venue (conf/journal).
+    year: :class:`int`
+        Year of publication.
+    """
+    with get_stream(source, chunk_size=chunk_size) as (stream, total):
+        with tqdm(total=total, unit="B", unit_scale=True, unit_divisor=1024, desc="Processing") as pbar:
+            decomp = zlib.decompressobj(16 + zlib.MAX_WBITS)
+            text_buffer = ""
+            for chunk in stream:
+                if not chunk:
+                    continue
+
+                pbar.update(len(chunk))
+                data = decomp.decompress(chunk)
+                if not data:
+                    continue
+                text_buffer += data.decode(encoding, errors="replace")
+
+                blocks = text_buffer.split("\n\n")
+                text_buffer = blocks[-1]
+                for block in blocks[:-1]:
+                    pub = parse_block(block)
+                    if pub:
+                        yield pub
+
+        data = decomp.flush()
+        if data:
+            text_buffer += data.decode(encoding, errors="replace")
+
+        if text_buffer:
+            blocks = text_buffer.split("\n\n")
+            for block in blocks:
+                pub = parse_block(block)
+                if pub:
+                    yield pub

gismap/sources/hal.py

@@ -44,8 +44,7 @@ class HAL(DB):
         >>> HAL.search_author("Ana Busic")
         [HALAuthor(name='Ana Busic', key='anabusic')]
         >>> HAL.search_author("Potop-Butucaru Maria")  # doctest:  +NORMALIZE_WHITESPACE
-        [HALAuthor(name='Potop-Butucaru Maria', key='858256', key_type='pid'),
-        HALAuthor(name='Potop-Butucaru Maria', key='841868', key_type='pid')]
+        [HALAuthor(name='Potop-Butucaru Maria', key='841868', key_type='pid')]
         >>> diego = HAL.search_author("Diego Perino")
         >>> diego  # doctest:  +NORMALIZE_WHITESPACE
         [HALAuthor(name='Diego Perino', key='847558', key_type='pid'),
@@ -122,8 +121,8 @@ class HAL(DB):
         >>> diego = publications[2].authors[3]
         >>> diego
         HALAuthor(name='Diego Perino', key='Diego Perino', key_type='fullname')
-        >>> len(diego.get_publications())
-        28
+        >>> len(diego.get_publications()) > 28
+        True
         >>> publications[-7] # doctest:  +NORMALIZE_WHITESPACE
         HALPublication(title='Upper bounds for stabilization in acyclic preference-based systems',
         authors=[HALAuthor(name='Fabien Mathieu', key='fabien-mathieu')],
@@ -134,12 +133,12 @@ class HAL(DB):
 
         >>> maria = HAL.search_author('Maria Potop-Butucaru')
         >>> maria  # doctest: +NORMALIZE_WHITESPACE
-        [HALAuthor(name='Maria Potop-Butucaru', key='858256', key_type='pid'),
-        HALAuthor(name='Maria Potop-Butucaru', key='841868', key_type='pid')]
-        >>> len(HAL.from_author(maria[0]))
-        26
-        >>> len(maria[1].get_publications())
-        124
+        [HALAuthor(name='Maria Potop-Butucaru', key='841868', key_type='pid')]
+        >>> n_pubs = len(HAL.from_author(maria[0]))
+        >>> n_pubs > 200
+        True
+        >>> n_pubs == len(maria[0].get_publications())
+        True
 
         Note: an error is raised if not enough data is provided
 
@@ -173,6 +172,9 @@ class HAL(DB):
         r = get(api, params=params)
         response = json.loads(r)["response"]
         res = [HALPublication.from_json(r) for r in response.get("docs", [])]
+        if len(res) == 0 and a.key_type != "fullname":
+            name = a.name
+            return HAL.from_author(HALAuthor(name=name, key=name, key_type="fullname"))
         return res
 
 
@@ -185,6 +187,13 @@ class HALAuthor(Author, HAL):
     _img: str = None
     _cv: bool = None
 
+    def __post_init__(self):
+        if self.key and self.key_type is None:
+            if self.key.isdigit():
+                self.key_type = "pid"
+            if " " in self.key:
+                self.key_type = "fullname"
+
     def check_cv(self):
         if self.key_type is not None:
             self._cv = False