gismap 0.3.0__py3-none-any.whl → 0.4.1__py3-none-any.whl

gismap/sources/dblp.py

@@ -13,7 +13,7 @@ from gismap.utils.requests import get
 class DBLP(DB):
     db_name: ClassVar[str] = "dblp"
     author_backoff: ClassVar[float] = 5.0
-    publi_backoff: ClassVar[float] = 1.0
+    publi_backoff: ClassVar[float] = 5.0
 
     @classmethod
     def search_author(cls, name, wait=True):
@@ -98,6 +98,7 @@ class DBLPAuthor(Author, DBLP):
     key='conf/sss/Mathieu07')
 
     """
+
     key: str
     aliases: list = field(default_factory=list)
 
@@ -124,6 +125,27 @@ DBLP_TYPES = {
 
 @dataclass(repr=False)
 class DBLPPublication(Publication, DBLP):
+    """
+    Publication from the DBLP database.
+
+    Parameters
+    ----------
+    title: :class:`str`
+        Publication title.
+    authors: :class:`list`
+        List of :class:`DBLPAuthor` objects.
+    venue: :class:`str`
+        Publication venue.
+    type: :class:`str`
+        Publication type.
+    year: :class:`int`
+        Publication year.
+    key: :class:`str`
+        DBLP record key.
+    metadata: :class:`dict`
+        Additional metadata (pages, volume, etc.).
+    """
+
     key: str
     metadata: dict = field(default_factory=dict)
 

gismap/sources/dblp_ttl.py

@@ -0,0 +1,187 @@
+import re
+import zlib
+from contextlib import contextmanager
+from pathlib import Path
+
+from tqdm.auto import tqdm
+
+from gismap.utils.requests import session
+from gismap.sources.dblp import DBLP_TYPES
+
+key_re = r"<https://dblp.org/rec/([^>]+)>"
+title_re = r'.*?dblp:title\s+"([^"]+)"'
+type_re = r".*?dblp:bibtexType\s+bibtex:(\w+)"
+authors_re = r".*?dblp:hasSignature\s+(\[.*\])\s*;"
+url_re = r"(?:.*?dblp:primaryDocumentPage <([^>]+)>)?"
+stream_re = r"(?:.*?dblp:publishedInStream ([^;]+) ;)?"
+pages_re = r'(?:.*?dblp:pagination "([^"]+)")?'
+venue_re = r'(?:.*?dblp:publishedIn\s+"([^"]+?)")?'
+year_re = r'.*?"(\d{4})"\^\^<http://www.w3.org/2001/XMLSchema#gYear>'
+
+pub_re = re.compile(
+    "".join(
+        [
+            key_re,
+            title_re,
+            type_re,
+            authors_re,
+            url_re,
+            stream_re,
+            pages_re,
+            venue_re,
+            year_re,
+        ]
+    ),
+    flags=re.S,
+)
+
+streams_re = re.compile(r"<https://dblp.org/streams/((?:conf|journals)/[^>]+)>")
+
+authid_re = re.compile(
+    r'\[.*?signatureDblpName\s*?"([^"]+?)(?:\s+\d+)?".*?signatureCreator\s*<https://dblp.org/pid/([^>]+?)>.*?]',
+    flags=re.S,
+)
+
+
+def parse_block(dblp_block):
+    """
+    Parameters
+    ----------
+    dblp_block: :class:`str`
+        A DBLP publication, turtle format.
+
+    Returns
+    -------
+    key: :class:`str`
+        DBLP key.
+    title: :class:`str`
+        Publication title.
+    type: :class:`str`
+        Type of publication.
+    authors: :class:`dict`
+        Publication authors (key -> name)
+    url: :class:`str` or :class:`NoneType`
+        Publication URL.
+    stream: :class:`list` or :class:`NoneType`
+        Publication streams (normalized journal/conf).
+    pages: :class:`str` or :class:`NoneType`
+        Publication pages.
+    venue: :class:`str` or :class:`NoneType`
+        Publication venue (conf/journal).
+    year: :class:`int`
+        Year of publication.
+    """
+    items = pub_re.search(dblp_block)
+    if items is None:
+        return None
+    key, title, typ, authors, url, stream, pages, venue, year = items.groups()
+    typ = typ.lower()
+    typ = DBLP_TYPES.get(typ, typ)
+    if stream:
+        stream = streams_re.findall(stream)
+    authors = {i: n for n, i in authid_re.findall(authors)}
+    if authors:
+        return key, title, typ, authors, url, stream, pages, venue, int(year)
+    return None
+
+
+@contextmanager
+def get_stream(source, chunk_size=1024 * 64):
+    """
+    Parameters
+    ----------
+    source: :class:`str` or :class:`~pathlib.Path`
+        Where the content. Can be on a local file or on the Internet.
+    chunk_size: :class:`int`, optional
+        Desired chunk size. For streaming gz content, must be a multiple of 32kB.
+
+    Yields
+    -------
+    iterable
+        Chunk iterator that streams the content.
+    :class:`int`
+        Source size (used later to compute ETA).
+    """
+    if isinstance(source, str) and source.startswith("https://"):
+        # URL HTTP
+        with session.get(source, stream=True) as r:
+            r.raise_for_status()
+            total = int(r.headers.get("content-length", 0)) or None
+            yield r.iter_content(chunk_size=chunk_size), total
+    else:
+        source = Path(source)
+        if not source.exists():
+            yield [], 0
+            return None
+        total = source.stat().st_size
+        with source.open("rb") as file_handle:
+
+            def read_chunks():
+                while True:
+                    chunk = file_handle.read(chunk_size)
+                    if not chunk:
+                        break
+                    yield chunk
+
+            yield read_chunks(), total
+
+
+def publis_streamer(source, chunk_size=1024 * 64, encoding="unicode_escape"):
+    """
+    Parameters
+    ----------
+    source: :class:`str` or :class:`~pathlib.Path`
+        Where the DBLP turtle content is. Can be on a local file or on the Internet.
+    chunk_size: :class:`int`, optional
+        Desired chunk size. Must be a multiple of 32kB.
+    encoding: :class:`str`, default=unicode_escape
+        Encoding of stream.
+
+    Yields
+    -------
+    key: :class:`str`
+        DBLP key.
+    title: :class:`str`
+        Publication title.
+    type: :class:`str`
+        Type of publication.
+    authors: :class:`dict`
+        Publication authors (key -> name).
+    venue: :class:`str`
+        Publication venue (conf/journal).
+    year: :class:`int`
+        Year of publication.
+    """
+    with get_stream(source, chunk_size=chunk_size) as (stream, total):
+        with tqdm(
+            total=total, unit="B", unit_scale=True, unit_divisor=1024, desc="Processing"
+        ) as pbar:
+            decomp = zlib.decompressobj(16 + zlib.MAX_WBITS)
+            text_buffer = ""
+            for chunk in stream:
+                if not chunk:
+                    continue
+
+                pbar.update(len(chunk))
+                data = decomp.decompress(chunk)
+                if not data:
+                    continue
+                text_buffer += data.decode(encoding, errors="replace")
+
+                blocks = text_buffer.split("\n\n")
+                text_buffer = blocks[-1]
+                for block in blocks[:-1]:
+                    pub = parse_block(block)
+                    if pub:
+                        yield pub
+
+        data = decomp.flush()
+        if data:
+            text_buffer += data.decode(encoding, errors="replace")
+
+        if text_buffer:
+            blocks = text_buffer.split("\n\n")
+            for block in blocks:
+                pub = parse_block(block)
+                if pub:
+                    yield pub

gismap/sources/hal.py

@@ -121,8 +121,8 @@ class HAL(DB):
         >>> diego = publications[2].authors[3]
         >>> diego
         HALAuthor(name='Diego Perino', key='Diego Perino', key_type='fullname')
-        >>> len(diego.get_publications())
-        28
+        >>> len(diego.get_publications()) > 28
+        True
         >>> publications[-7] # doctest:  +NORMALIZE_WHITESPACE
         HALPublication(title='Upper bounds for stabilization in acyclic preference-based systems',
         authors=[HALAuthor(name='Fabien Mathieu', key='fabien-mathieu')],
@@ -180,6 +180,23 @@ class HAL(DB):
 
 @dataclass(repr=False)
 class HALAuthor(Author, HAL):
+    """
+    Author from the HAL (Hyper Articles en Ligne) database.
+
+    HAL is a French open archive for scholarly publications.
+
+    Parameters
+    ----------
+    name: :class:`str`
+        The author's name.
+    key: :class:`str` or :class:`int`, optional
+        HAL identifier for the author.
+    key_type: :class:`str`, optional
+        Type of key ('pid', 'fullname', or None for idHal).
+    aliases: :class:`list`
+        Alternative names for the author.
+    """
+
     key: str | int = None
     key_type: str = None
     aliases: list = field(default_factory=list)
@@ -277,6 +294,27 @@ HAL_KEYS = {
 
 @dataclass(repr=False)
 class HALPublication(Publication, HAL):
+    """
+    Publication from the HAL database.
+
+    Parameters
+    ----------
+    title: :class:`str`
+        Publication title.
+    authors: :class:`list`
+        List of :class:`HALAuthor` objects.
+    venue: :class:`str`
+        Publication venue.
+    type: :class:`str`
+        Publication type.
+    year: :class:`int`
+        Publication year.
+    key: :class:`str`
+        HAL document identifier.
+    metadata: :class:`dict`
+        Additional metadata (abstract, URL, etc.).
+    """
+
     key: str
     metadata: dict = field(default_factory=dict)