paperscraper 0.2.5__tar.gz → 0.2.6__tar.gz

{paperscraper-0.2.5 → paperscraper-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paperscraper
-Version: 0.2.5
+Version: 0.2.6
 Summary: paperscraper: Package to scrape papers.
 Home-page: https://github.com/PhosphorylatedRabbits/paperscraper
 Author: Jannis Born, Matteo Manica

{paperscraper-0.2.5 → paperscraper-0.2.6}/paperscraper/__init__.py

@@ -1,6 +1,6 @@
 """Initialize the module."""
 __name__ = "paperscraper"
-__version__ = "0.2.5"
+__version__ = "0.2.6"
 
 import logging
 import os

{paperscraper-0.2.5 → paperscraper-0.2.6}/paperscraper/arxiv/arxiv.py

@@ -1,7 +1,9 @@
 from typing import Dict, List, Union
 
-import arxiv
 import pandas as pd
+from tqdm import tqdm
+
+import arxiv
 
 from ..utils import dump_papers
 from .utils import get_query_from_keywords
@@ -10,6 +12,7 @@ arxiv_field_mapper = {
     "published": "date",
     "journal_ref": "journal",
     "summary": "abstract",
+    "entry_id": "doi",
 }
 
 # Authors, date, and journal fields need specific processing
@@ -17,6 +20,7 @@ process_fields = {
     "authors": lambda authors: ", ".join([a.name for a in authors]),
     "date": lambda date: date.strftime("%Y-%m-%d"),
     "journal": lambda j: j if j is not None else "",
+    "doi": lambda entry_id: f"10.48550/arXiv.{entry_id.split('/')[-1].split('v')[0]}",
 }
 
 
@@ -57,9 +61,9 @@ def get_arxiv_papers(
                     arxiv_field_mapper.get(key, key), lambda x: x
                 )(value)
                 for key, value in vars(paper).items()
-                if arxiv_field_mapper.get(key, key) in fields
+                if arxiv_field_mapper.get(key, key) in fields and key != "doi"
             }
-            for paper in results
+            for paper in tqdm(results, desc=f"Processing {query}")
         ]
     )
     return processed
@@ -70,7 +74,7 @@ def get_and_dump_arxiv_papers(
     output_filepath: str,
     fields: List = ["title", "authors", "date", "abstract", "journal", "doi"],
     *args,
-    **kwargs
+    **kwargs,
 ):
     """
     Combines get_arxiv_papers and dump_papers.

{paperscraper-0.2.5 → paperscraper-0.2.6}/paperscraper.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paperscraper
-Version: 0.2.5
+Version: 0.2.6
 Summary: paperscraper: Package to scrape papers.
 Home-page: https://github.com/PhosphorylatedRabbits/paperscraper
 Author: Jannis Born, Matteo Manica