citation-map 3.4__tar.gz → 3.6__tar.gz

{citation-map-3.4 → citation-map-3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: citation-map
-Version: 3.4
+Version: 3.6
 Summary: A simple tool to generate your Google Scholar citation world map.
 Home-page: https://github.com/ChenLiu-1996/CitationMap
 Author: Chen Liu
@@ -75,9 +75,14 @@ If you open it on a browser, you will see your own version of the following cita
     output_path: str
         (default is 'citation_map.html')
         The path to the output HTML file.
-    num_processes: int
+    num_processes_major: int
         (default is 16)
-        Number of separate processes for parallel processing.
+        Number of processes for parallel processing.
+        Related to `scholarly.citedby()`. Primary reason for Google Scholar bans.
+    num_processes_minor: int
+        (default is 32)
+        Number of processes for parallel processing.
+        Related to all other processes. Usually not causing problems.
     use_proxy: bool
         (default is False)
         If true, we will use a scholarly proxy.
@@ -96,7 +101,7 @@ If you open it on a browser, you will see your own version of the following cita
 
     - From my experience, this is a good indicator that your IP address is blocked by Google Scholar due to excessive or frequent crawling (using the `scholarly` package).
     - One hot fix I found was to hop on a University VPN and run again. I typically experience this error after running the tool twice, and I need to disconnect and reconnect my VPN to "unblock" myself.
-    - In case changing your IP address does not help, you can try to reduce the number of processes (e.g., setting `num_processes=1`).
+    - In case this does not help, you can try to reduce the number of processes (e.g., setting `num_processes_major=1`).
 
 ## Changelog
 

{citation-map-3.4 → citation-map-3.6}/README.md

@@ -59,9 +59,14 @@ If you open it on a browser, you will see your own version of the following cita
     output_path: str
         (default is 'citation_map.html')
         The path to the output HTML file.
-    num_processes: int
+    num_processes_major: int
         (default is 16)
-        Number of separate processes for parallel processing.
+        Number of processes for parallel processing.
+        Related to `scholarly.citedby()`. Primary reason for Google Scholar bans.
+    num_processes_minor: int
+        (default is 32)
+        Number of processes for parallel processing.
+        Related to all other processes. Usually not causing problems.
     use_proxy: bool
         (default is False)
         If true, we will use a scholarly proxy.
@@ -80,7 +85,7 @@ If you open it on a browser, you will see your own version of the following cita
 
     - From my experience, this is a good indicator that your IP address is blocked by Google Scholar due to excessive or frequent crawling (using the `scholarly` package).
     - One hot fix I found was to hop on a University VPN and run again. I typically experience this error after running the tool twice, and I need to disconnect and reconnect my VPN to "unblock" myself.
-    - In case changing your IP address does not help, you can try to reduce the number of processes (e.g., setting `num_processes=1`).
+    - In case this does not help, you can try to reduce the number of processes (e.g., setting `num_processes_major=1`).
 
 ## Changelog
 

{citation-map-3.4 → citation-map-3.6}/citation_map/citation_map.py

@@ -6,6 +6,7 @@ import itertools
 from typing import List, Tuple
 from scholarly import scholarly, ProxyGenerator
 from geopy.geocoders import Nominatim
+
 import folium
 from tqdm import tqdm
 from multiprocessing import Pool
@@ -19,51 +20,52 @@ def fetch_google_scholar_profile(scholar_id: str):
     author = scholarly.fill(author, sections=['publications'])
     return author
 
-def find_all_citing_institutions(publications, num_processes: int = 16) -> List[Tuple[str]]:
+def find_all_citing_institutions(publications, num_processes_major: int = 16, num_processes_minor: int = 32) -> List[Tuple[str]]:
     '''
     Step 2. Final all citing institutions (i.e., insitutions of citing authors).
     '''
-    if num_processes > 1 and isinstance(num_processes, int):
+    if num_processes_minor > 1 and isinstance(num_processes_minor, int):
         # Fetch metadata for all publications, in parallel.
-        with Pool(processes=num_processes) as pool:
+        with Pool(processes=num_processes_minor) as pool:
             all_publications = list(tqdm(pool.imap(__fill_publication_metadata, publications),
                                         desc='Filling metadata for all publications...',
                                         total=len(publications)))
-
-        # Find all citing papers from all publications, in parallel.
-        with Pool(processes=num_processes) as pool:
-            all_citing_papers_nested = list(tqdm(pool.imap(__citing_papers_from_publication, all_publications),
-                                                desc='Finding all citations...',
-                                                total=len(all_publications)))
-        all_citing_papers = list(itertools.chain(*all_citing_papers_nested))
-
-        # Find all citing authors from all citing papers, in parallel.
-        with Pool(processes=num_processes) as pool:
-            all_citing_authors_nested = list(tqdm(pool.imap(__authors_from_papers, all_citing_papers),
-                                                desc='Finding all citing authors...',
-                                                total=len(all_citing_papers)))
-        all_citing_authors = list(itertools.chain(*all_citing_authors_nested))
-
-        # Find all citing insitutions from all citing authors, in parallel.
-        with Pool(processes=num_processes) as pool:
-            author_institution_tuple_list = list(tqdm(pool.imap(__institutions_from_authors, all_citing_authors),
-                                                    desc='Finding all citing institutions...',
-                                                    total=len(all_citing_authors)))
     else:
         all_publications = []
         for pub in tqdm(publications, desc='Filling metadata for all publications...', total=len(publications)):
             all_publications.append(__fill_publication_metadata(pub))
 
+    if num_processes_major > 1 and isinstance(num_processes_major, int):
+        # Find all citing papers from all publications, in parallel.
+        with Pool(processes=num_processes_major) as pool:
+            all_citing_papers_nested = list(tqdm(pool.imap(__citing_papers_from_publication, all_publications),
+                                                 desc='Finding all citations...',
+                                                 total=len(all_publications)))
+    else:
         all_citing_papers_nested = []
         for pub in tqdm(all_publications, desc='Finding all citations...', total=len(all_publications)):
-            all_citing_papers_nested.append(__citing_papers_from_publication(pub))
-        all_citing_papers = list(itertools.chain(*all_citing_papers_nested))
+            all_citing_papers_nested.append(__citing_papers_from_publication_difficult(pub))
+    all_citing_papers = list(itertools.chain(*all_citing_papers_nested))
 
+    if num_processes_minor > 1 and isinstance(num_processes_minor, int):
+        # Find all citing authors from all citing papers, in parallel.
+        with Pool(processes=num_processes_minor) as pool:
+            all_citing_authors_nested = list(tqdm(pool.imap(__authors_from_papers, all_citing_papers),
+                                                  desc='Finding all citing authors...',
+                                                  total=len(all_citing_papers)))
+    else:
         all_citing_authors_nested = []
         for paper in tqdm(all_citing_papers, desc='Finding all citing authors...', total=len(all_citing_papers)):
             all_citing_authors_nested.append(__authors_from_papers(paper))
-        all_citing_authors = list(itertools.chain(*all_citing_authors_nested))
+    all_citing_authors = list(itertools.chain(*all_citing_authors_nested))
 
+    if num_processes_minor > 1 and isinstance(num_processes_minor, int):
+        # Find all citing insitutions from all citing authors, in parallel.
+        with Pool(processes=num_processes_minor) as pool:
+            author_institution_tuple_list = list(tqdm(pool.imap(__institutions_from_authors, all_citing_authors),
+                                                    desc='Finding all citing institutions...',
+                                                    total=len(all_citing_authors)))
+    else:
         author_institution_tuple_list = []
         for author in tqdm(all_citing_authors, desc='Finding all citing institutions...', total=len(all_citing_authors)):
             author_institution_tuple_list.append(__institutions_from_authors(author))
@@ -91,12 +93,12 @@ def clean_institution_names(author_institution_tuple_list: List[Tuple[str]]) ->
                 cleaned_author_institution_tuple_list.append((author_name, cleaned_institution))
     return cleaned_author_institution_tuple_list
 
-def institution_text_to_geocode(author_institution_tuple_list: List[Tuple[str]], num_processes: int = 16) -> List[Tuple[str]]:
+def institution_text_to_geocode(author_institution_tuple_list: List[Tuple[str]], num_processes_minor: int = 32) -> List[Tuple[str]]:
     '''
     Step 4: Convert institutions in plain text to Geocode.
     '''
-    if num_processes > 1 and isinstance(num_processes, int):
-        with Pool(processes=num_processes) as pool:
+    if num_processes_minor > 1 and isinstance(num_processes_minor, int):
+        with Pool(processes=num_processes_minor) as pool:
             coordinates = list(tqdm(pool.imap(__text_to_geocode, author_institution_tuple_list),
                                     desc='Finding all geographic coordinates...',
                                     total=len(author_institution_tuple_list)))
@@ -134,10 +136,23 @@ def __fill_publication_metadata(pub):
 
 def __citing_papers_from_publication(pub):
     if 'citedby_url' in pub:
-        # time.sleep(random.uniform(1, 5))  # Random delay to reduce risk of being blocked.
-        citing_paper_obj = scholarly.citedby(pub)
-        # time.sleep(random.uniform(1, 5))  # Random delay to reduce risk of being blocked.
-        return [item for item in citing_paper_obj]
+        time.sleep(random.uniform(1, 5))  # Random delay to reduce risk of being blocked.
+        citing_paper_iterator = scholarly.citedby(pub)
+        citing_paper_list = [item for item in citing_paper_iterator]
+        return citing_paper_list
+    return []
+
+def __citing_papers_from_publication_difficult(pub):
+    if 'citedby_url' in pub:
+        time.sleep(random.uniform(1, 5))  # Random delay to reduce risk of being blocked.
+        citing_paper_iterator = scholarly.citedby(pub)
+        citing_paper_list = []
+        # Unpacking the iterator to a list invokes `scholarly`,
+        # which can easily get us blocked if not handled carefully.
+        for item in citing_paper_iterator:
+            time.sleep(random.uniform(1, 2))  # Random delay to reduce risk of being blocked.
+            citing_paper_list.append(item)
+        return citing_paper_list
     return []
 
 def __authors_from_papers(citing_paper):
@@ -155,19 +170,22 @@ def __institutions_from_authors(author_id):
         return (citing_author['name'], citing_author['affiliation'])
     return []
 
-def __text_to_geocode(author_institution_tuple):
+def __text_to_geocode(author_institution_tuple, max_attempts: int = 2):
     geolocator = Nominatim(user_agent='citation_mapper')
-    try:
-        author_name, institution_name = author_institution_tuple
-        geo_location = geolocator.geocode(institution_name)
-        if geo_location:
-            return (geo_location.latitude, geo_location.longitude, author_name, institution_name)
-    except:
-        return None
+    for _ in range(max_attempts):
+        try:
+            author_name, institution_name = author_institution_tuple
+            geo_location = geolocator.geocode(institution_name)
+            if geo_location:
+                return (geo_location.latitude, geo_location.longitude, author_name, institution_name)
+        except:
+            pass
+    return None
 
 def generate_citation_map(scholar_id: str,
                           output_path: str = 'citation_map.html',
-                          num_processes: int = 16,
+                          num_processes_major: int = 16,
+                          num_processes_minor: int = 32,
                           use_proxy: bool = False,
                           pin_colorful: bool = True,
                           print_citing_institutions: bool = True):
@@ -181,9 +199,14 @@ def generate_citation_map(scholar_id: str,
     output_path: str
         (default is 'citation_map.html')
         The path to the output HTML file.
-    num_processes: int
+    num_processes_major: int
         (default is 16)
-        Number of separate processes for parallel processing.
+        Number of processes for parallel processing.
+        Related to `scholarly.citedby()`. Primary reason for Google Scholar bans.
+    num_processes_minor: int
+        (default is 32)
+        Number of processes for parallel processing.
+        Related to all other processes. Usually not causing problems.
     use_proxy: bool
         (default is False)
         If true, we will use a scholarly proxy.
@@ -206,7 +229,9 @@ def generate_citation_map(scholar_id: str,
     author_profile = fetch_google_scholar_profile(scholar_id)
     print('\nAuthor profile found, with %d publications.\n' % len(author_profile['publications']))
 
-    author_institution_tuple_list = find_all_citing_institutions(author_profile['publications'], num_processes=num_processes)
+    author_institution_tuple_list = find_all_citing_institutions(author_profile['publications'],
+                                                                 num_processes_major=num_processes_major,
+                                                                 num_processes_minor=num_processes_minor)
     print('\nA total of %d citing institutions recorded.\n' % len(author_institution_tuple_list))
 
     if print_citing_institutions:
@@ -221,8 +246,9 @@ def generate_citation_map(scholar_id: str,
         for item in sorted(cleaned_author_institution_tuple_list):
             print(item)
 
-    coordinates = institution_text_to_geocode(author_institution_tuple_list + cleaned_author_institution_tuple_list, num_processes=num_processes)
-    print('\nConverted the institutions to Geocodes.')
+    coordinates = institution_text_to_geocode(author_institution_tuple_list + cleaned_author_institution_tuple_list,
+                                              num_processes_minor=num_processes_minor)
+    print('\nConverted the institutions to %d Geocodes.' % len(coordinates))
 
     citation_map = create_map(coordinates, pin_colorful=pin_colorful)
     citation_map.save(output_path)
@@ -234,4 +260,4 @@ if __name__ == '__main__':
     # Replace this with your Google Scholar ID.
     scholar_id = '3rDjnykAAAAJ'
     generate_citation_map(scholar_id, output_path='citation_map.html',
-                          num_processes=16, use_proxy=False, pin_colorful=True, print_citing_institutions=True)
+                          num_processes_minor=16, use_proxy=False, pin_colorful=True, print_citing_institutions=True)

{citation-map-3.4 → citation-map-3.6}/citation_map.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: citation-map
-Version: 3.4
+Version: 3.6
 Summary: A simple tool to generate your Google Scholar citation world map.
 Home-page: https://github.com/ChenLiu-1996/CitationMap
 Author: Chen Liu
@@ -75,9 +75,14 @@ If you open it on a browser, you will see your own version of the following cita
     output_path: str
         (default is 'citation_map.html')
         The path to the output HTML file.
-    num_processes: int
+    num_processes_major: int
         (default is 16)
-        Number of separate processes for parallel processing.
+        Number of processes for parallel processing.
+        Related to `scholarly.citedby()`. Primary reason for Google Scholar bans.
+    num_processes_minor: int
+        (default is 32)
+        Number of processes for parallel processing.
+        Related to all other processes. Usually not causing problems.
     use_proxy: bool
         (default is False)
         If true, we will use a scholarly proxy.
@@ -96,7 +101,7 @@ If you open it on a browser, you will see your own version of the following cita
 
     - From my experience, this is a good indicator that your IP address is blocked by Google Scholar due to excessive or frequent crawling (using the `scholarly` package).
     - One hot fix I found was to hop on a University VPN and run again. I typically experience this error after running the tool twice, and I need to disconnect and reconnect my VPN to "unblock" myself.
-    - In case changing your IP address does not help, you can try to reduce the number of processes (e.g., setting `num_processes=1`).
+    - In case this does not help, you can try to reduce the number of processes (e.g., setting `num_processes_major=1`).
 
 ## Changelog
 

{citation-map-3.4 → citation-map-3.6}/setup.py

@@ -5,7 +5,7 @@ with open('README.md') as f:
 
 setup(
     name='citation-map',
-    version='3.4',
+    version='3.6',
     license='Apache 2.0',
     author='Chen Liu',
     author_email='chen.liu.cl2482@yale.edu',