pub-analyzer 0.5.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

pub_analyzer/internal/limiter.py

@@ -0,0 +1,34 @@
+"""Rate limiter module."""
+
+import asyncio
+import time
+
+
+class RateLimiter:
+    """Rate limiter."""
+
+    def __init__(self, rate: int, per_second: float = 1.0) -> None:
+        self.rate = rate
+        self.per = per_second
+        self._tokens = float(rate)
+        self._updated_at = time.monotonic()
+        self._lock = asyncio.Lock()
+
+    async def acquire(self) -> None:
+        """Wait until new token is available."""
+        while True:
+            async with self._lock:
+                now = time.monotonic()
+                elapsed = now - self._updated_at
+                if elapsed > 0:
+                    self._tokens = min(self.rate, self._tokens + elapsed * (self.rate / self.per))
+                    self._updated_at = now
+
+                if self._tokens >= 1.0:
+                    self._tokens -= 1.0
+                    return
+
+                missing = 1.0 - self._tokens
+                wait_time = missing * (self.per / self.rate)
+
+            await asyncio.sleep(wait_time)

pub_analyzer/internal/report.py

@@ -9,6 +9,7 @@ from pydantic import TypeAdapter
 from textual import log
 
 from pub_analyzer.internal import identifier
+from pub_analyzer.internal.limiter import RateLimiter
 from pub_analyzer.models.author import Author, AuthorOpenAlexKey, AuthorResult, DehydratedAuthor
 from pub_analyzer.models.institution import DehydratedInstitution, Institution, InstitutionOpenAlexKey, InstitutionResult
 from pub_analyzer.models.report import (
@@ -31,6 +32,10 @@ FromDate = NewType("FromDate", datetime.datetime)
 ToDate = NewType("ToDate", datetime.datetime)
 """DateTime marker for works published up to this date."""
 
+REQUEST_RATE_PER_SECOND = 8
+"""The OpenAlex API requires a maximum of 10 requests per second. We limit this to 8 per second."""
+PER_PAGE_SIZE = 100
+
 
 def _get_author_profiles_keys(
     author: Author, extra_profiles: list[Author | AuthorResult | DehydratedAuthor] | None
@@ -149,7 +154,7 @@ def _get_valid_works(works: list[dict[str, Any]]) -> list[dict[str, Any]]:
     return valid_works
 
 
-async def _get_works(client: httpx.AsyncClient, url: str) -> list[Work]:
+async def _get_works(client: httpx.AsyncClient, url: str, limiter: RateLimiter) -> list[Work]:
     """Get all works given a URL.
 
     Iterate over all pages of the URL
@@ -164,6 +169,7 @@ async def _get_works(client: httpx.AsyncClient, url: str) -> list[Work]:
     Raises:
         httpx.HTTPStatusError: One response from OpenAlex API had an error HTTP status of 4xx or 5xx.
     """
+    await limiter.acquire()
     response = await client.get(url=url, follow_redirects=True)
     response.raise_for_status()
 
@@ -174,13 +180,14 @@ async def _get_works(client: httpx.AsyncClient, url: str) -> list[Work]:
     works_data = list(_get_valid_works(json_response["results"]))
 
     for page_number in range(1, page_count):
+        await limiter.acquire()
         page_result = (await client.get(url + f"&page={page_number + 1}", follow_redirects=True)).json()
         works_data.extend(_get_valid_works(page_result["results"]))
 
     return TypeAdapter(list[Work]).validate_python(works_data)
 
 
-async def _get_source(client: httpx.AsyncClient, url: str) -> Source:
+async def _get_source(client: httpx.AsyncClient, url: str, limiter: RateLimiter) -> Source:
     """Get source given a URL.
 
     Args:
@@ -193,6 +200,7 @@ async def _get_source(client: httpx.AsyncClient, url: str) -> Source:
     Raises:
         httpx.HTTPStatusError: One response from OpenAlex API had an error HTTP status of 4xx or 5xx.
     """
+    await limiter.acquire()
     response = await client.get(url=url, follow_redirects=True)
     response.raise_for_status()
 
@@ -237,13 +245,12 @@ async def make_author_report(
 
     pub_from_filter = f",from_publication_date:{pub_from_date:%Y-%m-%d}" if pub_from_date else ""
     pub_to_filter = f",to_publication_date:{pub_to_date:%Y-%m-%d}" if pub_to_date else ""
-    url = (
-        f"https://api.openalex.org/works?filter=author.id:{profiles_query_parameter}{pub_from_filter}{pub_to_filter}&sort=publication_date"
-    )
+    url = f"https://api.openalex.org/works?filter=author.id:{profiles_query_parameter}{pub_from_filter}{pub_to_filter}&sort=publication_date&per-page={PER_PAGE_SIZE}"
 
-    async with httpx.AsyncClient() as client:
+    limiter = RateLimiter(rate=REQUEST_RATE_PER_SECOND, per_second=1.0)
+    async with httpx.AsyncClient(http2=True, timeout=None) as client:
         # Getting all the author works.
-        author_works = await _get_works(client, url)
+        author_works = await _get_works(client, url, limiter)
 
         # Extra filters
         cited_from_filter = f",from_publication_date:{cited_from_date:%Y-%m-%d}" if cited_from_date else ""
@@ -263,9 +270,7 @@ async def make_author_report(
             log.info(f"[{work_id}] Work [{idx_work}/{author_works_count}]")
 
             work_authors = _get_authors_list(authorships=author_work.authorships)
-            cited_by_api_url = (
-                f"https://api.openalex.org/works?filter=cites:{work_id}{cited_from_filter}{cited_to_filter}&sort=publication_date"
-            )
+            cited_by_api_url = f"https://api.openalex.org/works?filter=cites:{work_id}{cited_from_filter}{cited_to_filter}&sort=publication_date&per-page={PER_PAGE_SIZE}"
 
             # Adding the type of OpenAccess in the counter.
             open_access_summary.add_oa_type(author_work.open_access.oa_status)
@@ -282,7 +287,7 @@ async def make_author_report(
                 if location.source and not any(source.id == location.source.id for source in dehydrated_sources):
                     dehydrated_sources.append(location.source)
 
-            cited_by_works = await _get_works(client, cited_by_api_url)
+            cited_by_works = await _get_works(client, cited_by_api_url, limiter)
             cited_by: list[CitationReport] = []
             work_citation_summary = CitationSummary()
             for cited_by_work in cited_by_works:
@@ -305,7 +310,7 @@ async def make_author_report(
             source_url = f"https://api.openalex.org/sources/{source_id}"
 
             log.info(f"Getting Sources... [{idx}/{sources_count}]")
-            sources.append(await _get_source(client, source_url))
+            sources.append(await _get_source(client, source_url, limiter))
 
         # Sort sources by h_index
         sources_sorted = sorted(sources, key=lambda source: source.summary_stats.two_yr_mean_citedness, reverse=True)
@@ -352,11 +357,12 @@ async def make_institution_report(
 
     pub_from_filter = f",from_publication_date:{pub_from_date:%Y-%m-%d}" if pub_from_date else ""
     pub_to_filter = f",to_publication_date:{pub_to_date:%Y-%m-%d}" if pub_to_date else ""
-    url = f"https://api.openalex.org/works?filter=institutions.id:{institution_query_parameter}{pub_from_filter}{pub_to_filter}&sort=publication_date"
+    url = f"https://api.openalex.org/works?filter=institutions.id:{institution_query_parameter}{pub_from_filter}{pub_to_filter}&sort=publication_date&per-page={PER_PAGE_SIZE}"
 
-    async with httpx.AsyncClient() as client:
+    limiter = RateLimiter(rate=REQUEST_RATE_PER_SECOND, per_second=1.0)
+    async with httpx.AsyncClient(http2=True, timeout=None) as client:
         # Getting all the institution works.
-        institution_works = await _get_works(client=client, url=url)
+        institution_works = await _get_works(client=client, url=url, limiter=limiter)
 
         # Extra filters
         cited_from_filter = f",from_publication_date:{cited_from_date:%Y-%m-%d}" if cited_from_date else ""
@@ -376,9 +382,7 @@ async def make_institution_report(
             log.info(f"[{work_id}] Work [{idx_work}/{institution_works_count}]")
 
             work_authors = _get_authors_list(authorships=institution_work.authorships)
-            cited_by_api_url = (
-                f"https://api.openalex.org/works?filter=cites:{work_id}{cited_from_filter}{cited_to_filter}&sort=publication_date"
-            )
+            cited_by_api_url = f"https://api.openalex.org/works?filter=cites:{work_id}{cited_from_filter}{cited_to_filter}&sort=publication_date&per-page={PER_PAGE_SIZE}"
 
             # Adding the type of OpenAccess in the counter.
             open_access_summary.add_oa_type(institution_work.open_access.oa_status)
@@ -395,7 +399,7 @@ async def make_institution_report(
                 if location.source and not any(source.id == location.source.id for source in dehydrated_sources):
                     dehydrated_sources.append(location.source)
 
-            cited_by_works = await _get_works(client, cited_by_api_url)
+            cited_by_works = await _get_works(client, cited_by_api_url, limiter)
             cited_by: list[CitationReport] = []
             work_citation_summary = CitationSummary()
             for cited_by_work in cited_by_works:
@@ -418,7 +422,7 @@ async def make_institution_report(
             source_url = f"https://api.openalex.org/sources/{source_id}"
 
             log.debug(f"[{work_id}] Getting Sources... [{idx}/{sources_count}]")
-            sources.append(await _get_source(client, source_url))
+            sources.append(await _get_source(client, source_url, limiter))
 
         # Sort sources by h_index
         sources_sorted = sorted(sources, key=lambda source: source.summary_stats.two_yr_mean_citedness, reverse=True)

pub_analyzer/internal/templates/author_report.typ

@@ -191,7 +191,7 @@
       [*Year*], [*Works count*], [*Cited by count*],
 
       // Content
-      ..author.at("counts_by_year").slice(0, 8).map(
+      ..author.at("counts_by_year").slice(0, calc.min(author.at("counts_by_year").len(), 8)).map(
         ((year, works_count, cited_by_count)) => (
           table.cell([#year]),
           table.cell([#works_count]),
@@ -216,7 +216,7 @@
         x-label: none, y-label: none,
         {
           plot.add((
-            ..author.at("counts_by_year").slice(0, 8).map(
+            ..author.at("counts_by_year").slice(0, calc.min(author.at("counts_by_year").len(), 8)).map(
               ((year, works_count, cited_by_count)) => (
                 (year, cited_by_count)
               )

pub_analyzer/widgets/report/core.py

@@ -3,6 +3,7 @@
 import datetime
 import pathlib
 from enum import Enum
+from time import time
 from typing import ClassVar
 
 import httpx
@@ -105,7 +106,9 @@ class CreateReportWidget(Static):
     async def mount_report(self) -> None:
         """Mount report."""
         try:
+            start = time()
             report_widget = await self.make_report()
+            elapsed = time() - start
         except httpx.HTTPStatusError as exc:
             self.query_one(LoadingIndicator).display = False
             status_error = f"HTTP Exception for url: {exc.request.url}. Status code: {exc.response.status_code}"
@@ -117,6 +120,13 @@ class CreateReportWidget(Static):
             )
             return None
 
+        self.app.notify(
+            title="Report created!",
+            message=f"Elapsed {elapsed:.2f}s",
+            severity="information",
+            timeout=20.0,
+        )
+
         container = self.query_one(Container)
         await container.mount(report_widget)
 

{pub_analyzer-0.5.0.dist-info → pub_analyzer-0.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pub-analyzer
-Version: 0.5.0
+Version: 0.5.1
 Summary: A text user interface, written in python, which automates the generation of scientific production reports using OpenAlex
 License: MIT
 Author: Alejandro Gaspar
@@ -22,7 +22,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Typing :: Typed
-Requires-Dist: httpx (==0.28.1)
+Requires-Dist: httpx[http2] (==0.28.1)
 Requires-Dist: pydantic (==2.11.7)
 Requires-Dist: textual (==0.85.2)
 Requires-Dist: typst (==0.13.2)

{pub_analyzer-0.5.0.dist-info → pub_analyzer-0.5.1.dist-info}/RECORD

@@ -13,9 +13,10 @@ pub_analyzer/css/tabs.tcss,sha256=dS7y6ZZmo1Vw7Wqpx66-O-oE7zeqPE9reWqIhQ1KcZs,31
 pub_analyzer/css/tree.tcss,sha256=5BSabX9ZmRL3VTz0Gya2RRJnWrwdIF9cTf6dXj2R4kE,818
 pub_analyzer/internal/__init__.py,sha256=9aqrBJDedUiBO5kEO81kSAuPbOSFoaDZZK8w5NydPhs,22
 pub_analyzer/internal/identifier.py,sha256=LDYew25TLuwqJHmLg9iRNTURWynN27ZbTxTVGbuOUD0,2939
+pub_analyzer/internal/limiter.py,sha256=1YaVBSSG7IfFg0nhD_up21NNL_H2Q4qaIQTvZS674Vo,1002
 pub_analyzer/internal/render.py,sha256=uF1LsY39UkTpkTJgU4hyYnVv6b1MCQayubrPwrGW2DI,1271
-pub_analyzer/internal/report.py,sha256=EzMB0oeD1D9yl_0JopdN6o-Im94WJ18QKJvgxsHyVNE,17437
-pub_analyzer/internal/templates/author_report.typ,sha256=1JVrqQFRAh0KPE7p1L6mnhU2C1jUzWeI-N5wRVvqLiQ,16837
+pub_analyzer/internal/report.py,sha256=RnX3EELW33ABwEu1W506_0q7gWQyF6Rcds-YAbXYlow,18046
+pub_analyzer/internal/templates/author_report.typ,sha256=XdqPmBptlC46vDORDFs-YaILehWh7lDuCo0cbyMPGHo,16927
 pub_analyzer/main.py,sha256=0iNj4cggG-HJ8FMODwZ67Yp3-GaFPw-gUEcSCCzwMcc,2332
 pub_analyzer/models/__init__.py,sha256=hvR6m379slQw7gSwnl_OFY21Ytv90mmmOe7bp8vZYkk,59
 pub_analyzer/models/author.py,sha256=NvFmvSsmnchz2lo7m69NE3kjLYP0CXICRAolnvcznW8,2118
@@ -48,7 +49,7 @@ pub_analyzer/widgets/report/__init__.py,sha256=oolRVss3JKaQHaQVDncjtxbLINRJ5Rd1u
 pub_analyzer/widgets/report/author.py,sha256=IEfRDfsA8jcmFwQQk1O-iuh8MKr4DbzBPpjoE8xECZA,1459
 pub_analyzer/widgets/report/cards.py,sha256=2jf9cpfzVFZO0I9b29bkNaVhENMnfL26etEpUG-NMk0,4854
 pub_analyzer/widgets/report/concept.py,sha256=xiGXy_RXO_XmdqnlePkOozYPmQrsDdqKPMRXHsZbDP0,1485
-pub_analyzer/widgets/report/core.py,sha256=V4fwBHnSqkUzC1da8sGYqzsPKQRofA7GqpEhbEXsk4s,11436
+pub_analyzer/widgets/report/core.py,sha256=Bgy_fK-IwGjoIidcr687xXsHzN3LEml-A3ykyXNeVW8,11704
 pub_analyzer/widgets/report/editor.py,sha256=WlhjNQCrqeot2rvV1266Vr8yDYJQLL1lJ1XY040UoJI,2768
 pub_analyzer/widgets/report/export.py,sha256=as2yM2FXsqgvMnF4KVWVuxboULXqJ62v7wzMYek23s4,4633
 pub_analyzer/widgets/report/grants.py,sha256=m183W6djVhucAuYs-EhjkHuA9heqpGwsW_iRouVQsns,1347
@@ -61,8 +62,8 @@ pub_analyzer/widgets/search/__init__.py,sha256=90L9IghqXD2jAWBKWK6-UeHLSVlci7D3_
 pub_analyzer/widgets/search/core.py,sha256=4NvowtBcrH1fmob9kuF7v9Tq3Nd99jzB2S7xaD8OYeI,3861
 pub_analyzer/widgets/search/results.py,sha256=3ko7zcToGp9MV-mzz_9uTJxSec7IozlIWDZe7QeRmj0,3703
 pub_analyzer/widgets/sidebar.py,sha256=XlIshlCVW5Bb3MXFPnU9is0qQrUrGdT6xlkKiYNEcAM,2704
-pub_analyzer-0.5.0.dist-info/LICENSE,sha256=OPopoEowTMKqIea8Kbxk3TKdCQ97YkLvIknjTHE5oCI,1080
-pub_analyzer-0.5.0.dist-info/METADATA,sha256=Qkks_jP-JNnhGunpMmHuOgpnkpf6YZ2YuzuwcXYlVRU,4540
-pub_analyzer-0.5.0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-pub_analyzer-0.5.0.dist-info/entry_points.txt,sha256=mVb_gUNX_-aVWHlNKLjcMAS8YLgNnSq9JLRXVJGIF2c,54
-pub_analyzer-0.5.0.dist-info/RECORD,,
+pub_analyzer-0.5.1.dist-info/LICENSE,sha256=OPopoEowTMKqIea8Kbxk3TKdCQ97YkLvIknjTHE5oCI,1080
+pub_analyzer-0.5.1.dist-info/METADATA,sha256=ebqDbZ41qqzCXNSHqmxabArEXoEcWsmx9BNOML_r8Mk,4547
+pub_analyzer-0.5.1.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+pub_analyzer-0.5.1.dist-info/entry_points.txt,sha256=mVb_gUNX_-aVWHlNKLjcMAS8YLgNnSq9JLRXVJGIF2c,54
+pub_analyzer-0.5.1.dist-info/RECORD,,