pub-analyzer 0.5.0__tar.gz → 0.5.1__tar.gz

{pub_analyzer-0.5.0 → pub_analyzer-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pub-analyzer
-Version: 0.5.0
+Version: 0.5.1
 Summary: A text user interface, written in python, which automates the generation of scientific production reports using OpenAlex
 License: MIT
 Author: Alejandro Gaspar
@@ -22,7 +22,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Typing :: Typed
-Requires-Dist: httpx (==0.28.1)
+Requires-Dist: httpx[http2] (==0.28.1)
 Requires-Dist: pydantic (==2.11.7)
 Requires-Dist: textual (==0.85.2)
 Requires-Dist: typst (==0.13.2)

pub_analyzer-0.5.1/pub_analyzer/internal/limiter.py

@@ -0,0 +1,34 @@
+"""Rate limiter module."""
+
+import asyncio
+import time
+
+
+class RateLimiter:
+    """Rate limiter."""
+
+    def __init__(self, rate: int, per_second: float = 1.0) -> None:
+        self.rate = rate
+        self.per = per_second
+        self._tokens = float(rate)
+        self._updated_at = time.monotonic()
+        self._lock = asyncio.Lock()
+
+    async def acquire(self) -> None:
+        """Wait until new token is available."""
+        while True:
+            async with self._lock:
+                now = time.monotonic()
+                elapsed = now - self._updated_at
+                if elapsed > 0:
+                    self._tokens = min(self.rate, self._tokens + elapsed * (self.rate / self.per))
+                    self._updated_at = now
+
+                if self._tokens >= 1.0:
+                    self._tokens -= 1.0
+                    return
+
+                missing = 1.0 - self._tokens
+                wait_time = missing * (self.per / self.rate)
+
+            await asyncio.sleep(wait_time)

{pub_analyzer-0.5.0 → pub_analyzer-0.5.1}/pub_analyzer/internal/report.py

@@ -9,6 +9,7 @@ from pydantic import TypeAdapter
 from textual import log
 
 from pub_analyzer.internal import identifier
+from pub_analyzer.internal.limiter import RateLimiter
 from pub_analyzer.models.author import Author, AuthorOpenAlexKey, AuthorResult, DehydratedAuthor
 from pub_analyzer.models.institution import DehydratedInstitution, Institution, InstitutionOpenAlexKey, InstitutionResult
 from pub_analyzer.models.report import (
@@ -31,6 +32,10 @@ FromDate = NewType("FromDate", datetime.datetime)
 ToDate = NewType("ToDate", datetime.datetime)
 """DateTime marker for works published up to this date."""
 
+REQUEST_RATE_PER_SECOND = 8
+"""The OpenAlex API requires a maximum of 10 requests per second. We limit this to 8 per second."""
+PER_PAGE_SIZE = 100
+
 
 def _get_author_profiles_keys(
     author: Author, extra_profiles: list[Author | AuthorResult | DehydratedAuthor] | None
@@ -149,7 +154,7 @@ def _get_valid_works(works: list[dict[str, Any]]) -> list[dict[str, Any]]:
     return valid_works
 
 
-async def _get_works(client: httpx.AsyncClient, url: str) -> list[Work]:
+async def _get_works(client: httpx.AsyncClient, url: str, limiter: RateLimiter) -> list[Work]:
     """Get all works given a URL.
 
     Iterate over all pages of the URL
@@ -164,6 +169,7 @@ async def _get_works(client: httpx.AsyncClient, url: str) -> list[Work]:
     Raises:
         httpx.HTTPStatusError: One response from OpenAlex API had an error HTTP status of 4xx or 5xx.
     """
+    await limiter.acquire()
     response = await client.get(url=url, follow_redirects=True)
     response.raise_for_status()
 
@@ -174,13 +180,14 @@ async def _get_works(client: httpx.AsyncClient, url: str) -> list[Work]:
     works_data = list(_get_valid_works(json_response["results"]))
 
     for page_number in range(1, page_count):
+        await limiter.acquire()
         page_result = (await client.get(url + f"&page={page_number + 1}", follow_redirects=True)).json()
         works_data.extend(_get_valid_works(page_result["results"]))
 
     return TypeAdapter(list[Work]).validate_python(works_data)
 
 
-async def _get_source(client: httpx.AsyncClient, url: str) -> Source:
+async def _get_source(client: httpx.AsyncClient, url: str, limiter: RateLimiter) -> Source:
     """Get source given a URL.
 
     Args:
@@ -193,6 +200,7 @@ async def _get_source(client: httpx.AsyncClient, url: str) -> Source:
     Raises:
         httpx.HTTPStatusError: One response from OpenAlex API had an error HTTP status of 4xx or 5xx.
     """
+    await limiter.acquire()
     response = await client.get(url=url, follow_redirects=True)
     response.raise_for_status()
 
@@ -237,13 +245,12 @@ async def make_author_report(
 
     pub_from_filter = f",from_publication_date:{pub_from_date:%Y-%m-%d}" if pub_from_date else ""
     pub_to_filter = f",to_publication_date:{pub_to_date:%Y-%m-%d}" if pub_to_date else ""
-    url = (
-        f"https://api.openalex.org/works?filter=author.id:{profiles_query_parameter}{pub_from_filter}{pub_to_filter}&sort=publication_date"
-    )
+    url = f"https://api.openalex.org/works?filter=author.id:{profiles_query_parameter}{pub_from_filter}{pub_to_filter}&sort=publication_date&per-page={PER_PAGE_SIZE}"
 
-    async with httpx.AsyncClient() as client:
+    limiter = RateLimiter(rate=REQUEST_RATE_PER_SECOND, per_second=1.0)
+    async with httpx.AsyncClient(http2=True, timeout=None) as client:
         # Getting all the author works.
-        author_works = await _get_works(client, url)
+        author_works = await _get_works(client, url, limiter)
 
         # Extra filters
         cited_from_filter = f",from_publication_date:{cited_from_date:%Y-%m-%d}" if cited_from_date else ""
@@ -263,9 +270,7 @@ async def make_author_report(
             log.info(f"[{work_id}] Work [{idx_work}/{author_works_count}]")
 
             work_authors = _get_authors_list(authorships=author_work.authorships)
-            cited_by_api_url = (
-                f"https://api.openalex.org/works?filter=cites:{work_id}{cited_from_filter}{cited_to_filter}&sort=publication_date"
-            )
+            cited_by_api_url = f"https://api.openalex.org/works?filter=cites:{work_id}{cited_from_filter}{cited_to_filter}&sort=publication_date&per-page={PER_PAGE_SIZE}"
 
             # Adding the type of OpenAccess in the counter.
             open_access_summary.add_oa_type(author_work.open_access.oa_status)
@@ -282,7 +287,7 @@ async def make_author_report(
                 if location.source and not any(source.id == location.source.id for source in dehydrated_sources):
                     dehydrated_sources.append(location.source)
 
-            cited_by_works = await _get_works(client, cited_by_api_url)
+            cited_by_works = await _get_works(client, cited_by_api_url, limiter)
             cited_by: list[CitationReport] = []
             work_citation_summary = CitationSummary()
             for cited_by_work in cited_by_works:
@@ -305,7 +310,7 @@ async def make_author_report(
             source_url = f"https://api.openalex.org/sources/{source_id}"
 
             log.info(f"Getting Sources... [{idx}/{sources_count}]")
-            sources.append(await _get_source(client, source_url))
+            sources.append(await _get_source(client, source_url, limiter))
 
         # Sort sources by h_index
         sources_sorted = sorted(sources, key=lambda source: source.summary_stats.two_yr_mean_citedness, reverse=True)
@@ -352,11 +357,12 @@ async def make_institution_report(
 
     pub_from_filter = f",from_publication_date:{pub_from_date:%Y-%m-%d}" if pub_from_date else ""
     pub_to_filter = f",to_publication_date:{pub_to_date:%Y-%m-%d}" if pub_to_date else ""
-    url = f"https://api.openalex.org/works?filter=institutions.id:{institution_query_parameter}{pub_from_filter}{pub_to_filter}&sort=publication_date"
+    url = f"https://api.openalex.org/works?filter=institutions.id:{institution_query_parameter}{pub_from_filter}{pub_to_filter}&sort=publication_date&per-page={PER_PAGE_SIZE}"
 
-    async with httpx.AsyncClient() as client:
+    limiter = RateLimiter(rate=REQUEST_RATE_PER_SECOND, per_second=1.0)
+    async with httpx.AsyncClient(http2=True, timeout=None) as client:
         # Getting all the institution works.
-        institution_works = await _get_works(client=client, url=url)
+        institution_works = await _get_works(client=client, url=url, limiter=limiter)
 
         # Extra filters
         cited_from_filter = f",from_publication_date:{cited_from_date:%Y-%m-%d}" if cited_from_date else ""
@@ -376,9 +382,7 @@ async def make_institution_report(
             log.info(f"[{work_id}] Work [{idx_work}/{institution_works_count}]")
 
             work_authors = _get_authors_list(authorships=institution_work.authorships)
-            cited_by_api_url = (
-                f"https://api.openalex.org/works?filter=cites:{work_id}{cited_from_filter}{cited_to_filter}&sort=publication_date"
-            )
+            cited_by_api_url = f"https://api.openalex.org/works?filter=cites:{work_id}{cited_from_filter}{cited_to_filter}&sort=publication_date&per-page={PER_PAGE_SIZE}"
 
             # Adding the type of OpenAccess in the counter.
             open_access_summary.add_oa_type(institution_work.open_access.oa_status)
@@ -395,7 +399,7 @@ async def make_institution_report(
                 if location.source and not any(source.id == location.source.id for source in dehydrated_sources):
                     dehydrated_sources.append(location.source)
 
-            cited_by_works = await _get_works(client, cited_by_api_url)
+            cited_by_works = await _get_works(client, cited_by_api_url, limiter)
             cited_by: list[CitationReport] = []
             work_citation_summary = CitationSummary()
             for cited_by_work in cited_by_works:
@@ -418,7 +422,7 @@ async def make_institution_report(
             source_url = f"https://api.openalex.org/sources/{source_id}"
 
             log.debug(f"[{work_id}] Getting Sources... [{idx}/{sources_count}]")
-            sources.append(await _get_source(client, source_url))
+            sources.append(await _get_source(client, source_url, limiter))
 
         # Sort sources by h_index
         sources_sorted = sorted(sources, key=lambda source: source.summary_stats.two_yr_mean_citedness, reverse=True)

{pub_analyzer-0.5.0 → pub_analyzer-0.5.1}/pub_analyzer/internal/templates/author_report.typ

@@ -191,7 +191,7 @@
       [*Year*], [*Works count*], [*Cited by count*],
 
       // Content
-      ..author.at("counts_by_year").slice(0, 8).map(
+      ..author.at("counts_by_year").slice(0, calc.min(author.at("counts_by_year").len(), 8)).map(
         ((year, works_count, cited_by_count)) => (
           table.cell([#year]),
           table.cell([#works_count]),
@@ -216,7 +216,7 @@
         x-label: none, y-label: none,
         {
           plot.add((
-            ..author.at("counts_by_year").slice(0, 8).map(
+            ..author.at("counts_by_year").slice(0, calc.min(author.at("counts_by_year").len(), 8)).map(
               ((year, works_count, cited_by_count)) => (
                 (year, cited_by_count)
               )

{pub_analyzer-0.5.0 → pub_analyzer-0.5.1}/pub_analyzer/widgets/report/core.py

@@ -3,6 +3,7 @@
 import datetime
 import pathlib
 from enum import Enum
+from time import time
 from typing import ClassVar
 
 import httpx
@@ -105,7 +106,9 @@ class CreateReportWidget(Static):
     async def mount_report(self) -> None:
         """Mount report."""
         try:
+            start = time()
             report_widget = await self.make_report()
+            elapsed = time() - start
         except httpx.HTTPStatusError as exc:
             self.query_one(LoadingIndicator).display = False
             status_error = f"HTTP Exception for url: {exc.request.url}. Status code: {exc.response.status_code}"
@@ -117,6 +120,13 @@ class CreateReportWidget(Static):
             )
             return None
 
+        self.app.notify(
+            title="Report created!",
+            message=f"Elapsed {elapsed:.2f}s",
+            severity="information",
+            timeout=20.0,
+        )
+
         container = self.query_one(Container)
         await container.mount(report_widget)
 

{pub_analyzer-0.5.0 → pub_analyzer-0.5.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pub-analyzer"
-version = "0.5.0"
+version = "0.5.1"
 description = "A text user interface, written in python, which automates the generation of scientific production reports using OpenAlex"
 
 authors = ["Alejandro Gaspar <alejandro@gaspar.land>"]
@@ -42,7 +42,7 @@ pub-analyzer = "pub_analyzer.main:run"
 python = "^3.10"
 
 textual = "0.85.2"
-httpx = "0.28.1"
+httpx = {version = "0.28.1", extras = ["http2"]}
 pydantic = "2.11.7"
 
 typst = "0.13.2"