pyrefdev 2025.6__tar.gz → 2025.8__tar.gz

{pyrefdev-2025.6 → pyrefdev-2025.8}/CHANGELOG.md

@@ -4,6 +4,21 @@
 
 Nothing noticeable unreleased.
 
+## v2025.8
+
+- When no symbol is specified, the lucky button now redirects to a random API reference URL.
+- Support [passlib](https://passlib.readthedocs.io/en/stable/).
+- Updated certain torch URLs with more detailed docs.
+- Fixed symbols for [ipython](https://ipython.readthedocs.io/en/stable/).
+- Indexer: in the `add-docs` command, guess the API docs based on PyPI package information when `--index-url` is not specified.
+- Indexer: add a `crawl-pypi` command to cache the PyPI package information.
+- Indexer: add a `parse-pypi` command to parse cached PyPI package information and update the config when API docs are found.
+
+## v2025.7
+
+- Added search to [pyref.dev](https://pyref.dev).
+- When the symbol is not found, redirect to the search page.
+
 ## v2025.6
 
 - Added a `-p/--print` flag to the `pyrefdev` CLI to print the URL instead of opening it in the browser.

{pyrefdev-2025.6 → pyrefdev-2025.8}/CLAUDE.md

@@ -6,7 +6,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 
 The project has several components:
 
-1. **Web Server** (`src/pyrefdev/server.py`): FastAPI application that serves redirects based on symbol lookup. `index.html` is the landing page of the server
+1. **Web Server** (`src/pyrefdev/server.py`): FastAPI application that serves redirects based on symbol lookup. `templates/index.html` is the landing page of the server, `static/styles.css` contains CSS styles
 2. **CLI Tool** (`src/pyrefdev/__main__.py`): Command-line interface that opens documentation in browser
 3. **Indexer** (`src/pyrefdev/indexer/`): Tools for crawling, parsing, and managing documentation mappings
 4. **Mapping System** (`src/pyrefdev/mapping/`): Individual Python files per package containing symbol-to-URL mappings
@@ -48,4 +48,7 @@ pyrefdev-indexer update-landing-page
 
 ## Important Notes
 
+- Do not add tests unless explictly asked to.
+- When creating a new file with content, ensure the file has an extra new line at the end.
+- Do NOT add redundant comments describe what the cod does. When needed, DO add comments that explains "why".
 - Server deployment uses systemd and is configured in `pyrefdev.service`

{pyrefdev-2025.6 → pyrefdev-2025.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyrefdev
-Version: 2025.6
+Version: 2025.8
 Summary: The entry point to Python reference docs
 Project-URL: Homepage, https://pyref.dev
 Project-URL: Source, https://github.com/mangoumbrella/pyref.dev
@@ -51,7 +51,19 @@ It allows you to quickly jump to the official documentation for Python standard
 https://pyref.dev/<fully.qualified.symbol.name>
 ```
 
-Alternatively, you could also `pip install pyrefdev` and run the `pyrefdev` CLI tool.
+You can also search for symbols using:
+
+```
+https://pyref.dev/is?symbol=<SYMBOL>
+```
+
+And if you are feeling lucky, ask it to redirect to the first result:
+
+```
+https://pyref.dev/is?lucky=true&symbol=<SYMBOL>
+```
+
+Lastly, you can `pip install pyrefdev` and run the `pyrefdev` CLI tool.
 
 ## Examples
 

{pyrefdev-2025.6 → pyrefdev-2025.8}/README.md

@@ -14,7 +14,19 @@ It allows you to quickly jump to the official documentation for Python standard
 https://pyref.dev/<fully.qualified.symbol.name>
 ```
 
-Alternatively, you could also `pip install pyrefdev` and run the `pyrefdev` CLI tool.
+You can also search for symbols using:
+
+```
+https://pyref.dev/is?symbol=<SYMBOL>
+```
+
+And if you are feeling lucky, ask it to redirect to the first result:
+
+```
+https://pyref.dev/is?lucky=true&symbol=<SYMBOL>
+```
+
+Lastly, you can `pip install pyrefdev` and run the `pyrefdev` CLI tool.
 
 ## Examples
 

pyrefdev-2025.8/src/pyrefdev/_version.py

@@ -0,0 +1 @@
+version = "2025.8"

{pyrefdev-2025.6 → pyrefdev-2025.8}/src/pyrefdev/config.py

@@ -113,10 +113,11 @@ _packages = [
     Package(pypi="coverage", index_url="https://coverage.readthedocs.io/en/latest/"),
     Package(pypi="matplotlib", index_url="https://matplotlib.org/stable/api/index.html"),
     Package(pypi="networkx", index_url="https://networkx.org/documentation/stable/reference/index.html"),
-    Package(pypi="ipython", index_url="https://ipython.readthedocs.io/en/stable/"),
+    Package(pypi="ipython", index_url="https://ipython.readthedocs.io/en/stable/", namespaces=["IPython"]),
     Package(pypi="httplib2", index_url="https://httplib2.readthedocs.io/en/latest/"),
     Package(pypi="cyclopts", index_url="https://cyclopts.readthedocs.io/en/latest/"),
     Package(pypi="yib", index_url="https://yib.readthedocs.io/en/latest/"),
+    Package(pypi="passlib", index_url="https://passlib.readthedocs.io/en/stable/"),
     # ENTRY-LINE-MARKER
 ]
 # fmt: on

{pyrefdev-2025.6 → pyrefdev-2025.8}/src/pyrefdev/indexer/__main__.py

@@ -3,7 +3,9 @@ import cyclopts
 import pyrefdev
 from pyrefdev.indexer.add_docs import add_docs
 from pyrefdev.indexer.crawl_docs import crawl_docs
+from pyrefdev.indexer.crawl_pypi import crawl_pypi
 from pyrefdev.indexer.parse_docs import parse_docs
+from pyrefdev.indexer.parse_pypi import parse_pypi
 from pyrefdev.indexer.update_docs import update_docs
 from pyrefdev.indexer.update_landing_page import update_landing_page
 from pyrefdev.config import console
@@ -17,7 +19,9 @@ app = cyclopts.App(
 )
 app.command(add_docs)
 app.command(crawl_docs)
+app.command(crawl_pypi)
 app.command(parse_docs)
+app.command(parse_pypi)
 app.command(update_docs)
 app.command(update_landing_page)
 

{pyrefdev-2025.6 → pyrefdev-2025.8}/src/pyrefdev/indexer/add_docs.py

@@ -4,6 +4,7 @@ import re
 
 from pyrefdev import config
 from pyrefdev.config import console
+from pyrefdev.indexer.index import Index
 from pyrefdev.indexer.update_docs import update_docs
 from pyrefdev.indexer.update_landing_page import update_landing_page_with_packages
 
@@ -14,15 +15,31 @@ _MARKER = re.compile("(\n.*ENTRY-LINE-MARKER.*\n)")
 def add_docs(
     *,
     package: str,
-    docs_directory: Path | None = None,
-    url: str,
+    url: str | None = None,
     namespaces: list[str] | None = None,
     crawl: bool = True,
-    num_threads_per_package: int | None = None,
+    index: Index = Index(),
+    num_threads_per_package: int = 1,
 ) -> None:
+    """Add a new package."""
     if package in config.SUPPORTED_PACKAGES:
         console.fatal(f"Package exists: {package}")
 
+    if url is None:
+        url = index.guess_index_url(package, should_die_if_not_found=True)
+    update_config(package, url, namespaces)
+
+    if crawl:
+        importlib.reload(config)
+        update_docs(
+            index=index,
+            package=package,
+            num_threads_per_package=num_threads_per_package,
+        )
+        update_landing_page_with_packages(config.SUPPORTED_PACKAGES)
+
+
+def update_config(package: str, url: str, namespaces: list[str] | None = None) -> None:
     if namespaces:
         ns_content = ", ".join(f'"{ns}"' for ns in namespaces)
         ns_str = f", namespaces=[{ns_content}]"
@@ -33,12 +50,3 @@ def add_docs(
     config_content = config_file.read_text()
     config_content = _MARKER.sub(config_entry + r"\g<1>", config_content)
     config_file.write_text(config_content)
-
-    if crawl:
-        importlib.reload(config)
-        update_docs(
-            docs_directory=docs_directory,
-            package=package,
-            num_threads_per_package=num_threads_per_package,
-        )
-        update_landing_page_with_packages(config.SUPPORTED_PACKAGES)

{pyrefdev-2025.6 → pyrefdev-2025.8}/src/pyrefdev/indexer/crawl_docs.py

@@ -1,7 +1,6 @@
 from concurrent import futures
 from pathlib import Path
 import queue
-import tempfile
 import threading
 from urllib import error, parse
 
@@ -10,53 +9,43 @@ from packaging import version
 from rich.progress import Progress, TaskID
 
 from pyrefdev.config import console, get_packages, Package
-from pyrefdev.indexer.requests import fetch_package_version, urlopen
-from pyrefdev.indexer.schema import CrawlState
+from pyrefdev.indexer.index import Index, IndexState, urlopen
 
 
 def crawl_docs(
     *,
     package: str | None = None,
-    docs_directory: Path | None = None,
     force: bool = False,
+    index: Index = Index(),
     num_parallel_packages: int = 2,
-    num_threads_per_package: int | None = None,
+    num_threads_per_package: int = 1,
 ) -> None:
     """Crawl the docs into a local directory."""
     if num_parallel_packages <= 0:
         raise ValueError(
             f"--num-parallel-packages must be > 0, found {num_parallel_packages}"
         )
-    if num_threads_per_package is None:
-        num_threads_per_package = 2
     if num_threads_per_package <= 0:
         raise ValueError(
             f"--num-threads-per-package must be > 0, found {num_threads_per_package}"
         )
 
-    if docs_directory:
-        if docs_directory.exists():
-            if not docs_directory.is_dir():
-                console.fatal(f"{docs_directory} is not a directory")
-    else:
-        docs_directory = Path(tempfile.mkdtemp(prefix="pyref.dev."))
-
-    console.print(f"Crawling documents into {docs_directory}")
+    console.print(f"Crawling documents into {index.docs_directory}")
     packages = get_packages(package)
     with Progress(console=console) as progress:
         task = progress.add_task(
             f"Crawling {len(packages)} packages", total=len(packages)
         )
+
         def crawl_package(pkg: Package):
             try:
-                package_version = fetch_package_version(pkg)
+                package_version = index.fetch_package_version(pkg)
                 if package_version is None:
                     return
-                subdir = docs_directory / pkg.pypi
-                subdir.mkdir(parents=True, exist_ok=True)
-                crawl_state_file = docs_directory / f"{pkg.pypi}.json"
-                if not force and crawl_state_file.exists():
-                    crawl_state = CrawlState.loads(crawl_state_file.read_text())
+                if (
+                    not force
+                    and (crawl_state := index.load_crawl_state(pkg.pypi)) is not None
+                ):
                     crawled_version = version.parse(crawl_state.package_version)
                     if package_version > crawled_version:
                         console.print(
@@ -72,12 +61,12 @@ def crawl_docs(
                 crawler = _Crawler(
                     pkg,
                     progress,
-                    docs_directory / pkg.pypi,
+                    index,
                     pkg.index_url,
                     crawl_state,
                 )
                 crawler.crawl(num_threads=num_threads_per_package)
-                crawler.save_crawl_state(package_version, crawl_state_file)
+                crawler.save_crawl_state(package_version, index)
             finally:
                 progress.advance(task)
 
@@ -92,13 +81,13 @@ class _Crawler:
         self,
         package: Package,
         progress: Progress,
-        docs_directory: Path,
+        index: Index,
         root_url: str,
-        crawl_state: CrawlState | None,
+        crawl_state: IndexState | None,
     ):
         self._package = package
         self._progress = progress
-        self._docs_directory = docs_directory
+        self._docs_directory = index.docs_directory / package.pypi
         self._root_url = root_url
         self._prefix = (
             root_url
@@ -162,18 +151,18 @@ class _Crawler:
                     ] = url
             self._crawl_state.failed_urls = failed_urls
 
-    def save_crawl_state(self, package_version: version.Version, output: Path) -> None:
+    def save_crawl_state(self, package_version: version.Version, index: Index) -> None:
         if (state := self._crawl_state) is None:
             file_to_urls = {
                 str(file.relative_to(self._docs_directory)): url
                 for url, file in self._crawled_url_to_files.items()
             }
-            state = CrawlState(
+            state = IndexState(
                 package_version=str(package_version),
                 file_to_urls=file_to_urls,
                 failed_urls=self._failed_urls,
             )
-        output.write_text(state.dumps())
+        index.save_crawl_state(self._package.pypi, state)
 
     def _crawl_thread(self, task: TaskID) -> None:
         while True:

pyrefdev-2025.8/src/pyrefdev/indexer/crawl_pypi.py

@@ -0,0 +1,27 @@
+import json
+from urllib.error import HTTPError
+
+from rich.progress import track
+
+from pyrefdev.config import console
+from pyrefdev.indexer.index import Index, urlopen
+
+
+def crawl_pypi(*, refresh: bool = True, index: Index = Index()) -> None:
+    """Crawl top 15000 PyPI packages."""
+    with urlopen(
+        "https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json"
+    ) as f:
+        top_packages_data = json.load(f)
+    rows = top_packages_data["rows"]
+    for row in track(
+        rows, description=f"Reading {len(rows)} packages", console=console
+    ):
+        package_name = row["project"]
+        try:
+            index.fetch_pypi_data(package_name, refresh=refresh)
+        except HTTPError as e:
+            if e.code == 404:
+                console.warning(f"Package {package_name} not found")
+                continue
+            raise

pyrefdev-2025.8/src/pyrefdev/indexer/index.py

@@ -0,0 +1,171 @@
+import dataclasses
+import json
+import re
+import random
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+from typing import Literal, overload
+from urllib import error, request
+
+from packaging import version
+from cyclopts import Parameter
+
+from pyrefdev import __version__
+from pyrefdev.config import Package, console
+
+
+_RTD_URL_PATTERN = re.compile(r"https?://([^\s/]+\.readthedocs\.io)\b")
+
+
+def urlopen(url: str):
+    req = request.Request(
+        url,
+        method="GET",
+        headers={"User-Agent": f"pyrefdev/{__version__} (+https://pyref.dev)"},
+    )
+    backoffs = [1, 2, 5, 15, 30, 60]
+    while True:
+        try:
+            return request.urlopen(req, timeout=60)
+        except error.HTTPError as e:
+            if e.code == 429:  # Too Many Request
+                if not backoffs:
+                    raise
+                backoff = backoffs.pop(0) * (0.9 + random.random() / 5.0)
+                time.sleep(backoff)
+            else:
+                raise
+
+
+@dataclasses.dataclass
+class IndexState:
+    package_version: str
+    file_to_urls: dict[str, str]
+    failed_urls: list[str]
+
+    @classmethod
+    def loads(cls, content: str) -> "IndexState":
+        return cls(**json.loads(content))
+
+    def dumps(self) -> str:
+        return json.dumps(dataclasses.asdict(self))
+
+
+def _get_default_api_docs_directory() -> Path:
+    cwd = Path(__file__).parent
+    git_root = subprocess.check_output(
+        ["git", "rev-parse", "--show-toplevel"], cwd=cwd, text=True
+    ).strip()
+    git_remote = subprocess.check_output(
+        ["git", "remote", "get-url", "origin"], cwd=git_root, text=True
+    ).strip()
+    if "/pyref.dev" in git_remote:
+        return Path(git_root) / "api-docs"
+    else:
+        directory = Path(tempfile.mkdtemp(prefix="pyref.dev."))
+        console.print(f"Using temporary directory for API docs: {directory}")
+        return directory
+
+
+@Parameter(name="*")
+@dataclasses.dataclass
+class Index:
+    docs_directory: Path = dataclasses.field(
+        default_factory=_get_default_api_docs_directory
+    )
+
+    def _ensure_directory(self) -> None:
+        self.docs_directory.mkdir(parents=True, exist_ok=True)
+        (self.docs_directory / "__pypi__").mkdir(parents=True, exist_ok=True)
+
+    def load_crawl_state(self, package: str) -> IndexState | None:
+        self._ensure_directory()
+        crawl_state_file = self.docs_directory / f"{package}.json"
+        if not crawl_state_file.exists():
+            return None
+        return IndexState.loads(crawl_state_file.read_text())
+
+    def save_crawl_state(self, package: str, crawl_state: IndexState) -> None:
+        self._ensure_directory()
+        crawl_state_file = self.docs_directory / f"{package}.json"
+        crawl_state_file.write_text(crawl_state.dumps())
+
+    def fetch_pypi_data(self, package: str, *, refresh: bool) -> bytes:
+        pypi_data_file = self.docs_directory / "__pypi__" / f"{package}.json"
+        if not refresh and pypi_data_file.exists():
+            return pypi_data_file.read_bytes()
+        with urlopen(f"https://pypi.org/pypi/{package}/json") as f:
+            data = f.read()
+        self._ensure_directory()
+        pypi_data_file.write_bytes(data)
+        return data
+
+    def get_pypi_packages(self) -> list[str]:
+        return list(f.stem for f in self.docs_directory.glob("__pypi__/*.json"))
+
+    def fetch_package_version(self, package: Package) -> version.Version | None:
+        if package.is_cpython():
+            return _fetch_latest_cpython_version()
+        try:
+            data = self.fetch_pypi_data(package.pypi, refresh=True)
+            pypi_info = json.loads(data)
+            return version.parse(pypi_info["info"]["version"])
+        except error.URLError as e:
+            console.warning(
+                f"Failed to fetch pypi version for {package.pypi}, error: {e}"
+            )
+            return None
+
+    @overload
+    def guess_index_url(
+        self, package: str, *, should_die_if_not_found: Literal[True]
+    ) -> str: ...
+    def guess_index_url(
+        self, package: str, *, should_die_if_not_found: bool
+    ) -> str | None:
+        data = self.fetch_pypi_data(package, refresh=False)
+        pypi_info = json.loads(data).get("info", {})
+        candidates = list((pypi_info.get("project_urls") or {}).values())
+        candidates.append(pypi_info.get("description", ""))
+
+        readthedocs_urls = set()
+
+        for url in candidates:
+            for match in _RTD_URL_PATTERN.findall(url):
+                readthedocs_urls.add(f"https://{match}")
+
+        if len(readthedocs_urls) == 1:
+            url = next(iter(readthedocs_urls))
+            try:
+                with urlopen(url) as f:
+                    url = f.url  # Maybe redirected URL.
+                return url
+            except error.URLError as e:
+                console.warning(f"Failed to fetch {url}, error: {e}")
+                readthedocs_urls = []
+
+        msg_fn = console.fatal if should_die_if_not_found else console.warning
+        if len(readthedocs_urls) == 0:
+            msg_fn(f"No readthedocs.io URLs found for package: {package}")
+        else:
+            msg_fn(
+                f"Multiple readthedocs.io URLs found for package: {package}. URLs:\n"
+                + "\n".join(readthedocs_urls)
+            )
+
+
+def _fetch_latest_cpython_version() -> version.Version | None:
+    try:
+        with urlopen("https://endoflife.date/api/python.json") as f:
+            content = f.read().decode("utf-8")
+        latest_version = version.parse("3.13.5")  # Known version as of 2025-06-28
+        cycles = json.loads(content)
+        for cycle in cycles:
+            if (latest := version.parse(cycle["latest"])) > latest_version:
+                latest_version = latest
+        return latest_version
+    except error.URLError as e:
+        console.warning(f"Failed to fetch latest CPython version, error: {e}")
+        return None

{pyrefdev-2025.6 → pyrefdev-2025.8}/src/pyrefdev/indexer/parse_docs.py

@@ -8,14 +8,14 @@ from pathlib import Path
 import re
 import sys
 import threading
-from urllib import parse
+from urllib.parse import urlparse, urlunparse
 
 import bs4
 from rich.progress import Progress
 
 from pyrefdev import mapping
 from pyrefdev.config import console, get_packages, Package
-from pyrefdev.indexer.schema import CrawlState
+from pyrefdev.indexer.index import Index
 
 
 _STDLIB_MODULES_NAMES = frozenset({*sys.stdlib_module_names, "test"})
@@ -65,11 +65,12 @@ class ProgressExecutor(futures.ThreadPoolExecutor):
 def parse_docs(
     *,
     package: str | None = None,
-    docs_directory: Path,
     in_place: bool = False,
+    index: Index = Index(),
     num_parallel_packages: int = multiprocessing.cpu_count(),
     num_threads_per_package: int = multiprocessing.cpu_count(),
 ) -> None:
+    """Parse crawled docs and update the mapping files."""
     if sys.version_info[:2] != (3, 13):
         console.fatal("pyrefdev-indexer parse_docs must be run on Python 3.13.")
 
@@ -88,7 +89,7 @@ def parse_docs(
                 _parse_package,
                 executor.progress,
                 pkg,
-                docs_directory / pkg.pypi,
+                index,
                 in_place=in_place,
                 num_threads_per_package=num_threads_per_package,
             )
@@ -101,13 +102,13 @@ def parse_docs(
 def _parse_package(
     progress: Progress,
     package: Package,
-    package_docs: Path,
+    index: Index,
     *,
     in_place: bool,
     num_threads_per_package: int,
 ) -> None:
-    crawl_state_file = package_docs.parent / f"{package.pypi}.json"
-    crawl_state = CrawlState.loads(crawl_state_file.read_text())
+    crawl_state = index.load_crawl_state(package.pypi)
+    assert crawl_state is not None, f"No crawl state for {package.pypi}"
     file_and_urls: list[tuple[str, str]] = list(crawl_state.file_to_urls.items())
     if package.is_cpython():
         symbol_to_urls: dict[str, str] = _SPECIAL_SYMBOLS.copy()
@@ -117,6 +118,7 @@ def _parse_package(
         }
     parser = _Parser(package)
     lock = threading.RLock()
+    package_docs = index.docs_directory / package.pypi
 
     def parse(file, url):
         if package.is_cpython:
@@ -133,15 +135,16 @@ def _parse_package(
         )
         for symbol, fragment in symbols.items():
             with lock:
-                if symbol not in symbol_to_urls:
-                    if (
-                        fragment.startswith(_MODULE_FRAGMENT_PREFIX)
-                        and module_count == 1
-                    ):
-                        # This page contains a single module, let's redirect to this page without the anchor.
-                        symbol_to_urls[symbol] = f"{url}"
-                    else:
-                        symbol_to_urls[symbol] = f"{url}#{fragment}"
+                if (existing_url := symbol_to_urls.get(symbol)) and (
+                    # More components take precedence.
+                    existing_url.count("/") >= url.count("/")
+                ):
+                    continue
+                if fragment.startswith(_MODULE_FRAGMENT_PREFIX) and module_count == 1:
+                    # This page contains a single module, let's redirect to this page without the anchor.
+                    symbol_to_urls[symbol] = f"{url}"
+                else:
+                    symbol_to_urls[symbol] = f"{url}#{fragment}"
 
     with ProgressExecutor(
         f"Parsing {len(file_and_urls)} files for {package_docs}",
@@ -156,7 +159,12 @@ def _parse_package(
 
     console.print(f"Found {len(symbol_to_urls)} symbols in {package.pypi}")
     _heuristically_fillin_modules(package, symbol_to_urls)
-    lines = _create_symbols_map(symbol_to_urls)
+
+    lines = [
+        f'VERSION = "{crawl_state.package_version}"',
+        "",
+    ]
+    lines.extend(_create_symbols_map(symbol_to_urls))
     mapping_file = Path(mapping.__file__).parent / f"{package.pypi}.py"
     if in_place:
         mapping_file.write_text("\n".join(itertools.chain(lines, [""])))
@@ -165,8 +173,13 @@ def _parse_package(
             before = mapping_file.read_text().splitlines(keepends=True)
         else:
             before = []
-        diffs = difflib.unified_diff(
-            before, [line + "\n" for line in lines], fromfile="before", tofile="after"
+        diffs = list(
+            difflib.unified_diff(
+                before,
+                [line + "\n" for line in lines],
+                fromfile="before",
+                tofile="after",
+            )
         )
         if diffs:
             console.print("".join(diffs))
@@ -175,6 +188,8 @@ def _parse_package(
 def _heuristically_fillin_modules(
     package: Package, symbol_to_urls: dict[str, str]
 ) -> None:
+    if package.is_cpython():
+        return
     extra_module_to_urls: dict[str, str] = {}
     for symbol in symbol_to_urls:
         if symbol in package.namespaces:
@@ -200,7 +215,7 @@ def _heuristically_fillin_modules(
 
 
 def _remove_fragment(url: str) -> str:
-    return parse.urlunparse(parse.urlparse(url)._replace(fragment=""))
+    return urlunparse(urlparse(url)._replace(fragment=""))
 
 
 def _create_symbols_map(symbol_to_urls: dict[str, str]) -> list[str]:
@@ -210,7 +225,6 @@ def _create_symbols_map(symbol_to_urls: dict[str, str]) -> list[str]:
 
     lines = [
         "# fmt: off",
-        "",
         "MAPPING = {",
     ]
     previous_symbol = None

pyrefdev-2025.8/src/pyrefdev/indexer/parse_pypi.py

@@ -0,0 +1,14 @@
+from pyrefdev.indexer.index import Index
+from pyrefdev.indexer.add_docs import update_config
+from pyrefdev.config import SUPPORTED_PACKAGES
+
+
+def parse_pypi(*, index: Index = Index()) -> None:
+    """Parse PyPI data and add new packages to the config."""
+    for pypi in index.get_pypi_packages():
+        if pypi in SUPPORTED_PACKAGES:
+            continue
+        url = index.guess_index_url(pypi, should_die_if_not_found=False)
+        if url is None:
+            continue
+        update_config(pypi, url, None)