pipseek 2.1.0__py3-none-any.whl

pipseek/__init__.py

@@ -0,0 +1,3 @@
+"""pipseek: search PyPI from your terminal."""
+
+__version__ = "2.1.0"

pipseek/__main__.py

@@ -0,0 +1,4 @@
+from pipseek.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pipseek/build_index.py

@@ -0,0 +1,240 @@
+"""Build the pipseek search index: every PyPI project with its summary and keywords.
+
+PyPI has no public full-text search API, so pipseek ships its own index. This script is run
+on a schedule by GitHub Actions (see .github/workflows/index.yml) and publishes the result as
+a release asset that clients download.
+
+It is incremental: the PyPI Simple API reports a "last serial" for every project, which
+changes whenever the project changes. Only projects whose serial moved since the previous
+build are re-fetched from the JSON API, so after the first build a refresh touches a few
+thousand projects instead of ~1M. A time budget lets the first build span several runs.
+
+Output format (gzipped TSV, one project per line):
+
+    #pipseek-index v1 <unix-time>
+    name <TAB> serial <TAB> summary <TAB> keywords
+
+Usage:
+    python -m pipseek.build_index --out pypi-index.tsv.gz [--previous old.tsv.gz]
+    python -m pipseek.build_index --out popular.tsv.gz --popular-only
+"""
+
+from __future__ import annotations
+
+import argparse
+import gzip
+import os
+import sys
+import threading
+import time
+from collections.abc import Iterable, Iterator
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Optional
+
+import requests
+
+from pipseek import __version__
+
+SIMPLE_URL = "https://pypi.org/simple/"
+SIMPLE_ACCEPT = "application/vnd.pypi.simple.v1+json"
+TOP_URL = (
+    "https://raw.githubusercontent.com/hugovk/top-pypi-packages/main/top-pypi-packages.min.json"
+)
+HEADER = "#pipseek-index v1"
+USER_AGENT = f"pipseek-index-builder/{__version__} (+https://github.com/ohmybahgosh/pipseek)"
+
+Record = tuple[str, int, str, str]  # name, serial, summary, keywords
+
+
+def clean(text: Optional[str], limit: int) -> str:
+    if not text:
+        return ""
+    text = " ".join(str(text).split())  # collapses tabs/newlines
+    if text.lower() in ("unknown", "none", "n/a", "null"):
+        return ""
+    return text[:limit]
+
+
+def read_index(path: str) -> Iterator[Record]:
+    """Read a pipseek index file (gzipped or plain TSV)."""
+    opener = gzip.open if path.endswith(".gz") else open
+    with opener(path, "rt", encoding="utf-8") as fh:
+        yield from parse_lines(fh)
+
+
+def parse_lines(lines: Iterable[str]) -> Iterator[Record]:
+    for line in lines:
+        if not line or line.startswith("#"):
+            continue
+        parts = line.rstrip("\n").split("\t")
+        if len(parts) < 2:
+            continue
+        name = parts[0]
+        try:
+            serial = int(parts[1])
+        except ValueError:
+            serial = 0
+        summary = parts[2] if len(parts) > 2 else ""
+        keywords = parts[3] if len(parts) > 3 else ""
+        yield name, serial, summary, keywords
+
+
+def write_index(path: str, records: Iterable[Record]) -> int:
+    count = 0
+    tmp = path + ".tmp"
+    with gzip.open(tmp, "wt", encoding="utf-8", compresslevel=9) as fh:
+        fh.write(f"{HEADER} {int(time.time())}\n")
+        for name, serial, summary, keywords in records:
+            fh.write(f"{name}\t{serial}\t{summary}\t{keywords}\n")
+            count += 1
+    os.replace(tmp, path)
+    return count
+
+
+def fetch_projects(session: requests.Session) -> dict[str, int]:
+    resp = session.get(SIMPLE_URL, headers={"Accept": SIMPLE_ACCEPT}, timeout=120)
+    resp.raise_for_status()
+    return {p["name"]: int(p.get("_last-serial") or 0) for p in resp.json()["projects"]}
+
+
+def fetch_popular(session: requests.Session) -> list[str]:
+    resp = session.get(TOP_URL, timeout=60)
+    resp.raise_for_status()
+    return [row["project"] for row in resp.json()["rows"]]
+
+
+def fetch_metadata(session: requests.Session, name: str) -> Optional[tuple[str, str]]:
+    """Return (summary, keywords), ("", "") for projects without releases, None on error."""
+    for attempt in range(4):
+        try:
+            resp = session.get(f"https://pypi.org/pypi/{name}/json", timeout=30)
+        except requests.RequestException:
+            time.sleep(2**attempt)
+            continue
+        if resp.status_code == 404:
+            return "", ""
+        if resp.status_code == 429 or resp.status_code >= 500:
+            time.sleep(2**attempt * 2)
+            continue
+        if resp.status_code != 200:
+            return None
+        try:
+            info = resp.json().get("info") or {}
+        except ValueError:
+            return None
+        keywords = info.get("keywords") or ""
+        if isinstance(keywords, list):
+            keywords = ", ".join(keywords)
+        return clean(info.get("summary"), 300), clean(keywords, 200)
+    return None
+
+
+def build(
+    out: str,
+    previous: Optional[str] = None,
+    popular_only: bool = False,
+    time_budget: float = 5 * 3600,
+    concurrency: int = 32,
+    limit: Optional[int] = None,
+) -> None:
+    started = time.time()
+    session = requests.Session()
+    session.headers["User-Agent"] = USER_AGENT
+    adapter = requests.adapters.HTTPAdapter(pool_connections=concurrency, pool_maxsize=concurrency)
+    session.mount("https://", adapter)
+
+    log("Fetching the PyPI project list...")
+    serials = fetch_projects(session)
+    log(f"{len(serials):,} projects on PyPI")
+
+    old: dict[str, Record] = {}
+    if previous:
+        try:
+            old = {r[0]: r for r in read_index(previous)}
+            log(f"Loaded {len(old):,} records from {previous}")
+        except OSError as exc:
+            log(f"Could not read previous index ({exc}); starting fresh")
+
+    if popular_only:
+        by_norm = {n.lower().replace("_", "-").replace(".", "-"): n for n in serials}
+        names = []
+        for p in fetch_popular(session):
+            real = by_norm.get(p.lower().replace("_", "-").replace(".", "-"))
+            if real:
+                names.append(real)
+    else:
+        names = list(serials)
+
+    todo = [n for n in names if n not in old or old[n][1] != serials[n] or old[n][1] == 0]
+    if limit:
+        todo = todo[:limit]
+    log(f"{len(todo):,} projects need fetching")
+
+    results: dict[str, Record] = {}
+    lock = threading.Lock()
+    done = 0
+    out_of_time = threading.Event()
+
+    def work(name: str) -> None:
+        nonlocal done
+        if out_of_time.is_set():
+            return
+        meta = fetch_metadata(session, name)
+        with lock:
+            if meta is not None:
+                results[name] = (name, serials[name], meta[0], meta[1])
+            done += 1
+            if done % 5000 == 0:
+                rate = done / max(time.time() - started, 1)
+                log(f"  {done:,}/{len(todo):,} fetched ({rate:.0f}/s)")
+        if time.time() - started > time_budget:
+            out_of_time.set()
+
+    with ThreadPoolExecutor(max_workers=concurrency) as pool:
+        for fut in as_completed([pool.submit(work, n) for n in todo]):
+            fut.result()
+
+    if out_of_time.is_set():
+        log("Time budget reached; remaining projects will be fetched on the next run.")
+
+    def merged() -> Iterator[Record]:
+        for name in names:
+            if name in results:
+                yield results[name]
+            elif name in old:
+                yield old[name]
+            else:
+                yield name, 0, "", ""  # serial 0 means "fetch me next time"
+
+    count = write_index(out, merged())
+    missing = sum(1 for n in names if n not in results and n not in old)
+    elapsed = time.time() - started
+    log(f"Wrote {count:,} records to {out} ({missing:,} pending) in {elapsed:.0f}s")
+
+
+def log(msg: str) -> None:
+    print(msg, file=sys.stderr, flush=True)
+
+
+def main(argv: Optional[list[str]] = None) -> int:
+    p = argparse.ArgumentParser(description="Build the pipseek search index.")
+    p.add_argument("--out", required=True, help="output .tsv.gz path")
+    p.add_argument("--previous", help="previous index to update incrementally")
+    p.add_argument("--popular-only", action="store_true", help="only the ~15k most popular")
+    p.add_argument("--time-budget", type=float, default=5 * 3600, help="seconds (default 5h)")
+    p.add_argument("--concurrency", type=int, default=32)
+    p.add_argument("--limit", type=int, help="fetch at most N projects (for testing)")
+    args = p.parse_args(argv)
+    build(
+        args.out,
+        previous=args.previous,
+        popular_only=args.popular_only,
+        time_budget=args.time_budget,
+        concurrency=args.concurrency,
+        limit=args.limit,
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pipseek/cache.py

@@ -0,0 +1,80 @@
+"""Tiny on-disk cache for the package index, popularity data, and API responses."""
+
+from __future__ import annotations
+
+import gzip
+import hashlib
+import json
+import os
+import sys
+import time
+from pathlib import Path
+from typing import Any, Optional
+
+
+def cache_dir() -> Path:
+    """Return the per-user cache directory, honoring PIPSEEK_CACHE_DIR and platform norms."""
+    override = os.environ.get("PIPSEEK_CACHE_DIR")
+    if override:
+        base = Path(override)
+    elif sys.platform == "win32":
+        base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) / "pipseek"
+    elif sys.platform == "darwin":
+        base = Path.home() / "Library" / "Caches" / "pipseek"
+    else:
+        base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "pipseek"
+    base.mkdir(parents=True, exist_ok=True)
+    return base
+
+
+class Cache:
+    """Gzipped JSON blobs keyed by name, each stored with a timestamp and optional ETag."""
+
+    def __init__(self, root: Optional[Path] = None) -> None:
+        self.root = root or cache_dir()
+        self.root.mkdir(parents=True, exist_ok=True)
+
+    def _path(self, key: str) -> Path:
+        safe = hashlib.sha1(key.encode()).hexdigest()[:16] if len(key) > 64 or "/" in key else key
+        return self.root / f"{safe}.json.gz"
+
+    def load(self, key: str) -> Optional[dict[str, Any]]:
+        path = self._path(key)
+        try:
+            with gzip.open(path, "rt", encoding="utf-8") as fh:
+                return json.load(fh)
+        except (OSError, ValueError):
+            return None
+
+    def get(self, key: str, max_age: float) -> Optional[Any]:
+        """Return cached data if younger than max_age seconds, else None."""
+        entry = self.load(key)
+        if entry and time.time() - entry.get("ts", 0) < max_age:
+            return entry.get("data")
+        return None
+
+    def set(self, key: str, data: Any, etag: Optional[str] = None) -> None:
+        path = self._path(key)
+        tmp = path.with_suffix(".tmp")
+        try:
+            with gzip.open(tmp, "wt", encoding="utf-8", compresslevel=5) as fh:
+                json.dump({"ts": time.time(), "etag": etag, "data": data}, fh)
+            os.replace(tmp, path)
+        except OSError:
+            tmp.unlink(missing_ok=True)
+
+    def touch(self, key: str) -> None:
+        """Mark an existing entry fresh (used after a 304 Not Modified)."""
+        entry = self.load(key)
+        if entry is not None:
+            self.set(key, entry.get("data"), entry.get("etag"))
+
+    def clear(self) -> int:
+        removed = 0
+        for path in self.root.glob("*.json.gz"):
+            try:
+                path.unlink()
+                removed += 1
+            except OSError:
+                pass
+        return removed

pipseek/cli.py

@@ -0,0 +1,217 @@
+"""Command-line entry point."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import textwrap
+from typing import Optional
+
+from rich.console import Console
+
+from pipseek import __version__
+from pipseek.cache import Cache
+from pipseek.env import PythonEnv
+from pipseek.index import IndexUnavailable, PackageIndex, load_downloads, normalize
+from pipseek.pypi import PyPIClient
+from pipseek.render import package_panel, results_table
+from pipseek.search import SORTS, Searcher
+
+EPILOG = textwrap.dedent(
+    """\
+    examples:
+      pipseek                       open the interactive search
+      pipseek fastapi               search and browse results
+      pipseek "pdf parser" -P       print a table instead of opening the TUI
+      pipseek http -s downloads -n 10 --json | jq '.[].name'
+      pipseek --info requests       full details for one package
+      pipseek --outdated            what's out of date or vulnerable in your environment
+      pipseek --outdated --strict   same, but exit 1 if anything installed is vulnerable (CI)
+
+    interactive keys:
+      /  search    enter  run search    up/down or j/k  move
+      n/p  next/previous page    s  cycle sort    c  copy install command
+      o  open on PyPI    h  open homepage/repo    i  pip install (or upgrade)    q  quit
+
+    environment:
+      PIPSEEK_CACHE_DIR         override the cache location
+    """
+)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="pipseek",
+        description="Search PyPI from your terminal.",
+        epilog=EPILOG,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p.add_argument("query", nargs="*", help="search terms")
+    p.add_argument("-i", "--info", metavar="PACKAGE", help="show full details for one package")
+    p.add_argument(
+        "-o",
+        "--outdated",
+        action="store_true",
+        help="list installed packages that are out of date or have known vulnerabilities",
+    )
+    p.add_argument(
+        "--all", action="store_true", help="with --outdated: list every installed package"
+    )
+    p.add_argument(
+        "--strict",
+        action="store_true",
+        help="with --outdated: exit 1 if an installed version has a known vulnerability",
+    )
+    p.add_argument("-P", "--plain", action="store_true", help="print results, no interactive UI")
+    p.add_argument("-j", "--json", action="store_true", help="print results as JSON")
+    p.add_argument("-n", "--limit", type=int, default=20, metavar="N", help="results per page")
+    p.add_argument("-p", "--page", type=int, default=1, metavar="N", help="page number")
+    p.add_argument("-s", "--sort", choices=SORTS, default="relevance", help="result ordering")
+    p.add_argument(
+        "--python", metavar="PATH", help="interpreter used for installed checks and installs"
+    )
+    p.add_argument(
+        "--no-stats",
+        "--no-github",
+        dest="no_stats",
+        action="store_true",
+        help="skip repository stats and vulnerability checks",
+    )
+    p.add_argument("--refresh", action="store_true", help="re-download the package index now")
+    p.add_argument("--clear-cache", action="store_true", help="delete cached data and exit")
+    p.add_argument("-V", "--version", action="version", version=f"pipseek {__version__}")
+    return p
+
+
+def main(argv: Optional[list[str]] = None) -> int:
+    args = build_parser().parse_args(argv)
+    err = Console(stderr=True)
+    out = Console()
+    cache = Cache()
+
+    if args.clear_cache:
+        removed = cache.clear()
+        err.print(f"Removed {removed} cached file(s) from {cache.root}")
+        return 0
+
+    if args.limit < 1 or args.page < 1:
+        err.print("[red]--limit and --page must be positive.[/]")
+        return 2
+
+    env = PythonEnv(args.python)
+    client = PyPIClient(cache=cache, stats=not args.no_stats, installed=env.version_of)
+    query = " ".join(args.query).strip()
+    interactive = (
+        not (args.plain or args.json or args.info or args.outdated) and sys.stdout.isatty()
+    )
+
+    def make_searcher(progress=None) -> Searcher:
+        index = PackageIndex.load(
+            session=client.session, cache=cache, refresh=args.refresh, progress=progress
+        )
+        return Searcher(index, client)
+
+    try:
+        if args.info:
+            return _info(args, client, cache, out, err)
+
+        if args.outdated:
+            return _outdated(args, env, cache, out, err)
+
+        if interactive:
+            from pipseek.tui import PipSeekApp
+
+            PipSeekApp(make_searcher, query, args.sort, args.limit, env).run()
+            return 0
+
+        if not query:
+            build_parser().print_help()
+            return 2
+
+        with err.status("Loading package index...") as status:
+            searcher = make_searcher(lambda msg: status.update(msg))
+            status.update(f"Searching {len(searcher.index):,} packages...")
+            page = searcher.search(query, page=args.page, per_page=args.limit, sort=args.sort)
+    except IndexUnavailable as exc:
+        err.print(f"[red]{exc}[/]")
+        return 1
+    except KeyboardInterrupt:
+        return 130
+
+    if args.json:
+        print(json.dumps([p.to_dict() for p in page.packages], indent=2))
+        return 0
+
+    if not page.packages:
+        err.print(f"No packages match [bold]{query}[/].")
+    else:
+        out.print(results_table(page.packages, show_stars=not args.no_stats))
+        footer = (
+            f"{page.start}-{page.end} of {page.total:,} matches  |  page {page.page}/{page.pages}"
+        )
+        if page.has_next:
+            footer += f"  |  next: --page {page.page + 1}"
+        err.print(f"[dim]{footer}[/]")
+    if page.suggestions:
+        err.print(f"[yellow]Did you mean:[/] {', '.join(page.suggestions)}")
+    return 0
+
+
+def _outdated(args, env: PythonEnv, cache: Cache, out: Console, err: Console) -> int:
+    from pipseek.outdated import check_environment, outdated_dict, outdated_table, upgrade_command
+
+    client = PyPIClient(cache=cache, stats=False, installed=env.version_of)
+    if not env.installed:
+        err.print(f"[red]Could not list the packages installed for {env.python}.[/]")
+        return 1
+    with err.status("Checking installed packages...") as status:
+        packages = check_environment(
+            env, client, include_all=args.all, progress=lambda m: status.update(m)
+        )
+    vulnerable = [p for p in packages if p.installed_vulns]
+    if args.json:
+        print(json.dumps([outdated_dict(p) for p in packages], indent=2))
+    else:
+        err.print(f"[dim]Environment: {env.python}[/]")
+        if not packages:
+            err.print(
+                f"[green]All {len(env.installed)} installed packages are up to date, "
+                "with no known vulnerabilities.[/]"
+            )
+        else:
+            out.print(outdated_table(packages))
+            outdated = sum(1 for p in packages if p.update_available)
+            err.print(
+                f"[dim]{outdated} outdated, {len(vulnerable)} with known vulnerabilities, "
+                f"out of {len(env.installed)} installed.  Vulnerability data: OSV.dev[/]"
+            )
+            if env.externally_managed:
+                err.print(
+                    "[yellow]This is a system Python managed by your OS, so pip shouldn't "
+                    "upgrade it directly.[/]  Update these through your package manager, "
+                    "or install what you need in a virtualenv or with pipx."
+                )
+            else:
+                cmd = upgrade_command(env, packages)
+                if cmd:
+                    label = "Upgrade" if outdated <= 12 else "Start with"
+                    err.print(f"[bold]{label}:[/] {cmd}")
+    return 1 if args.strict and vulnerable else 0
+
+
+def _info(args, client: PyPIClient, cache: Cache, out: Console, err: Console) -> int:
+    downloads = load_downloads(client.session, cache).get(normalize(args.info))
+    pkg = client.package(args.info, downloads)
+    if pkg is None:
+        err.print(f"[red]Package '{args.info}' was not found on PyPI.[/]")
+        return 1
+    if args.json:
+        print(json.dumps(pkg.to_dict(), indent=2))
+    else:
+        out.print(package_panel(pkg, full=True))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

pipseek/env.py

@@ -0,0 +1,116 @@
+"""Find the Python environment the user actually works in, and what's installed there.
+
+pipseek is usually installed with pipx or uv, so its own interpreter is an isolated venv.
+To show "installed" badges and to install packages, we target the user's environment:
+an explicit --python, else the active virtualenv/conda env, else the first python on PATH.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import Optional
+
+from pipseek.index import normalize
+
+_LIST_SCRIPT = (
+    "import json, importlib.metadata as m\n"
+    "out = {}\n"
+    "for d in m.distributions():\n"
+    "    n = d.metadata.get('Name')\n"
+    "    if n: out[n] = d.version\n"
+    "print(json.dumps(out))\n"
+)
+
+
+def _venv_python(root: str) -> Optional[str]:
+    base = Path(root)
+    for candidate in (base / "bin" / "python", base / "Scripts" / "python.exe"):
+        if candidate.exists():
+            return str(candidate)
+    return None
+
+
+def find_python(explicit: Optional[str] = None) -> str:
+    if explicit:
+        return shutil.which(explicit) or explicit
+    for var in ("VIRTUAL_ENV", "CONDA_PREFIX"):
+        root = os.environ.get(var)
+        if root:
+            found = _venv_python(root)
+            if found:
+                return found
+    for name in ("python3", "python"):
+        found = shutil.which(name)
+        if found:
+            return found
+    return sys.executable
+
+
+_MANAGED_SCRIPT = (
+    "import os, sys, sysconfig\n"
+    "venv = sys.prefix != getattr(sys, 'base_prefix', sys.prefix)\n"
+    "marker = os.path.join(sysconfig.get_paths()['stdlib'], 'EXTERNALLY-MANAGED')\n"
+    "print(int(not venv and os.path.exists(marker)))\n"
+)
+
+
+class PythonEnv:
+    def __init__(self, python: Optional[str] = None) -> None:
+        self.python = find_python(python)
+        self._installed: Optional[dict[str, str]] = None
+        self._managed: Optional[bool] = None
+
+    @property
+    def externally_managed(self) -> bool:
+        """True for an OS-managed Python (PEP 668), where pip shouldn't install packages."""
+        if self._managed is None:
+            try:
+                out = subprocess.run(
+                    [self.python, "-c", _MANAGED_SCRIPT],
+                    capture_output=True,
+                    text=True,
+                    timeout=10,
+                    check=True,
+                ).stdout.strip()
+                self._managed = out == "1"
+            except (OSError, subprocess.SubprocessError):
+                self._managed = False
+        return self._managed
+
+    @property
+    def installed(self) -> dict[str, str]:
+        """Normalized name -> version for every distribution in the target environment."""
+        if self._installed is None:
+            self._installed = {}
+            try:
+                out = subprocess.run(
+                    [self.python, "-c", _LIST_SCRIPT],
+                    capture_output=True,
+                    text=True,
+                    timeout=15,
+                    check=True,
+                ).stdout
+                self._installed = {normalize(k): v for k, v in json.loads(out).items()}
+            except (OSError, subprocess.SubprocessError, ValueError):
+                pass
+        return self._installed
+
+    def version_of(self, name: str) -> Optional[str]:
+        return self.installed.get(normalize(name))
+
+    def install_command(self, name: str, upgrade: bool = False) -> list[str]:
+        cmd = [self.python, "-m", "pip", "install"]
+        if upgrade:
+            cmd.append("--upgrade")
+        cmd.append(name)
+        return cmd
+
+    def install(self, name: str, upgrade: bool = False) -> int:
+        result = subprocess.run(self.install_command(name, upgrade))
+        self._installed = None
+        return result.returncode