pkgmap-py 0.1.0__py3-none-any.whl

pkgmap/__init__.py

@@ -0,0 +1,26 @@
+"""Cross-distribution Linux package name resolution library."""
+
+from pkgmap.db import ensure_database, get_database_path, update_database
+from pkgmap.resolver import (
+    PackageResolution,
+    list_repositories,
+    resolve,
+    resolve_all,
+    resolve_batch,
+    resolve_batch_all,
+    resolve_project,
+)
+
+__version__ = "0.1.0"
+__all__ = [
+    "PackageResolution",
+    "ensure_database",
+    "get_database_path",
+    "list_repositories",
+    "resolve",
+    "resolve_all",
+    "resolve_batch",
+    "resolve_batch_all",
+    "resolve_project",
+    "update_database",
+]

pkgmap/apps.py

@@ -0,0 +1,83 @@
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from pkgmap import pipeline
+from pkgmap.cli import cli_app
+from pkgmap.pipeline import download_dump_file, set_github_output
+
+pipeline_app = typer.Typer(
+    name="pkgmap-pipeline",
+    help="Download dump, transform, and build package map database",
+    add_completion=False,
+)
+
+
+@pipeline_app.command(name="download")
+def pipeline_download(
+    link: Annotated[
+        str | None,
+        typer.Option("--link", "-l", help="Dump link to download from"),
+    ] = None,
+    dest_dir: Annotated[
+        Path | None,
+        typer.Option("--dest-dir", "-d", help="Directory where to save the dump file"),
+    ] = None,
+) -> None:
+    """Download the latest Repology database dump."""
+    dump_file, dump_date = download_dump_file(dump_link=link, dest_dir=dest_dir)
+    set_github_output(
+        date=dump_date,
+        tag=f"db-{dump_date}",
+        dump_path=str(dump_file.resolve()),
+    )
+    typer.echo(f"Downloaded Repology dump ({dump_date}) to {dump_file}")
+
+
+@pipeline_app.command(name="build")
+def pipeline_build(
+    input_dump: Annotated[
+        Path | None,
+        typer.Option(
+            "--input-dump",
+            "-i",
+            help="Dump file to take as an input to the processing pipeline",
+        ),
+    ] = None,
+    output_file: Annotated[
+        Path,
+        typer.Option(
+            "-o",
+            "--output-path",
+            help="Path for the output SQLite database",
+        ),
+    ] = Path("pkgmap.db"),
+    batch_size: Annotated[
+        int,
+        typer.Option(
+            "--batch-size",
+            help="Number of records per batch in the processing pipeline",
+        ),
+    ] = 50_000,
+    release_notes: Annotated[
+        Path | None,
+        typer.Option("--release-notes", help="Path to generate release notes markdown"),
+    ] = None,
+    dump_date: Annotated[
+        str | None,
+        typer.Option("--dump-date", help="Snapshot date (YYYY-MM-DD)"),
+    ] = None,
+) -> None:
+    """Process Repology dump and generate SQLite package map database."""
+    pipeline.main(
+        dump_file=input_dump,
+        output_file=output_file,
+        batch_size=batch_size,
+        release_notes=release_notes,
+        dump_date=dump_date,
+    )
+
+
+if __name__ == "__main__":
+    cli_app()

pkgmap/cli.py

@@ -0,0 +1,171 @@
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from pkgmap.db import get_database_path, update_database
+from pkgmap.resolver import list_repositories, resolve_all, resolve_project
+
+cli_app = typer.Typer(
+    name="pkgmap",
+    help="Cross-distribution Linux package name resolution CLI",
+    add_completion=False,
+)
+
+
+@cli_app.callback()
+def cli_callback(
+    verbose: Annotated[
+        bool,
+        typer.Option("--verbose", "-v", help="Enable verbose debug logging"),
+    ] = False,
+) -> None:
+    if verbose:
+        import logging
+
+        for name in ("pkgmap", "pkgmap-db", "pkgmap-downloader", "pkgmap-consumer"):
+            log = logging.getLogger(name)
+            log.setLevel(logging.DEBUG)
+            for handler in log.handlers:
+                handler.setLevel(logging.DEBUG)
+
+
+@cli_app.command(name="resolve")
+def resolve_cmd(
+    package: Annotated[
+        str, typer.Argument(help="Package or upstream project name to resolve")
+    ],
+    to_repo: Annotated[
+        str | None,
+        typer.Option(
+            "--to",
+            "-t",
+            help="Target repository/distribution (e.g. nixpkgs, alpine). If omitted, matches across all repositories.",
+        ),
+    ] = None,
+    from_repo: Annotated[
+        str | None,
+        typer.Option(
+            "--from",
+            "-f",
+            help="Source repository/distribution (e.g. debian, arch). If omitted, package is resolved by upstream project name.",
+        ),
+    ] = None,
+    project: Annotated[
+        bool,
+        typer.Option(
+            "--project",
+            "-p",
+            help="Resolve package directly as an upstream project name",
+        ),
+    ] = False,
+    all_matches: Annotated[
+        bool,
+        typer.Option("--all", "-a", help="Show all matching target packages"),
+    ] = False,
+    json_output: Annotated[
+        bool,
+        typer.Option("--json", "-j", help="Output results in JSON format"),
+    ] = False,
+    db_path: Annotated[
+        Path | None,
+        typer.Option("--db-path", help="Custom path to pkgmap.db database"),
+    ] = None,
+) -> None:
+    """Resolve a package name across Linux distributions."""
+    if project and from_repo:
+        typer.secho(
+            "Error: Cannot specify both --from and --project.",
+            fg=typer.colors.RED,
+        )
+        raise typer.Exit(code=2)
+
+    if from_repo and not project:
+        results = resolve_all(
+            package, from_repo=from_repo, to_repo=to_repo, db_path=db_path
+        )
+    else:
+        results = resolve_project(package, to_repo=to_repo, db_path=db_path)
+
+    if not results:
+        if json_output:
+            typer.echo(json.dumps([]))
+        else:
+            if from_repo and not project:
+                if to_repo:
+                    msg = f"No equivalent package found for '{package}' from '{from_repo}' to '{to_repo}'."
+                else:
+                    msg = f"No equivalent package found for '{package}' from '{from_repo}'."
+            else:
+                if to_repo:
+                    msg = f"No equivalent package found for project '{package}' in '{to_repo}'."
+                else:
+                    msg = f"No equivalent package found for project '{package}'."
+            typer.secho(msg, fg=typer.colors.YELLOW)
+        raise typer.Exit(code=1)
+
+    items_to_show = results if (all_matches or to_repo is None) else results[:1]
+
+    if json_output:
+        data = [
+            {
+                "name": r.name,
+                "version": r.version,
+                "category": r.category,
+                "repo": r.repo,
+                "repo_family": r.repo_family,
+                "install_name": r.install_name,
+            }
+            for r in items_to_show
+        ]
+        typer.echo(json.dumps(data, indent=2))
+        return
+
+    for res in items_to_show:
+        version_str = f" ({res.version})" if res.version else ""
+        meta_parts = []
+        if res.category and res.repo_family != "nix":
+            meta_parts.append(f"category: {res.category}")
+        meta_str = f" [{', '.join(meta_parts)}]" if meta_parts else ""
+        repo_prefix = f"{res.repo}: " if to_repo is None else ""
+        typer.echo(f"{repo_prefix}{res.install_name}{version_str}{meta_str}")
+
+
+@cli_app.command(name="update")
+def update_cmd(
+    db_path: Annotated[
+        Path | None,
+        typer.Option("--db-path", help="Custom path to save pkgmap.db database"),
+    ] = None,
+) -> None:
+    """Explicitly download and update the local package map database."""
+    target_path = get_database_path(db_path)
+    typer.echo(f"Updating package database at {target_path}...")
+    update_database(db_path=target_path, show_progress=True)
+    typer.secho(
+        f"Successfully updated database at {target_path}", fg=typer.colors.GREEN
+    )
+
+
+@cli_app.command(name="repos")
+def repos_cmd(
+    db_path: Annotated[
+        Path | None,
+        typer.Option("--db-path", help="Custom path to pkgmap.db database"),
+    ] = None,
+) -> None:
+    """List all available distributions and repositories."""
+    repos = list_repositories(db_path=db_path)
+    typer.echo(f"{'Repository':<25} {'Family':<15} {'Packages':>10}")
+    typer.echo("-" * 52)
+    for r in sorted(repos, key=lambda x: x.name):
+        typer.echo(f"{r.name:<25} {r.family:<15} {r.package_count:>10,}")
+
+
+def main() -> None:
+    cli_app()
+
+
+if __name__ == "__main__":
+    main()

pkgmap/consumer.py

@@ -0,0 +1,145 @@
+import sqlite3
+from collections.abc import Iterable
+from pathlib import Path
+
+from pkgmap.logger import setup_logger
+from pkgmap.schema import PackageRecord, RepoFamilyMap, Repository
+
+logger = setup_logger("pkgmap-consumer")
+
+SCHEMA_SQL = """
+CREATE TABLE IF NOT EXISTS repositories (
+    id INTEGER PRIMARY KEY,
+    name TEXT UNIQUE NOT NULL,
+    family TEXT NOT NULL,
+    package_count INTEGER DEFAULT 0
+);
+CREATE TABLE IF NOT EXISTS repository_aliases (
+    alias TEXT PRIMARY KEY,
+    target_repo TEXT NOT NULL REFERENCES repositories(name)
+);
+CREATE TABLE IF NOT EXISTS packages (
+    repo_id INTEGER NOT NULL REFERENCES repositories(id),
+    package_name TEXT NOT NULL,
+    effname TEXT NOT NULL,
+    version TEXT,
+    category TEXT,
+    PRIMARY KEY (repo_id, package_name)
+) WITHOUT ROWID;
+"""
+
+INSERT_PACKAGES_QUERY = (
+    "INSERT OR IGNORE INTO packages (repo_id, package_name, effname, version, category) "
+    "VALUES (?, ?, ?, ?, ?)"
+)
+
+INSERT_REPOSITORIES_QUERY = "INSERT OR REPLACE INTO repositories (id, name, family, package_count) VALUES (?, ?, ?, ?)"
+
+SELECT_REPO_COUNTS_QUERY = "SELECT repo_id, COUNT(*) FROM packages GROUP BY repo_id"
+
+CREATE_INDEX_QUERY = "CREATE INDEX IF NOT EXISTS idx_effname ON packages (effname);"
+
+
+class SQLiteConsumer:
+    def __init__(self, output_file: Path, batch_size: int = 50_000) -> None:
+        self.output_file = output_file
+        self.batch_size = batch_size
+        self.batch_counter = 0
+        self.total_records = 0
+        self.repo_ids: dict[str, int] = {}
+
+    def init_db_schema(self, conn: sqlite3.Connection) -> None:
+        conn.executescript(SCHEMA_SQL)
+
+    def send_batch(self, conn: sqlite3.Connection, batch: list[PackageRecord]) -> None:
+        self.batch_counter += 1
+        self.total_records += len(batch)
+        logger.info(
+            f"Sending batch {self.batch_counter} (size={len(batch):,}, total={self.total_records:,})"
+        )
+        rows = [
+            (
+                self.repo_ids.setdefault(pkg.repo, len(self.repo_ids) + 1),
+                pkg.package_name,
+                pkg.effname,
+                pkg.version,
+                pkg.category,
+            )
+            for pkg in batch
+        ]
+        conn.executemany(INSERT_PACKAGES_QUERY, rows)
+        conn.commit()
+        batch.clear()
+        logger.info(
+            f"Batch {self.batch_counter} saved ({self.total_records:,} records committed so far)."
+        )
+
+    def populate_repositories(
+        self, conn: sqlite3.Connection, repo_families: RepoFamilyMap
+    ) -> None:
+        cursor = conn.cursor()
+        cursor.execute(SELECT_REPO_COUNTS_QUERY)
+        counts = dict(cursor.fetchall())
+        for repo_name in repo_families:
+            if repo_name not in self.repo_ids:
+                self.repo_ids[repo_name] = len(self.repo_ids) + 1
+
+        repositories = [
+            Repository(
+                id=repo_id,
+                name=repo_name,
+                family=repo_families.get(repo_name, "unknown"),
+                package_count=counts.get(repo_id, 0),
+            )
+            for repo_name, repo_id in sorted(
+                self.repo_ids.items(), key=lambda item: item[1]
+            )
+        ]
+        cursor.executemany(
+            INSERT_REPOSITORIES_QUERY,
+            [(r.id, r.name, r.family, r.package_count) for r in repositories],
+        )
+        conn.commit()
+
+    def __call__(
+        self, records: Iterable[PackageRecord], repo_families: RepoFamilyMap
+    ) -> None:
+        self.output_file.parent.mkdir(parents=True, exist_ok=True)
+        self.repo_ids = {}
+        conn = sqlite3.connect(self.output_file)
+
+        # WAL mode: Appends writes sequentially to a write-ahead log rather than altering the main DB
+        # file directly with rollback journals, providing high write throughput for bulk ingestion.
+        conn.execute("PRAGMA journal_mode = WAL;")
+
+        # NORMAL sync: Syncs WAL to disk at checkpoint intervals rather than blocking on every
+        # batch commit with hardware fsync, eliminating disk I/O bottlenecks.
+        conn.execute("PRAGMA synchronous = NORMAL;")
+
+        self.init_db_schema(conn)
+
+        batch: list[PackageRecord] = []
+        for pkg in records:
+            batch.append(pkg)
+            if len(batch) >= self.batch_size:
+                self.send_batch(conn, batch)
+
+        if batch:
+            self.send_batch(conn, batch)
+
+        self.populate_repositories(conn, repo_families)
+
+        conn.execute(CREATE_INDEX_QUERY)
+        conn.commit()
+
+        # Checkpoint WAL: Merges all pages from the write-ahead log back into the main database
+        # file and truncates the WAL file, leaving a clean, self-contained single .sqlite file.
+        conn.execute("PRAGMA wal_checkpoint(TRUNCATE);")
+
+        # VACUUM: Rebuilds the database file into a contiguous structure and reclaims unallocated pages.
+        conn.execute("VACUUM;")
+        conn.close()
+
+        logger.info(
+            f"Database generation complete: {self.total_records:,} packages saved to {self.output_file}"
+        )

pkgmap/db.py

@@ -0,0 +1,90 @@
+import os
+from pathlib import Path
+
+import zstandard as zstd
+
+from pkgmap.downloader import download_file
+from pkgmap.logger import setup_logger
+
+logger = setup_logger("pkgmap-db")
+
+DEFAULT_CACHE_DIR = Path.home() / ".cache" / "pkgmap"
+DEFAULT_DB_PATH = DEFAULT_CACHE_DIR / "pkgmap.db"
+GITHUB_RELEASE_LATEST_URL = (
+    "https://github.com/Levizor/pkgmap/releases/latest/download/pkgmap.db.zst"
+)
+
+
+def get_database_path(db_path: Path | str | None = None) -> Path:
+    """Resolve the path to the pkgmap.db database.
+
+    Precedence:
+    1. Explicit db_path argument
+    2. PKGMAP_DB_PATH environment variable
+    3. Default cache location: ~/.cache/pkgmap/pkgmap.db
+    """
+    if db_path is not None:
+        return Path(db_path).expanduser().resolve()
+    env_path = os.environ.get("PKGMAP_DB_PATH")
+    if env_path:
+        return Path(env_path).expanduser().resolve()
+    return DEFAULT_DB_PATH.expanduser().resolve()
+
+
+def decompress_zstd_file(src_path: Path, dest_path: Path) -> None:
+    """Stream decompress a .zst file to destination using chunked streaming."""
+    dest_path.parent.mkdir(parents=True, exist_ok=True)
+    dctx = zstd.ZstdDecompressor()
+    with open(src_path, "rb") as in_fh, open(dest_path, "wb") as out_fh:
+        dctx.copy_stream(in_fh, out_fh)
+
+
+def update_database(
+    db_path: Path | str | None = None,
+    url: str = GITHUB_RELEASE_LATEST_URL,
+    show_progress: bool = True,
+) -> Path:
+    """Download the latest database release from GitHub and decompress it locally."""
+    target_path = get_database_path(db_path)
+    target_path.parent.mkdir(parents=True, exist_ok=True)
+
+    compressed_tmp = target_path.with_name(f"{target_path.name}.zst.tmp")
+    decompressed_tmp = target_path.with_name(f"{target_path.name}.decompressed.tmp")
+
+    try:
+        logger.info(f"Downloading latest database release from {url}")
+        download_file(
+            url=url,
+            dest_path=compressed_tmp,
+            show_progress=show_progress,
+            label="Downloading package database",
+        )
+
+        logger.info("Decompressing package database...")
+        decompress_zstd_file(compressed_tmp, decompressed_tmp)
+
+        # Atomic replacement of final database file
+        decompressed_tmp.replace(target_path)
+        logger.info(f"Database successfully saved to {target_path}")
+        return target_path
+    finally:
+        if compressed_tmp.exists():
+            compressed_tmp.unlink(missing_ok=True)
+        if decompressed_tmp.exists():
+            decompressed_tmp.unlink(missing_ok=True)
+
+
+def ensure_database(
+    db_path: Path | str | None = None,
+    url: str = GITHUB_RELEASE_LATEST_URL,
+    show_progress: bool = True,
+) -> Path:
+    """Ensure the package database exists locally, downloading it lazily if missing."""
+    resolved_path = get_database_path(db_path)
+    if resolved_path.exists():
+        return resolved_path
+
+    logger.info(
+        f"Local package map database not found at {resolved_path}. Fetching release..."
+    )
+    return update_database(db_path=resolved_path, url=url, show_progress=show_progress)

pkgmap/downloader.py

@@ -0,0 +1,64 @@
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+import typer
+
+from pkgmap.logger import setup_logger
+
+logger = setup_logger("pkgmap-downloader")
+
+
+def download_file(
+    url: str,
+    dest_path: Path,
+    show_progress: bool = True,
+    label: str = "Downloading",
+    chunk_size: int = 1024 * 1024,
+) -> Path:
+    """Download a file with streaming chunks and optional Typer progress bar.
+
+    Writes to a temporary `.tmp` file and atomically renames on completion to
+    prevent corrupt or partial files.
+    """
+    dest_path.parent.mkdir(parents=True, exist_ok=True)
+    logger.info(f"Downloading from {url} to {dest_path}")
+
+    req = urllib.request.Request(
+        url,
+        headers={"User-Agent": "pkgmap/0.1.0 (https://github.com/levizor/pkgmap)"},
+    )
+
+    temp_path = dest_path.with_name(f"{dest_path.name}.tmp")
+    try:
+        with urllib.request.urlopen(req) as response:
+            content_length = getattr(response, "headers", {}).get("Content-Length")
+            total_size = (
+                int(content_length)
+                if isinstance(content_length, str) and content_length.isdigit()
+                else None
+            )
+
+            if show_progress and total_size is not None:
+                with typer.progressbar(
+                    length=total_size,
+                    label=label,
+                    show_eta=True,
+                    show_percent=True,
+                ) as progress:
+                    with open(temp_path, "wb") as out_file:
+                        while chunk := response.read(chunk_size):
+                            out_file.write(chunk)
+                            progress.update(len(chunk))
+            else:
+                with open(temp_path, "wb") as out_file:
+                    while chunk := response.read(chunk_size):
+                        out_file.write(chunk)
+
+        temp_path.replace(dest_path)
+    finally:
+        if temp_path.exists():
+            temp_path.unlink()
+
+    logger.info(f"Download complete: {dest_path}")
+    return dest_path

pkgmap/logger.py

@@ -0,0 +1,40 @@
+import logging
+import os
+import sys
+from pathlib import Path
+
+
+def setup_logger(
+    name: str = "pkgmap",
+    level: int = logging.WARNING,
+    file: bool = False,
+    log_file: Path | str | None = None,
+) -> logging.Logger:
+    """Configure and return a logger suitable for CLI or pipeline use."""
+    env_level = os.environ.get("PKGMAP_LOG_LEVEL")
+    effective_level = (
+        getattr(logging, env_level.upper(), logging.WARNING) if env_level else level
+    )
+
+    target_logger = logging.getLogger(name)
+    target_logger.setLevel(effective_level)
+    target_logger.handlers.clear()
+
+    formatter = logging.Formatter(
+        "%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+    console_handler = logging.StreamHandler(sys.stderr)
+    console_handler.setLevel(effective_level)
+    console_handler.setFormatter(formatter)
+    target_logger.addHandler(console_handler)
+
+    if file:
+        dest = log_file or f"{name}.log"
+        file_handler = logging.FileHandler(dest, mode="a", encoding="utf-8")
+        file_handler.setLevel(effective_level)
+        file_handler.setFormatter(formatter)
+        target_logger.addHandler(file_handler)
+
+    return target_logger

pkgmap/pipeline.py

@@ -0,0 +1,253 @@
+import io
+import logging
+import os
+import re
+import urllib.parse
+import urllib.request
+from collections.abc import Callable, Iterable, Iterator
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+from pathlib import Path
+
+import zstandard as zstd
+
+from pkgmap.consumer import SQLiteConsumer
+from pkgmap.downloader import download_file
+from pkgmap.logger import setup_logger
+from pkgmap.schema import (
+    PackageRecord,
+    RepoFamilyMap,
+    RepologyPackageRow,
+)
+
+REPOLOGY_LATEST_DUMP_URL = (
+    "https://dumps.repology.org/repology-database-dump-latest.sql.zst"
+)
+
+logger = setup_logger("pkgmap-pipeline", logging.DEBUG, file=True)
+
+OutputConsumer = Callable[[Iterable[PackageRecord], RepoFamilyMap], None]
+
+
+def fetch_latest_database_dump_link(
+    url: str = REPOLOGY_LATEST_DUMP_URL,
+) -> tuple[str, str]:
+    """Inspect latest dump URL and return (url, date) based on HTTP Last-Modified header."""
+    req = urllib.request.Request(
+        url,
+        method="HEAD",
+        headers={"User-Agent": "pkgmap/0.1.0 (https://github.com/Levizor/pkgmap)"},
+    )
+    with urllib.request.urlopen(req) as resp:
+        last_modified = resp.headers.get("Last-Modified")
+        if last_modified:
+            dump_date = parsedate_to_datetime(last_modified).strftime("%Y-%m-%d")
+        else:
+            dump_date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+
+    logger.info(f"Targeting latest dump from {dump_date}: {url}")
+    return url, dump_date
+
+
+def download_dump_file(
+    dump_link: str | None = None,
+    dest_dir: Path | None = None,
+    show_progress: bool = True,
+) -> tuple[Path, str]:
+    """Download the Repology dump file and return (dump_path, dump_date)."""
+    if not dump_link:
+        dump_link, dump_date = fetch_latest_database_dump_link()
+    else:
+        match = re.search(r"(\d{4}-\d{2}-\d{2})", dump_link)
+        dump_date = (
+            match.group(1) if match else datetime.now(timezone.utc).strftime("%Y-%m-%d")
+        )
+
+    target_dir = dest_dir or Path.cwd()
+    parsed_path = urllib.parse.urlparse(dump_link).path
+    filename = Path(urllib.parse.unquote(parsed_path)).name or "repology_dump.sql.zst"
+    dest_path = target_dir / filename
+
+    dump_path = download_file(
+        url=dump_link,
+        dest_path=dest_path,
+        show_progress=show_progress,
+        label="Downloading Repology dump",
+    )
+    return dump_path, dump_date
+
+
+def stream_dump_lines(dump: Path) -> Iterator[str]:
+    """Stream lines from a dump file (.sql.zst or plain .sql)."""
+    if str(dump).endswith(".zst"):
+        dctx = zstd.ZstdDecompressor()
+        with open(dump, "rb") as fh:
+            with dctx.stream_reader(fh) as reader:
+                text_stream = io.TextIOWrapper(
+                    reader, encoding="utf-8", errors="replace"
+                )
+                for line in text_stream:
+                    yield line
+    else:
+        with open(dump, "r", encoding="utf-8", errors="replace") as fh:
+            for line in fh:
+                yield line
+
+
+def parse_repology_records(lines: Iterable[str]) -> Iterator[RepologyPackageRow]:
+    """Parse tab-separated package rows from PostgreSQL COPY stream."""
+    in_packages = False
+    columns: list[str] = []
+
+    for line in lines:
+        if line.startswith("COPY "):
+            table_part = line[5 : line.find("(")].strip()
+            table_name = table_part.split(".")[-1]
+            if table_name == "packages":
+                in_packages = True
+                col_str = line[line.find("(") + 1 : line.rfind(")")]
+                columns = [c.strip().strip('"') for c in col_str.split(",")]
+                continue
+
+        if in_packages:
+            if line.startswith(r"\.") or line.startswith("\\."):
+                in_packages = False
+                continue
+
+            parts = line.rstrip("\r\n").split("\t")
+            row = dict(zip(columns, parts, strict=False))
+
+            repo = row.get("repo", "")
+            visiblename = row.get("visiblename", "")
+            effname = row.get("effname", "")
+            if not repo or not visiblename or not effname:
+                continue
+
+            yield RepologyPackageRow(
+                repo=repo,
+                family=row.get("family", ""),
+                visiblename=visiblename,
+                effname=effname,
+                srcname=None
+                if row.get("srcname") in (None, r"\N")
+                else row.get("srcname"),
+                version=None
+                if row.get("version") in (None, r"\N")
+                else row.get("version"),
+                category=None
+                if row.get("category") in (None, r"\N")
+                else row.get("category"),
+                subrepo=None
+                if row.get("subrepo") in (None, r"\N")
+                else row.get("subrepo"),
+                origversion=None
+                if row.get("origversion") in (None, r"\N")
+                else row.get("origversion"),
+                flags=int(row["flags"]) if row.get("flags", "").isdigit() else 0,
+                shadow=row.get("shadow") in ("t", "true", "1"),
+            )
+
+
+def do_fn(
+    records: Iterable[RepologyPackageRow], repo_families: RepoFamilyMap
+) -> Iterator[PackageRecord]:
+    for row in records:
+        if row.shadow:
+            continue
+        repo_families[row.repo] = row.family
+        yield row.to_package_record()
+
+
+def run_pipeline(dump: Path, consumer: OutputConsumer) -> None:
+    lines = stream_dump_lines(dump)
+    raw_records = parse_repology_records(lines)
+    repo_families: RepoFamilyMap = {}
+    packages = do_fn(raw_records, repo_families)
+    consumer(packages, repo_families)
+
+
+def set_github_output(**kwargs: str) -> None:
+    """Append key-value pairs to GITHUB_OUTPUT environment file if present."""
+    gh_output = os.environ.get("GITHUB_OUTPUT")
+    if not gh_output:
+        return
+    with open(gh_output, "a", encoding="utf-8") as f:
+        for key, value in kwargs.items():
+            f.write(f"{key}={value}\n")
+
+
+def generate_release_notes(
+    dump_date: str,
+    total_packages: int,
+    total_repos: int,
+    output_path: Path,
+) -> str:
+    """Generate compact markdown release notes for GitHub Releases."""
+    content = (
+        f"> **{dump_date} snapshot**: {total_packages:,} packages · {total_repos} repositories"
+        f" · Sourced from [Repology](https://repology.org) ([CC-BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/))\n"
+    )
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(content, encoding="utf-8")
+    logger.info(f"Generated release notes at {output_path}")
+    return content
+
+
+def main(
+    dump_file: Path | None = None,
+    dump_link: str | None = None,
+    output_file: Path = Path("pkgmap.db"),
+    batch_size: int = 50_000,
+    release_notes: Path | None = None,
+    dump_date: str | None = None,
+) -> SQLiteConsumer:
+    logger.debug(
+        f"input arguments: {
+            dict(
+                {
+                    'dump_file': dump_file,
+                    'dump_link': dump_link,
+                    'output_file': output_file,
+                }
+            )
+        }"
+    )
+
+    if not dump_file:
+        dump_file, detected_date = download_dump_file(dump_link=dump_link)
+        if not dump_date:
+            dump_date = detected_date
+    elif not dump_date:
+        match = re.search(r"(\d{4}-\d{2}-\d{2})", dump_file.name)
+        dump_date = (
+            match.group(1) if match else datetime.now(timezone.utc).strftime("%Y-%m-%d")
+        )
+
+    if not dump_file.exists():
+        raise FileNotFoundError(f"Dump file does not exist: {dump_file}")
+
+    logger.info("Starting the pipeline!")
+    consumer = SQLiteConsumer(output_file, batch_size)
+    run_pipeline(dump_file, consumer)
+    logger.info("Finished!")
+
+    if release_notes:
+        generate_release_notes(
+            dump_date=dump_date,
+            total_packages=consumer.total_records,
+            total_repos=len(consumer.repo_ids),
+            output_path=release_notes,
+        )
+
+    set_github_output(
+        packages=str(consumer.total_records),
+        repos=str(len(consumer.repo_ids)),
+        date=dump_date,
+        tag=f"db-{dump_date}",
+    )
+
+    return consumer
+
+
+if __name__ == "__main__":
+    main()

pkgmap/resolver.py

@@ -0,0 +1,203 @@
+import sqlite3
+from collections.abc import Sequence
+from pathlib import Path
+
+from pkgmap.db import ensure_database
+from pkgmap.schema import PackageResolution, Repository
+
+DEFAULT_ALIASES: dict[str, str] = {
+    "debian": "debian_12",
+    "debian_stable": "debian_12",
+    "ubuntu": "ubuntu_24_04",
+    "ubuntu_lts": "ubuntu_24_04",
+    "nix": "nix_unstable",
+    "nixpkgs": "nix_unstable",
+    "alpine": "alpine_3_20",
+    "fedora": "fedora_rawhide",
+    "arch": "arch",
+    "archlinux": "arch",
+    "void": "void_packages",
+    "freebsd": "freebsd",
+    "gentoo": "gentoo",
+    "macports": "macports",
+    "opensuse": "opensuse_tumbleweed",
+}
+
+
+def resolve_repo_name(conn: sqlite3.Connection, repo_input: str) -> str:
+    """Resolve a user-supplied repository alias or name to its canonical slug."""
+    normalized = repo_input.strip().lower().replace("-", "_")
+    target = DEFAULT_ALIASES.get(normalized, normalized)
+
+    cursor = conn.cursor()
+
+    # 1. Exact match in repositories
+    cursor.execute("SELECT name FROM repositories WHERE name = ?", (target,))
+    row = cursor.fetchone()
+    if row:
+        return row[0]
+
+    # 2. Check repository_aliases table
+    cursor.execute(
+        "SELECT target_repo FROM repository_aliases WHERE alias = ?", (target,)
+    )
+    row = cursor.fetchone()
+    if row:
+        return row[0]
+
+    # 3. Prefix matching (e.g. 'debian' -> highest package_count debian_* repo)
+    cursor.execute(
+        "SELECT name FROM repositories WHERE name LIKE ? ORDER BY package_count DESC LIMIT 1",
+        (f"{target}%",),
+    )
+    row = cursor.fetchone()
+    if row:
+        return row[0]
+
+    # 4. Family matching (e.g. 'debuntu' or family name)
+    cursor.execute(
+        "SELECT name FROM repositories WHERE family = ? ORDER BY package_count DESC LIMIT 1",
+        (target,),
+    )
+    row = cursor.fetchone()
+    if row:
+        return row[0]
+
+    return repo_input
+
+
+BATCH_CHUNK_SIZE = 500
+
+
+def resolve_batch_all(
+    packages: Sequence[str],
+    from_repo: str | None = None,
+    to_repo: str | None = None,
+    db_path: Path | str | None = None,
+) -> dict[str, list[PackageResolution]]:
+    """Resolve a batch of packages or projects, returning all matches for each package."""
+    if not packages:
+        return {}
+
+    results: dict[str, list[PackageResolution]] = {pkg: [] for pkg in packages}
+    unique_pkgs = list(dict.fromkeys(packages))
+    conn = sqlite3.connect(ensure_database(db_path))
+
+    try:
+        cursor = conn.cursor()
+        src_repo = resolve_repo_name(conn, from_repo) if from_repo else None
+        dst_repo = resolve_repo_name(conn, to_repo) if to_repo else None
+
+        for i in range(0, len(unique_pkgs), BATCH_CHUNK_SIZE):
+            chunk = unique_pkgs[i : i + BATCH_CHUNK_SIZE]
+            placeholders = ",".join("?" for _ in chunk)
+
+            if src_repo:
+                sql = f"""
+                SELECT src.package_name, dst.package_name, dst.version, dst.category, dst_repo.name, dst_repo.family
+                FROM packages src
+                JOIN repositories src_repo ON src.repo_id = src_repo.id
+                JOIN packages dst ON src.effname = dst.effname
+                JOIN repositories dst_repo ON dst.repo_id = dst_repo.id
+                WHERE src_repo.name = ? AND src.package_name IN ({placeholders})
+                """
+                params: list[str] = [src_repo, *chunk]
+            else:
+                sql = f"""
+                SELECT dst.effname, dst.package_name, dst.version, dst.category, dst_repo.name, dst_repo.family
+                FROM packages dst
+                JOIN repositories dst_repo ON dst.repo_id = dst_repo.id
+                WHERE dst.effname IN ({placeholders})
+                """
+                params = list(chunk)
+
+            if dst_repo:
+                sql += " AND dst_repo.name = ?"
+                params.append(dst_repo)
+
+            sql += " ORDER BY 1, dst_repo.name, dst.package_name;"
+
+            cursor.execute(sql, params)
+            for row in cursor.fetchall():
+                key = row[0]
+                if key in results:
+                    results[key].append(
+                        PackageResolution(
+                            name=row[1],
+                            version=row[2],
+                            category=row[3],
+                            repo=row[4],
+                            repo_family=row[5],
+                        )
+                    )
+        return results
+    finally:
+        conn.close()
+
+
+def resolve_batch(
+    packages: Sequence[str],
+    from_repo: str | None = None,
+    to_repo: str | None = None,
+    db_path: Path | str | None = None,
+) -> dict[str, PackageResolution | None]:
+    """Resolve a batch of packages or projects, returning the first match for each package."""
+    all_res = resolve_batch_all(
+        packages, from_repo=from_repo, to_repo=to_repo, db_path=db_path
+    )
+    return {pkg: matches[0] if matches else None for pkg, matches in all_res.items()}
+
+
+def resolve_all(
+    package: str,
+    from_repo: str | None = None,
+    to_repo: str | None = None,
+    db_path: Path | str | None = None,
+) -> list[PackageResolution]:
+    """Resolve all equivalent packages across target repository or all repositories."""
+    return resolve_batch_all(
+        [package], from_repo=from_repo, to_repo=to_repo, db_path=db_path
+    ).get(package, [])
+
+
+def resolve(
+    package: str,
+    from_repo: str | None = None,
+    to_repo: str | None = None,
+    db_path: Path | str | None = None,
+) -> PackageResolution | None:
+    """Resolve a package or project to its primary equivalent."""
+    return resolve_batch(
+        [package], from_repo=from_repo, to_repo=to_repo, db_path=db_path
+    ).get(package)
+
+
+def resolve_project(
+    effname: str,
+    to_repo: str | None = None,
+    db_path: Path | str | None = None,
+) -> list[PackageResolution]:
+    """Resolve packages matching the canonical upstream project name."""
+    return resolve_all(effname, from_repo=None, to_repo=to_repo, db_path=db_path)
+
+
+def list_repositories(db_path: Path | str | None = None) -> list[Repository]:
+    """Return all repositories registered in the package database."""
+    resolved_db = ensure_database(db_path)
+    conn = sqlite3.connect(resolved_db)
+    try:
+        cursor = conn.cursor()
+        cursor.execute(
+            "SELECT name, family, package_count, id FROM repositories ORDER BY name"
+        )
+        return [
+            Repository(
+                name=row[0],
+                family=row[1],
+                package_count=row[2],
+                id=row[3],
+            )
+            for row in cursor.fetchall()
+        ]
+    finally:
+        conn.close()

pkgmap/schema.py

@@ -0,0 +1,78 @@
+from typing import NamedTuple
+
+
+class Repository(NamedTuple):
+    name: str
+    family: str
+    package_count: int = 0
+    id: int | None = None
+
+
+class RepositoryAlias(NamedTuple):
+    alias: str
+    target_repo: str
+
+
+type RepoFamilyMap = dict[str, str]
+
+
+class PackageRecord(NamedTuple):
+    repo: str
+    package_name: str  # native package name
+    effname: str  # unique project name, same for all packages
+    version: str | None = None
+    category: str | None = None
+
+
+class PackageResolution(NamedTuple):
+    name: str
+    version: str | None = None
+    category: str | None = None
+    repo: str = ""
+    repo_family: str = ""
+
+    @property
+    def install_name(self) -> str:
+        """The canonical package identifier for package managers or declarative configs."""
+        return self.name
+
+
+# Upstream Repology PostgreSQL schema definition:
+# https://github.com/repology/repology-updater/blob/master/sql.d/schema/create_schema_tables.sql
+REPOLOGY_SCHEMA_URL = "https://github.com/repology/repology-updater/blob/master/sql.d/schema/create_schema_tables.sql"
+
+
+class RepologyPackageRow(NamedTuple):
+    """Repology 'packages' table row schema extracted from PostgreSQL dumps.
+
+    Reference:
+    https://github.com/repology/repology-updater/blob/master/sql.d/schema/create_schema_tables.sql#L31-L75
+    """
+
+    repo: str
+    family: str
+    visiblename: str
+    effname: str  # Canonical upstream project name
+    srcname: str | None = None
+    version: str | None = None
+    category: str | None = None
+    subrepo: str | None = None
+    origversion: str | None = None
+    flags: int = 0
+    shadow: bool = False
+
+    def to_package_record(self) -> "PackageRecord":
+        package_name = self.visiblename
+        if self.family == "nix":
+            if self.srcname:
+                package_name = self.srcname
+            elif self.category:
+                package_name = f"{self.category}.{self.visiblename}"
+
+        return PackageRecord(
+            repo=self.repo,
+            package_name=package_name,
+            effname=self.effname,
+            version=self.version,
+            category=self.category,
+        )

pkgmap_py-0.1.0.dist-info/METADATA

@@ -0,0 +1,155 @@
+Metadata-Version: 2.3
+Name: pkgmap-py
+Version: 0.1.0
+Summary: Cross-distribution Linux package name resolution
+Author: Levizor
+Author-email: Levizor <levizor@disroot.org>
+License: MIT
+Requires-Dist: typer>=0.12.0
+Requires-Dist: zstandard>=0.22.0
+Requires-Python: >=3.12
+Project-URL: Homepage, https://github.com/Levizor/pkgmap
+Project-URL: Repository, https://github.com/Levizor/pkgmap
+Project-URL: Issues, https://github.com/Levizor/pkgmap/issues
+Description-Content-Type: text/markdown
+
+# pkgmap
+
+Cross-distribution Linux package name resolution.
+
+Different distributions often package the exact same software under different names (for example, `ninja` on Arch is `ninja-build` on Debian and Alpine; `bat` on Arch is `rust-bat` on Debian and Fedora; `fd` on Arch is `rust-fd-find` on Debian; Nixpkgs uses specific attribute paths). `pkgmap` provides an offline SQLite database and a companion Python library/CLI to translate package names between any pair of Linux distributions, resolve packages across all repositories, or look up native packages by upstream project name.
+
+Data is updated daily from [Repology](https://repology.org) and indexed for instant local lookups. On first run, `pkgmap` automatically downloads and decompresses the latest daily database release (`~/.cache/pkgmap/pkgmap.db`).
+
+---
+
+## CLI Usage
+
+### Translate between distributions
+
+Use `--from` (`-f`) and `--to` (`-t`):
+
+```bash
+# What is Arch's 'bat' called in Debian?
+pkgmap resolve bat --from arch --to debian
+# Output: rust-bat (0.22.1) [category: misc]
+
+# What is Arch's 'fd' in Nixpkgs?
+pkgmap resolve fd -f arch -t nixpkgs
+# Output: fd (10.5.0)
+```
+
+### Resolve across all distributions
+
+Omit `--to` to see equivalents across every indexed distribution:
+
+```bash
+pkgmap resolve bat --from arch
+```
+
+Output:
+```text
+alpine_3_20: bat (0.24.0)
+debian_12: rust-bat (0.22.1) [category: misc]
+fedora_rawhide: rust-bat (0.26.1)
+nix_unstable: bat (0.26.1)
+ubuntu_24_04: rust-bat (0.24.0)
+...
+```
+
+### Look up by upstream project name
+
+Omit `--from` (or pass `--project` / `-p`) to resolve packages directly from an upstream project name:
+
+```bash
+pkgmap resolve ripgrep --to debian
+# Output: rust-ripgrep (13.0.0) [category: misc]
+```
+
+### JSON output
+
+To output machine-readable JSON (great for automation and scripts), add `--json` / `-j`:
+
+```bash
+pkgmap resolve bat -f arch -t debian --json
+```
+
+```json
+[
+  {
+    "name": "rust-bat",
+    "version": "0.22.1",
+    "category": "misc",
+    "repo": "debian_12",
+    "repo_family": "debuntu",
+    "install_name": "rust-bat"
+  }
+]
+```
+
+### Other commands
+
+```bash
+# List available distributions and repository slugs
+pkgmap repos
+
+# Manually refresh the local database snapshot
+pkgmap update
+```
+
+---
+
+## Python API
+
+You can also use `pkgmap` as a Python library:
+
+```python
+from pkgmap import resolve, resolve_all, resolve_batch
+
+# 1. Single package resolution
+res = resolve("bat", from_repo="arch", to_repo="debian")
+if res:
+    print(res.install_name, res.version)
+    # Output: rust-bat 0.22.1
+
+# 2. Query all repositories for equivalents
+matches = resolve_all("bat", from_repo="arch")
+for r in matches:
+    print(f"{r.repo}: {r.install_name}")
+
+# 3. High-performance batch resolution (executed in a single SQL query)
+packages = ["bat", "fd", "ripgrep"]
+results = resolve_batch(packages, from_repo="arch", to_repo="nixpkgs")
+
+for pkg, match in results.items():
+    if match:
+        print(f"{pkg} -> {match.install_name}")
+```
+
+### API Methods
+
+- `resolve(package, from_repo=None, to_repo=None, db_path=None)`: Resolves a package to a single match.
+- `resolve_all(package, from_repo=None, to_repo=None, db_path=None)`: Returns all matching packages.
+- `resolve_batch(packages, from_repo=None, to_repo=None, db_path=None)`: Resolves a sequence of packages in a single batch query, returning `{package: PackageResolution | None}`.
+- `resolve_batch_all(packages, from_repo=None, to_repo=None, db_path=None)`: Batch resolution returning `{package: list[PackageResolution]}`.
+- `list_repositories(db_path=None)`: Returns all repositories with package counts and families.
+
+---
+
+## Configuration
+
+By default, the database is stored at:
+- `~/.cache/pkgmap/pkgmap.db`
+
+You can customize the location using:
+- The `PKGMAP_DB_PATH` environment variable:
+  ```bash
+  export PKGMAP_DB_PATH="/path/to/custom/pkgmap.db"
+  ```
+- Or the `--db-path` argument on the CLI / `db_path` parameter in Python.
+
+---
+
+## Attribution
+
+Data is sourced daily from [Repology](https://repology.org). Repology's data is published under the [CC-BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/) license.

pkgmap_py-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+pkgmap/__init__.py,sha256=BEokLUaLyHjBrnk6N1VDsx3FrR62cvYKZoyIQLBDYgw,571
+pkgmap/apps.py,sha256=CetY2UbJ-VcTCf8gok6wL8q0eSzEww1vSc08GuxCrDw,2307
+pkgmap/cli.py,sha256=Hl_TaieaKPK7C0xJIIk4gLz2cLpQPAl45R0xeFgcBAY,5421
+pkgmap/consumer.py,sha256=rrdQlTGHNfUnsEa9NjIrXI16pyhsnmCqfjzVdSsDnmM,5065
+pkgmap/db.py,sha256=IgUFFd0eGoD8LE_FwpQ-Eyl9k7hNhxxrX_bnhbJaT9k,3084
+pkgmap/downloader.py,sha256=_burjvCvrNtQAiK2BghRi8VMjDlDUuthFxd0f_Wgiag,2053
+pkgmap/logger.py,sha256=wmBZIXOSZqfvKhGebEqrS-l6a3aFo-ywI_AAMnLdmgs,1220
+pkgmap/pipeline.py,sha256=7NXwTG-zlRhRQgzQqqpZhrKCtGUYIQHWuatxtBuOPE0,8297
+pkgmap/resolver.py,sha256=nCfAReUow4BToshwzWAOFni8fqYOJ3ofaGX4Rb7QbV0,6674
+pkgmap/schema.py,sha256=XnMP72QMeG6fHAR_8cNpf_Eos2QefKRdWNBgoARnZK0,2140
+pkgmap_py-0.1.0.dist-info/WHEEL,sha256=xTCBFIZIIyRSxyowOv5Wqg_jFU6KsdIffuulBB09Ph8,80
+pkgmap_py-0.1.0.dist-info/entry_points.txt,sha256=cXriBEVusq58y9gFDN6AbNuVj0LwEWAU9Iwg6zfuo-Y,91
+pkgmap_py-0.1.0.dist-info/METADATA,sha256=4j4ufQWXUEguoqWltamONZ6fh_xuY0kXe_heg6KGnRU,4614
+pkgmap_py-0.1.0.dist-info/RECORD,,

pkgmap_py-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.13.0
+Root-Is-Purelib: true
+Tag: py3-none-any

pkgmap_py-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,4 @@
+[console_scripts]
+pkgmap = pkgmap.apps:cli_app
+pkgmap-pipeline = pkgmap.apps:pipeline_app
+