fc-data 0.2.0__py3-none-any.whl

datasmith/publish/records.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+import io
+from typing import Any
+
+import pyarrow as pa  # type: ignore[import-untyped]
+import pyarrow.parquet as pq  # type: ignore[import-untyped]
+
+from datasmith.github.models import FormulaCodeRecord
+from datasmith.utils import get_logger
+from datasmith.utils.db import fetch_all
+
+logger = get_logger("publish.records")
+
+
+def records_to_parquet(records: list[FormulaCodeRecord]) -> bytes:
+    """Serialize FormulaCodeRecords to Parquet bytes via pyarrow."""
+    if not records:
+        return b""
+
+    rows = [r.model_dump(mode="json") for r in records]
+
+    # Build schema from first record
+    table = pa.Table.from_pylist(rows)
+    buf = io.BytesIO()
+    pq.write_table(table, buf)
+    return buf.getvalue()
+
+
+def records_from_parquet(data: bytes) -> list[FormulaCodeRecord]:
+    """Deserialize Parquet bytes back to FormulaCodeRecords."""
+    if not data:
+        return []
+    buf = io.BytesIO(data)
+    table = pq.read_table(buf)
+    rows = table.to_pylist()
+    return [FormulaCodeRecord(**row) for row in rows]
+
+
+def records_from_supabase(
+    start_date: str | None = None,
+    end_date: str | None = None,
+    unpublished_only: bool = True,
+) -> list[FormulaCodeRecord]:
+    """Query Supabase for FormulaCodeRecords, optionally filtered by date and publish status."""
+    filters: dict[str, Any] = {"is_performance_commit": True}
+    is_null: list[str] = []
+    gte_filters: dict[str, Any] = {}
+    lte_filters: dict[str, Any] = {}
+
+    if unpublished_only:
+        is_null.append("published_at")
+    if start_date:
+        gte_filters["merged_at"] = start_date
+    if end_date:
+        lte_filters["merged_at"] = end_date
+
+    rows = fetch_all(
+        "pull_requests",
+        select="*",
+        filters=filters,
+        is_null=is_null or None,
+        gte_filters=gte_filters or None,
+        lte_filters=lte_filters or None,
+    )
+
+    records: list[FormulaCodeRecord] = []
+    for row in rows:
+        try:
+            records.append(
+                FormulaCodeRecord(
+                    owner=row["owner"],
+                    repo=row["repo"],
+                    issue_number=row["issue_number"],
+                    task_id=f"{row['owner']}__{row['repo']}-{row['issue_number']}",
+                    gt_hash=row.get("merge_commit_sha", ""),
+                    base_commit=row.get("base_sha", ""),
+                    date=row.get("merged_at"),
+                    instructions=row.get("rendered_problem", ""),
+                    classification=row.get("classification", ""),
+                    difficulty=row.get("difficulty", ""),
+                    container_name=row.get("container_name", ""),
+                    patch=row.get("patch", ""),
+                )
+            )
+        except Exception:
+            logger.warning(
+                "Failed to create record for %s/%s#%s", row.get("owner"), row.get("repo"), row.get("issue_number")
+            )
+
+    return records

datasmith/py.typed

@@ -0,0 +1 @@
+# This file marks the datasmith package as typed for mypy and other type checkers

datasmith/resolution/__init__.py

@@ -0,0 +1,14 @@
+"""Resolution package for analyzing commits and resolving dependencies."""
+
+from __future__ import annotations
+
+from typing import Any
+
+__all__ = ["analyze_commit"]
+
+
+def analyze_commit(sha: str, repo_name: str, bypass_cache: bool = False) -> dict[str, Any] | None:
+    """Lazy wrapper around :func:`datasmith.resolution.orchestrator.analyze_commit`."""
+    from .orchestrator import analyze_commit as _analyze_commit
+
+    return _analyze_commit(sha=sha, repo_name=repo_name, bypass_cache=bypass_cache)

datasmith/resolution/blocklist.py

@@ -0,0 +1,145 @@
+"""Dynamic blocklist for packages that don't exist on PyPI or can't be resolved."""
+
+from __future__ import annotations
+
+import json
+import re
+import threading
+
+from datasmith.utils import get_logger
+
+from .constants import GIT_CACHE_DIR
+
+logger = get_logger("resolution.blocklist")
+
+
+def normalize_package_name(name: str) -> str:
+    """Normalize a package name according to PEP 503."""
+    return re.sub(r"[-_.]+", "-", name).lower()
+
+
+BLOCKLIST_PATH = GIT_CACHE_DIR / "package_blocklist.json"
+
+_blocklist_lock = threading.Lock()
+_blocklist_cache: set[str] | None = None
+
+
+def _load_blocklist() -> set[str]:
+    """Load the blocklist from disk."""
+    if not BLOCKLIST_PATH.exists():
+        return set()
+    try:
+        with BLOCKLIST_PATH.open("r") as f:
+            data = json.load(f)
+            return set(data.get("blocked_packages", []))
+    except Exception as e:
+        logger.warning("Failed to load blocklist from %s: %s", BLOCKLIST_PATH, e)
+        return set()
+
+
+def _save_blocklist(blocklist: set[str]) -> None:
+    """Save the blocklist to disk."""
+    try:
+        BLOCKLIST_PATH.parent.mkdir(parents=True, exist_ok=True)
+        with BLOCKLIST_PATH.open("w") as f:
+            json.dump(
+                {
+                    "blocked_packages": sorted(blocklist),
+                    "description": "Packages that don't exist on PyPI or can't be resolved",
+                },
+                f,
+                indent=2,
+            )
+    except Exception as e:
+        logger.warning("Failed to save blocklist to %s: %s", BLOCKLIST_PATH, e)
+
+
+def get_blocklist() -> set[str]:
+    """Get the current blocklist of packages to filter out."""
+    global _blocklist_cache
+
+    with _blocklist_lock:
+        if _blocklist_cache is None:
+            _blocklist_cache = _load_blocklist()
+        return _blocklist_cache.copy()
+
+
+def add_to_blocklist(package_name: str) -> bool:
+    """Add a package to the blocklist. Returns True if newly added."""
+    global _blocklist_cache
+
+    if not package_name or not package_name.strip():
+        return False
+
+    package_name = normalize_package_name(package_name.strip())
+
+    with _blocklist_lock:
+        blocklist = _load_blocklist()
+        if package_name in blocklist:
+            return False
+        blocklist.add(package_name)
+        _save_blocklist(blocklist)
+        _blocklist_cache = blocklist.copy()
+        logger.info("Added '%s' to package blocklist", package_name)
+        return True
+
+
+def extract_failing_package(error_log: str) -> str | None:
+    """Extract the package name that caused a resolution failure from uv error logs."""
+    if not error_log:
+        return None
+
+    match = re.search(r"Because ([\w\-]+) was not found in the package registry", error_log)
+    if match:
+        return match.group(1)
+
+    match = re.search(r"Because there are no versions of ([\w\-]+)", error_log)
+    if match:
+        return match.group(1)
+
+    match = re.search(
+        r"Because you require ([\w\-]+)==[\d\.]+ and \1[><=!]+[\d\.]+, we can conclude",
+        error_log,
+    )
+    if match:
+        pkg = match.group(1)
+        if re.match(r"^\d+[\-\d]+$", pkg) or pkg in {"uninstall", "install"}:
+            return pkg
+
+    return None
+
+
+def should_retry_without_package(error_log: str) -> bool:
+    """Determine if a resolution failure should trigger a retry without the failing package."""
+    if not error_log:
+        return False
+    if "was not found in the package registry" in error_log:
+        return True
+    if "Because there are no versions of" in error_log:
+        return True
+    if "Failed to build" in error_log:
+        return False
+    if "Failed to download" in error_log:
+        return False
+    return False
+
+
+def remove_package_from_requirements(requirements: list[str], package_name: str) -> tuple[list[str], bool]:
+    """Remove all requirements for a given package from a list."""
+    if not package_name:
+        return requirements, False
+
+    package_name_normalized = normalize_package_name(package_name)
+    filtered: list[str] = []
+    was_removed = False
+
+    for req in requirements:
+        pkg_match = re.match(r"^([a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?)", req)
+        if pkg_match:
+            req_pkg_name = pkg_match.group(1)
+            if normalize_package_name(req_pkg_name) == package_name_normalized:
+                was_removed = True
+                continue
+        filtered.append(req)
+
+    return filtered, was_removed

datasmith/resolution/cache.py

@@ -0,0 +1,120 @@
+"""SQLite-backed caching helpers for resolution results."""
+
+from __future__ import annotations
+
+import contextlib
+import functools
+import os
+import pickle
+import re
+import sqlite3
+import threading
+from collections.abc import Iterator
+from typing import Callable, ParamSpec, TypeVar, cast
+
+_cache_lock = threading.Lock()
+_P = ParamSpec("_P")
+_T = TypeVar("_T")
+
+
+@contextlib.contextmanager
+def _file_lock(lock_path: str) -> Iterator[None]:
+    """Cross-process exclusive lock using a sidecar .lock file."""
+    lockfile = lock_path + ".lock"
+    fd = os.open(lockfile, os.O_CREAT | os.O_RDWR, 0o644)
+    try:
+        try:
+            import fcntl  # POSIX
+
+            fcntl.flock(fd, fcntl.LOCK_EX)
+            yield
+            fcntl.flock(fd, fcntl.LOCK_UN)
+        except ImportError:  # pragma: no cover
+            import msvcrt
+
+            msvcrt.locking(fd, msvcrt.LK_LOCK, 1)  # type: ignore[attr-defined]
+            yield
+            msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)  # type: ignore[attr-defined]
+    finally:
+        os.close(fd)
+
+
+def _get_db_connection(db_loc: str) -> sqlite3.Connection:
+    """Open a SQLite connection configured for concurrent workloads."""
+    conn = sqlite3.connect(db_loc, timeout=30, isolation_level=None)
+    jm = conn.execute("PRAGMA journal_mode=WAL;").fetchone()[0].lower()
+    if jm != "wal":
+        conn.execute("PRAGMA journal_mode=TRUNCATE;")
+        conn.execute("PRAGMA synchronous=FULL;")
+    else:
+        conn.execute("PRAGMA synchronous=NORMAL;")
+    conn.execute("PRAGMA busy_timeout=30000;")
+    return conn
+
+
+def cache_completion(db_loc: str, table_name: str = "cache") -> Callable[[Callable[_P, _T]], Callable[_P, _T]]:
+    """Cache function results in a SQLite table keyed by args/kwargs.
+
+    Passing ``bypass_cache=True`` to the wrapped function forces a refresh and
+    overwrites the cached result.
+    """
+    if not re.match(r"^\w+$", table_name):
+        raise ValueError("table_name must be alphanumeric/underscore only")
+
+    def decorator(func: Callable[_P, _T]) -> Callable[_P, _T]:
+        @functools.wraps(func)
+        def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> _T:
+            bypass = cast(bool, kwargs.pop("bypass_cache", False))
+            key_kwargs = dict(sorted(kwargs.items()))
+
+            conn = _get_db_connection(db_loc)
+            try:
+                function_name = func.__name__
+
+                create_sql = (
+                    f"CREATE TABLE IF NOT EXISTS {table_name} ("
+                    "  function_name TEXT NOT NULL,"
+                    "  argument_blob BLOB NOT NULL,"
+                    "  result_blob   BLOB,"
+                    "  created_at    TEXT DEFAULT CURRENT_TIMESTAMP,"
+                    "  updated_at    TEXT DEFAULT CURRENT_TIMESTAMP,"
+                    "  PRIMARY KEY (function_name, argument_blob)"
+                    ")"
+                )
+                with _file_lock(db_loc), _cache_lock:
+                    conn.execute(create_sql)
+
+                set_clause = "result_blob=excluded.result_blob, updated_at=CURRENT_TIMESTAMP"
+                args_blob = pickle.dumps((function_name, args, key_kwargs))
+
+                if not bypass:
+                    with _cache_lock:
+                        row = conn.execute(
+                            f"SELECT result_blob FROM {table_name} WHERE function_name=? AND argument_blob=?",  # noqa: S608
+                            (function_name, args_blob),
+                        ).fetchone()
+                    if row is not None:
+                        return cast(_T, pickle.loads(row[0]))  # noqa: S301
+
+                if "bypass_cache" in func.__code__.co_varnames:
+                    kwargs["bypass_cache"] = bypass
+
+                result = func(*args, **kwargs)
+                result_blob = pickle.dumps(result)
+
+                with _file_lock(db_loc), _cache_lock:
+                    conn.execute(
+                        f"""
+                        INSERT INTO {table_name}(function_name, argument_blob, result_blob)
+                        VALUES(?, ?, ?)
+                        ON CONFLICT(function_name, argument_blob) DO UPDATE SET {set_clause}
+                        """,
+                        (function_name, args_blob, result_blob),
+                    )
+                return result
+            finally:
+                conn.close()
+
+        return wrapped
+
+    return decorator

datasmith/resolution/constants.py

@@ -0,0 +1,277 @@
+"""Constants used in dependency resolution."""
+
+from __future__ import annotations
+
+import os
+import re
+import sys
+from pathlib import Path
+
+# Regular expressions
+ASV_REGEX = re.compile(r"(^|/)\.?asv[^/]*\.jsonc?$")
+REQ_TXT_REGEX = re.compile(r"(^|/)(?:constraints(?:\.[-\w]+)?|requirements.*)\.txt$")
+ANSI_RE = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]")
+EXTRA_MARKER_RE = re.compile(r';\s*extra\s*==\s*["\']([^"\']+)["\']')
+
+# File names
+PYPROJECT = "pyproject.toml"
+SETUP_CFG = "setup.cfg"
+SETUP_PY = "setup.py"
+ENV_YML_NAMES = {"environment.yml", "environment.yaml"}
+
+# Cache location for SQLite caches
+CACHE_LOCATION: str = os.getenv("CACHE_LOCATION", "cache.db")
+
+# Git cache directory
+GIT_CACHE_DIR = Path(os.getenv("GIT_CACHE_DIR", str(Path(CACHE_LOCATION).parent / "git"))).expanduser()
+GIT_CACHE_DIR.mkdir(parents=True, exist_ok=True)
+
+# Import name to PyPI package name mapping
+SPECIAL_IMPORT_TO_PYPI = {
+    "sklearn": "scikit-learn",
+    "PIL": "Pillow",
+    "cv2": "opencv-python",
+    "yaml": "PyYAML",
+    "bs4": "beautifulsoup4",
+    "Crypto": "pycryptodome",
+}
+
+# Conda-only and system packages that don't exist on PyPI
+CONDA_SYSTEM_PACKAGES = {
+    "pkg-config",
+    "compilers",
+    "c-compiler",
+    "cxx-compiler",
+    "fortran-compiler",
+    "gcc",
+    "gxx",
+    "gfortran",
+    "clang",
+    "clangxx",
+    "make",
+    "cmake",
+    "autoconf",
+    "automake",
+    "libtool",
+    "m4",
+    "patch",
+    "bison",
+    "flex",
+}
+
+# Packages that are stdlib or not installable via PyPI
+NOT_REQUIREMENTS = {
+    # py2 names / stdlib modules seen in repos
+    "configparser",
+    "ConfigParser",
+    "asyncore",
+    "distutils",
+    "sqlite3",
+    "tkinter",
+    "Tkinter",
+    "cStringIO",
+    "urllib",
+    "urllib2",
+    # setuptools internals
+    "pkg_resources",
+    # platform frameworks (macOS)
+    "AppKit",
+    "Foundation",
+    # Build/packaging tools (not runtime dependencies)
+    "py2exe",
+    "cx_Freeze",
+    "py2app",
+    "nuitka",
+    # CLI verbs / interpreter references that slip through tokenization
+    "python",
+    "Python",
+    "python3",
+    "install",
+    "0-29-32",
+    "1-0",
+    "1-2",
+    "1-22-0",
+    "1-3-2",
+    "2-18-4",
+    "2024-1-1",
+    "3-0-0a10",
+    "absl",
+    "afl",
+    "allel",
+    "cartopy-userconfig",
+    "closest-peak-direction-getter",
+    "conans",
+    "cprofile",
+    "dask-core",
+    "dateutil",
+    "dbe",
+    "deepchecks-metrics",
+    "geopandas-base",
+    "interpnd",
+    "jpeg-ls",
+    "libblas",
+    "libpantab",
+    "libwriter",
+    "mo-pack",
+    "mpl-toolkits",
+    "pylab",
+    "pyqt4",
+    "pytables",
+    "skbuild",
+    "sklearnex",
+    "skspatial",
+    "system",
+    "tunits",
+    "vcr",
+    "0-29-21",
+    "0-29-33",
+    "1-11-2",
+    "1-12",
+    "1-14-0",
+    "1-23-5",
+    "1-8-1",
+    "3-0",
+    "3-0-0a11",
+    "3-1-2",
+    "59-2-0",
+    "c-distances-openmp",
+    "column-parsers",
+    "copy-reg",
+    "cpickle",
+    "cryptodome",
+    "cupyx",
+    "h5r",
+    "h5s",
+    "h5t",
+    "givens-elimination",
+    "imp",
+    "libreader",
+    "nattype",
+    "omniscidbe",
+    "openjpeg",
+    "patoolib",
+    "peerplaysbase",
+    "probabilistic-direction-getter",
+    "pyhdk",
+    "pymake",
+    "pyqt",
+    "sksparse",
+    "splitting",
+    "stringio",
+    "uninstall",
+    "urlparse",
+    "0-29-30",
+    "1-9-1",
+    "2-2",
+    "3-0-5",
+    "3-2-0",
+    "backports",
+    "cdms2",
+    "flatted",
+    "h5z",
+    "pnetdicom",
+    "pypocketfft",
+    "vectorized",
+    "voyager-ext",
+}
+
+# Well-known PyPI packages (allowlist for common names)
+ALLOWLIST_COMMON_PYPI = {
+    "numpy",
+    "scipy",
+    "pandas",
+    "matplotlib",
+    "xarray",
+    "shapely",
+    "fiona",
+    "pyproj",
+    "rtree",
+    "torch",
+    "functorch",
+    "pytest",
+    "ipython",
+    "IPython",
+    "ipykernel",
+    "ipywidgets",
+    "Cython",
+    "cython",
+    "numba",
+    "scikit-learn",
+    "sklearn",
+    "sympy",
+    "h5py",
+    "Pillow",
+    "pillow",
+    "networkx",
+    "dask",
+    "seaborn",
+    "xgboost",
+    "statsmodels",
+    "pyarrow",
+    "geopandas",
+    "cartopy",
+    "tqdm",
+    "psycopg2",
+    "sqlalchemy",
+    "SQLAlchemy",
+    "requests",
+    "setuptools",
+    "wheel",
+    "pip",
+    "sphinx",
+    "nbsphinx",
+    "sphinx-gallery",
+    "black",
+    "isort",
+    "flake8",
+    "hypothesis",
+    "pyqt5",
+    "pyqt4",
+    "qtpy",
+    "jupyter",
+    "pooch",
+    "graphviz",
+    "numexpr",
+}
+
+# Generic names that are likely local modules, not PyPI packages
+GENERIC_LOCAL_NAMES = {
+    "lib",
+    "libs",
+    "utils",
+    "util",
+    "utilities",
+    "core",
+    "helpers",
+    "helper",
+    "common",
+    "base",
+    "tools",
+    "tool",
+    "config",
+    "configs",
+    "constants",
+    "const",
+    "types",
+    "models",
+    "model",
+    "tests",
+    "test",
+    "testing",
+    "benchmarks",
+    "benchmark",
+    "examples",
+    "example",
+    "scripts",
+    "script",
+    "data",
+    "docs",
+    "doc",
+    "documentation",
+}
+
+# Python stdlib modules (Python 3.10+)
+try:
+    STDLIB = set(sys.stdlib_module_names)
+except Exception:  # pragma: no cover
+    STDLIB = set()