check-python-versions 0.23.0__py3-none-any.whl

check_python_versions/sources/base.py

@@ -0,0 +1,87 @@
+from typing import Callable, Optional
+
+from ..utils import FileLines, FileOrFilename
+from ..versions import SortedVersionList
+
+
+ExtractorFn = Callable[[FileOrFilename], Optional[SortedVersionList]]
+UpdaterFn = Callable[[FileOrFilename, SortedVersionList], Optional[FileLines]]
+
+
+class Source:
+    """Source for information about supported Pythons.
+
+    Examples of sources are: setup.py, CI configuration files.
+
+    A source has a ``title``, a typical ``filename`` (which can be a
+    glob pattern).
+
+    A source knows how to extract the list of supported Python versions
+    from a file, and it can know how to update the list of versions.
+
+    A source knows whether it's possible/typical to include PyPy support
+    for packages that want to support PyPy.
+    """
+
+    def __init__(
+        self,
+        *,
+        title: Optional[str] = None,
+        filename: str,
+        extract: ExtractorFn,
+        update: Optional[UpdaterFn] = None,
+        check_pypy_consistency: bool,
+        has_upper_bound: bool,
+    ) -> None:
+        self.title = title or filename
+        self.filename = filename
+        self.extract = extract
+        self.update = update
+        self.check_pypy_consistency = check_pypy_consistency
+        self.has_upper_bound = has_upper_bound
+
+    def for_file(
+        self,
+        pathname: str,
+        versions: SortedVersionList,
+        relpath: str,
+    ) -> 'SourceFile':
+        title = relpath if self.title == self.filename else self.title
+        source = SourceFile(
+            title=title,
+            filename=self.filename,
+            extract=self.extract,
+            update=self.update,
+            check_pypy_consistency=self.check_pypy_consistency,
+            has_upper_bound=self.has_upper_bound,
+            pathname=pathname,
+            versions=versions,
+        )
+        return source
+
+
+class SourceFile(Source):
+    """A concrete source file with  information about supported Pythons.
+
+    Some sources (GitHub Actions) can have multiple files matching a
+    glob pattern.  Each of those gets its own ``SourceFile`` instance.
+    """
+
+    def __init__(
+        self,
+        *,
+        title: Optional[str] = None,
+        filename: str,
+        extract: ExtractorFn,
+        update: Optional[UpdaterFn] = None,
+        check_pypy_consistency: bool,
+        has_upper_bound: bool,
+        pathname: str,
+        versions: SortedVersionList,
+    ) -> None:
+        super().__init__(
+            title=title, filename=filename, extract=extract, update=update,
+            check_pypy_consistency=check_pypy_consistency,
+            has_upper_bound=has_upper_bound)
+        self.pathname = pathname
+        self.versions = versions

check_python_versions/sources/github.py

@@ -0,0 +1,162 @@
+"""
+Support for GitHub Actions.
+
+GitHub Actions are very flexible, so this code is going to make some
+simplifying assumptions:
+
+- you use a matrix strategy
+  - on 'python-version' that contains python versions, or
+  - on 'config' that contains lists of [python_version, tox_env]
+"""
+
+from typing import Optional, Set, Union
+
+import yaml
+
+from .base import Source
+from ..parsers.yaml import quote_string, update_yaml_list
+from ..sources.tox import toxenv_for_version
+from ..utils import FileLines, FileOrFilename, open_file
+from ..versions import SortedVersionList, Version
+
+
+GHA_WORKFLOW_FILE = '.github/workflows/tests.yml'
+GHA_WORKFLOW_GLOB = '.github/workflows/*.yml'
+
+
+def get_gha_python_versions(
+    filename: FileOrFilename = GHA_WORKFLOW_FILE,
+) -> Optional[SortedVersionList]:
+    """Extract supported Python versions from a GitHub workflow."""
+    with open_file(filename) as fp:
+        conf = yaml.safe_load(fp)
+
+    versions: Set[Version] = set()
+    had_matrix = False
+    for job_name, job in conf.get('jobs', {}).items():
+        matrix = job.get('strategy', {}).get('matrix', {})
+        if 'python-version' in matrix:
+            had_matrix = True
+            versions.update(
+                e for e in map(parse_gh_ver, matrix['python-version']) if e)
+        if 'config' in matrix:
+            had_matrix = True
+            versions.update(
+                parse_gh_ver(c[0])
+                for c in matrix['config']
+                if isinstance(c, list)
+            )
+        if 'include' in matrix:
+            for extra in matrix['include']:
+                if 'python-version' in extra:
+                    had_matrix = True
+                    versions.add(parse_gh_ver(extra['python-version']))
+
+    if not had_matrix:
+        return None
+    return sorted(set(versions))
+
+
+def parse_gh_ver(v: Union[str, float]) -> Version:
+    """Parse Python versions used for actions/setup-python@v2.
+
+    This format is not fully well documented.  There's support for
+    specifying things like
+
+    - "3.x" (latest minor in Python 3.x; currently 3.9)
+    - "3.7" (latest bugfix in Python 3.7)
+    - "3.7.2" (specific version to be downloaded and installed)
+    - "pypy2"/"pypy3"
+    - "pypy-2.7"/"pypy-3.6"
+    - "pypy-3.7-v7.3.3"
+
+    https://github.com/actions/python-versions/blob/main/versions-manifest.json
+    contains a list of supported CPython versions that can be downloaded
+    and installed; this includes prereleases, but doesn't include PyPy.
+    """
+    v = str(v)
+    if v.startswith(('pypy3', 'pypy-3')):
+        return Version.from_string('PyPy3')
+    elif v.startswith(('pypy2', 'pypy-2')):
+        return Version.from_string('PyPy')
+    else:
+        return Version.from_string(v)
+
+
+def update_gha_python_versions(
+    filename: FileOrFilename,
+    new_versions: SortedVersionList,
+) -> FileLines:
+    """Update supported Python versions in a GitHub workflow file.
+
+    Does not touch the file but returns a list of lines with new file contents.
+    """
+    with open_file(filename) as fp:
+        orig_lines = fp.readlines()
+        fp.seek(0)
+        conf = yaml.safe_load(fp)
+    new_lines = orig_lines
+
+    def keep_old_version(value: str) -> bool:
+        """Determine if a Python version line should be preserved."""
+        parsed = yaml.safe_load(value)
+        ver = parse_gh_ver(parsed)
+        if ver == Version.from_string('PyPy'):
+            return any(v.major == 2 for v in new_versions)
+        if ver == Version.from_string('PyPy3'):
+            return any(v.major == 3 for v in new_versions)
+        return False
+
+    def keep_old_config(value: str) -> bool:
+        """Determine if a Python version line should be preserved."""
+        parsed = yaml.safe_load(value)
+        if isinstance(parsed, list) and len(parsed) == 2:
+            ver = parse_gh_ver(parsed[0])
+            toxenv = str(parsed[1])
+        else:
+            return True
+        if ver == Version.from_string('PyPy'):
+            return any(v.major == 2 for v in new_versions)
+        if ver == Version.from_string('PyPy3'):
+            return any(v.major == 3 for v in new_versions)
+        return toxenv != toxenv_for_version(ver)
+
+    for job_name, job in conf.get('jobs', {}).items():
+        matrix = job.get('strategy', {}).get('matrix', {})
+        if 'python-version' in matrix:
+            quote_style = ''
+            if all(isinstance(v, str) for v in matrix['python-version']):
+                quote_style = '"'
+            yaml_new_versions = [
+                quote_string(str(v), quote_style)
+                for v in new_versions
+            ]
+            new_lines = update_yaml_list(
+                new_lines,
+                ('jobs', job_name, 'strategy', 'matrix', 'python-version'),
+                yaml_new_versions, filename=fp.name,
+                keep=keep_old_version,
+            )
+        if 'config' in matrix:
+            yaml_configs = []
+            for v in new_versions:
+                quoted_ver = quote_string(str(v), '"')
+                toxenv = quote_string(toxenv_for_version(v), '"')
+                yaml_configs.append(f"[{quoted_ver + ',':<8} {toxenv}]")
+            new_lines = update_yaml_list(
+                new_lines,
+                ('jobs', job_name, 'strategy', 'matrix', 'config'),
+                yaml_configs, filename=fp.name,
+                keep=keep_old_config,
+            )
+
+    return new_lines
+
+
+GitHubActions = Source(
+    filename=GHA_WORKFLOW_GLOB,
+    extract=get_gha_python_versions,
+    update=update_gha_python_versions,
+    check_pypy_consistency=True,
+    has_upper_bound=True,
+)

check_python_versions/sources/manylinux.py

@@ -0,0 +1,96 @@
+"""
+Support for .manylinux-install.sh.  This is a shell script used by multiple
+ZopeFoundation packages that builds manylinux wheels inside
+quay.io/pypa/manylinux* Docker images.
+
+The script loops over all installed Pythons, checks if each is a supported
+version using a series of `if` statements, then builds wheels for each
+supported versions.  This looks like ::
+
+    for PYBIN in /opt/python/*/bin; do
+        if [[ "${PYBIN}" == *"cp27"* ]] || \
+           [[ "${PYBIN}" == *"cp34"* ]] || \
+           [[ "${PYBIN}" == *"cp35"* ]] || \
+           [[ "${PYBIN}" == *"cp36"* ]] || \
+           [[ "${PYBIN}" == *"cp37"* ]]; then
+            "${PYBIN}/pip" install -e /io/
+            "${PYBIN}/pip" wheel /io/ -w wheelhouse/
+            rm -rf /io/build /io/*.egg-info
+        fi
+    done
+
+"""
+
+import re
+
+from .base import Source
+from ..utils import FileLines, FileOrFilename, open_file, warn
+from ..versions import SortedVersionList, Version, VersionList
+
+
+MANYLINUX_INSTALL_SH = '.manylinux-install.sh'
+
+
+def get_manylinux_python_versions(
+    filename: FileOrFilename = MANYLINUX_INSTALL_SH,
+) -> SortedVersionList:
+    """Extract supported Python versions from .manylinux-install.sh."""
+    magic = re.compile(r'.*\[\[ "\$\{PYBIN\}" == \*"cp(\d)(\d+)"\* \]\]')
+    versions = []
+    with open_file(filename) as fp:
+        for line in fp:
+            m = magic.match(line)
+            if m:
+                v = Version.from_string('{}.{}'.format(*m.groups()))
+                versions.append(v)
+    return sorted(set(versions))
+
+
+def update_manylinux_python_versions(
+    filename: FileOrFilename,
+    new_versions: VersionList,
+) -> FileLines:
+    """Update supported Python versions in .manylinux_install_sh.
+
+    Does not touch the file but returns a list of lines with new file contents.
+    """
+    magic = re.compile(r'.*\[\[ "\$\{PYBIN\}" == \*"cp(\d)(\d)"\* \]\]')
+    with open_file(filename) as f:
+        orig_lines = f.readlines()
+    lines = iter(enumerate(orig_lines))
+    for n, line in lines:
+        m = magic.match(line)
+        if m:
+            start = n
+            break
+    else:
+        warn(f'Failed to understand {f.name}')
+        return orig_lines
+    for n, line in lines:
+        m = magic.match(line)
+        if not m:
+            end = n
+            break
+    else:
+        warn(f'Failed to understand {f.name}')
+        return orig_lines
+
+    indent = ' ' * 4
+    conditions = f' || \\\n{indent}   '.join(
+        f'[[ "${{PYBIN}}" == *"cp{ver.major}{ver.minor}"* ]]'
+        for ver in new_versions
+    )
+    new_lines = orig_lines[:start] + (
+        f'{indent}if {conditions}; then\n'
+    ).splitlines(True) + orig_lines[end:]
+
+    return new_lines
+
+
+Manylinux = Source(
+    filename=MANYLINUX_INSTALL_SH,
+    extract=get_manylinux_python_versions,
+    update=update_manylinux_python_versions,
+    check_pypy_consistency=False,
+    has_upper_bound=True,
+)

check_python_versions/sources/pyproject.py

@@ -0,0 +1,244 @@
+"""
+Support for pyproject.toml.
+
+There are several build tools that use pyproject.toml to specify metadata.
+Some of them use the PEP 621::
+
+    [project]
+    classifiers = [
+        ...
+        "Programming Language :: Python :: 3.8",
+        ...
+    ]
+    requires-python = ">= 3.8"
+
+check-python-versions also supports old-style Flit and Poetry metadata::
+
+    [tool.flit.metadata]
+    classifiers = [
+        ...
+        "Programming Language :: Python :: 3.8",
+        ...
+    ]
+    requires-python = ">= 3.8"
+
+    [tool.poetry]
+    classifiers = [
+        ...
+        "Programming Language :: Python :: 3.8",
+        ...
+    ]
+
+    [tool.poetry.dependencies]
+    python = "^3.8"
+
+"""
+from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union
+
+import tomlkit
+from tomlkit import TOMLDocument, dumps, load
+
+from check_python_versions.parsers.poetry_version_spec import (
+    compute_poetry_spec,
+    detect_poetry_version_spec_style,
+)
+from check_python_versions.utils import file_name
+
+from .base import Source
+from ..parsers.classifiers import (
+    get_versions_from_classifiers,
+    update_classifiers,
+)
+from ..parsers.poetry_version_spec import parse_poetry_version_constraint
+from ..parsers.requires_python import (
+    compute_python_requires,
+    detect_style,
+    parse_python_requires,
+)
+from ..utils import FileLines, FileOrFilename, open_file, warn
+from ..versions import SortedVersionList
+
+
+PYPROJECT_TOML = 'pyproject.toml'
+
+
+if TYPE_CHECKING:
+    from tomlkit.container import Container
+    from tomlkit.items import Item
+
+
+def traverse(document: TOMLDocument, path: str, default: Any = None) -> Any:
+    obj: Union[Container, Item] = document
+    for step in path.split('.'):
+        if not isinstance(obj, dict):
+            # complain
+            return default
+        if step not in obj:
+            return default
+        obj = obj[step]
+    return obj
+
+
+def _get_pyproject_toml_classifiers(
+    filename: FileOrFilename = PYPROJECT_TOML,
+) -> Tuple[TOMLDocument, str, Optional[List[str]]]:
+    """Extract the list of PyPI classifiers from a pyproject.toml"""
+
+    with open_file(filename) as fp:
+        document = load(fp)
+
+    for path in 'project', 'tool.flit.metadata', 'tool.poetry':
+        classifiers = traverse(document, f"{path}.classifiers")
+        if classifiers is not None:
+            break
+
+    if classifiers is None:
+        return document, path, None
+
+    if not isinstance(classifiers, list):
+        warn(f'The value specified for {path}.classifiers in {fp.name}'
+             ' is not an array')
+        # Returning None means that pyproject.toml doesn't have metadata.
+        # Returning [] is likely to cause a mismatch with other
+        # metadata sources, making the problem more noticeable.
+        return document, path, []
+
+    if not all(isinstance(s, str) for s in classifiers):
+        warn(f'The value specified for {path}.classifiers in {fp.name}'
+             ' is not an array of strings')
+        # Returning None means that pyproject.toml doesn't have metadata.
+        # Returning [] is likely to cause a mismatch with other
+        # metadata sources, making the problem more noticeable.
+        return document, path, []
+
+    return document, path, classifiers
+
+
+def get_supported_python_versions(
+    filename: FileOrFilename = PYPROJECT_TOML,
+) -> Optional[SortedVersionList]:
+    """Extract supported Python versions from classifiers in pyproject.toml."""
+
+    _d, _p, classifiers = _get_pyproject_toml_classifiers(filename)
+
+    if classifiers is None:
+        return None
+
+    return get_versions_from_classifiers(classifiers)
+
+
+def _get_pyproject_toml_requires_python(
+    filename: FileOrFilename = PYPROJECT_TOML,
+) -> Tuple[TOMLDocument, str, Optional[str]]:
+
+    with open_file(filename) as fp:
+        document = load(fp)
+
+    for path in (
+        "project.requires-python",
+        "tool.flit.metadata.requires-python",
+        "tool.poetry.dependencies.python",
+    ):
+        python_requires = traverse(document, path)
+        if python_requires is not None:
+            break
+
+    if python_requires is None:
+        return document, path, None
+
+    if not isinstance(python_requires, str):
+        warn(f'The value specified for {path} in {fp.name} is not a string')
+        return document, path, None
+
+    return document, path, python_requires
+
+
+def get_python_requires(
+    filename: FileOrFilename = PYPROJECT_TOML,
+) -> Optional[SortedVersionList]:
+    """Extract Python versions from require-python in pyproject.toml."""
+
+    _d, path, python_requires = _get_pyproject_toml_requires_python(filename)
+
+    if python_requires is None:
+        return None
+
+    if path == 'tool.poetry.dependencies.python':
+        return parse_poetry_version_constraint(
+            python_requires, path, filename=file_name(filename))
+    else:
+        return parse_python_requires(
+            python_requires, path, filename=file_name(filename))
+
+
+def update_supported_python_versions(
+    filename: FileOrFilename,
+    new_versions: SortedVersionList,
+) -> Optional[FileLines]:
+    """Update classifiers in a pyproject.toml.
+
+    Does not touch the file but returns a list of lines with new file contents.
+    """
+
+    document, path, classifiers = _get_pyproject_toml_classifiers(filename)
+
+    if classifiers is None:
+        return None
+
+    new_classifiers = update_classifiers(classifiers, new_versions)
+
+    table = traverse(document, path)
+    table['classifiers'] = a = tomlkit.array().multiline(True)
+    a.extend(new_classifiers)
+
+    return dumps(document).splitlines(True)
+
+
+def update_python_requires(
+    filename: FileOrFilename,
+    new_versions: SortedVersionList,
+) -> Optional[FileLines]:
+    """Update python dependency in a pyproject.toml, if it's defined there.
+
+    Does not touch the file but returns a list of lines with new file contents.
+    """
+
+    document, path, python_requires = _get_pyproject_toml_requires_python(
+        filename)
+
+    if python_requires is None:
+        return None
+
+    if path == 'tool.poetry.dependencies.python':
+        new_python_spec = compute_poetry_spec(
+            new_versions, **detect_poetry_version_spec_style(python_requires))
+
+        table = traverse(document, path.rpartition('.')[0])
+        table['python'] = new_python_spec
+    else:
+        new_python_requires = compute_python_requires(
+            new_versions, **detect_style(python_requires))
+
+        table = traverse(document, path.rpartition('.')[0])
+        table['requires-python'] = new_python_requires
+
+    return dumps(document).splitlines(True)
+
+
+PyProject = Source(
+    title=PYPROJECT_TOML,
+    filename=PYPROJECT_TOML,
+    extract=get_supported_python_versions,
+    update=update_supported_python_versions,
+    check_pypy_consistency=True,
+    has_upper_bound=True,
+)
+
+PyProjectPythonRequires = Source(
+    title='- python_requires',
+    filename=PYPROJECT_TOML,
+    extract=get_python_requires,
+    update=update_python_requires,
+    check_pypy_consistency=False,
+    has_upper_bound=False,  # TBH it might have one!
+)