fetchcode 0.5.2__tar.gz → 0.6.0__tar.gz

fetchcode-0.6.0/.github/workflows/docs-ci.yml

@@ -0,0 +1,34 @@
+name: CI Documentation
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: [3.9]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Dependencies
+        run:  pip install -e .[docs]
+
+      - name: Check Sphinx Documentation build minimally
+        working-directory: ./docs
+        run: sphinx-build -E -W source build
+
+      - name: Check for documentation style errors
+        working-directory: ./docs
+        run: ./scripts/doc8_style_check.sh
+
+

fetchcode-0.6.0/.github/workflows/pypi-release.yml

@@ -0,0 +1,83 @@
+name: Create library release archives, create a GH release and publish PyPI wheel and sdist on tag in main branch
+
+
+# This is executed automatically on a tag in the main branch
+
+# Summary of the steps:
+# - build wheels and sdist
+# - upload wheels and sdist to PyPI
+# - create gh-release and upload wheels and dists there
+# TODO: smoke test wheels and sdist
+# TODO: add changelog to release text body
+
+# WARNING: this is designed only for packages building as pure Python wheels
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - "v*.*.*"
+
+jobs:
+  build-pypi-distribs:
+    name: Build and publish library to PyPI
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install pypa/build
+        run: python -m pip install build --user
+
+      - name: Build a binary wheel and a source tarball
+        run: python -m build --sdist --wheel --outdir dist/
+
+      - name: Upload built archives
+        uses: actions/upload-artifact@v3
+        with:
+          name: pypi_archives
+          path: dist/*
+
+
+  create-gh-release:
+    name: Create GH release
+    needs:
+      - build-pypi-distribs
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Download built archives
+        uses: actions/download-artifact@v3
+        with:
+          name: pypi_archives
+          path: dist
+
+      - name: Create GH release
+        uses: softprops/action-gh-release@v1
+        with:
+          draft: true
+          files: dist/*
+
+
+  create-pypi-release:
+    name: Create PyPI release
+    needs:
+      - create-gh-release
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Download built archives
+        uses: actions/download-artifact@v3
+        with:
+          name: pypi_archives
+          path: dist
+
+      - name: Publish to PyPI
+        if: startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

fetchcode-0.6.0/CHANGELOG.rst

@@ -0,0 +1,78 @@
+Changelog
+=========
+
+v0.6.0
+------
+- Add package metadata support for CocoaPods.
+- Report all PyPI package versions even if the corresponding distributions are unavailable
+- Support Python ``3.11`` and ``3.12``, drop ``3.6`` and ``3.7`` support
+- Upgrade vendored pip to 24.2 (This removes the support for pseudo vcs URLs like ``git://``,
+    ``hg://``, ``svn://`` and ``bzr://`` see https://github.com/pypa/pip/pull/9436)
+
+
+v0.5.2
+------
+- Update link references of ownership from nexB to aboutcode-org
+
+
+v0.5.1
+-------
+- Use authenticated requests for GitHub REST API calls
+
+
+v0.5.0
+-------
+- FetchCode now supports retrieving package info for following generic packages:
+    * pkg:generic/linux
+    * pkg:generic/mtd-utils
+    * pkg:generic/barebox
+    * pkg:generic/e2fsprogs
+    * pkg:generic/udhcp
+    * pkg:generic/miniupnpc
+    * pkg:generic/miniupnpd
+    * pkg:generic/minissdpd
+    * pkg:generic/erofs-utils
+    * pkg:openssl/openssl
+
+- FetchCode also supports retrieving package info for packages hosted on GitHub specifically.
+    * pkg:github/avahi/avahi
+    * pkg:github/bestouff/genext2fs
+    * pkg:github/dosfstools/dosfstools
+    * pkg:github/google/brotli
+    * pkg:github/hewlettpackard/wireless-tools
+    * pkg:github/inotify-tools/inotify-tools
+    * pkg:github/libbpf/bpftool
+    * pkg:github/llvm/llvm-project
+    * pkg:github/nixos/nix
+    * pkg:github/plougher/squashfs-tools
+    * pkg:github/pupnp/pupnp
+    * pkg:github/python/cpython
+    * pkg:github/rpm-software-management/rpm
+    * pkg:github/shadow-maint/shadow
+    * pkg:github/sqlite/sqlite
+    * pkg:github/u-boot/u-boot
+
+
+v0.4.0
+-------
+- FetchCode now supports retrieving package info for following generic packages:
+    ``busybox``, ``bzip2``, ``dnsmasq``, ``dropbear``, ``ebtables``, ``hostapd``, ``ipkg``,
+    ``iproute2``, ``iptables``, ``libnl``, ``lighttpd``, ``nftables``, ``openssh``, ``samba``,
+    ``syslinux``, ``toybox``, ``uclibc``, ``uclibc-ng``, ``util-linux`` and ``wpa_supplicant``.
+- FetchCode also supports retrieving package info for GNU packages.
+
+
+v0.3.0
+-------
+- Add `package_versions` for retrieving all released versions of a given package.
+
+
+v0.2.0
+-------
+
+- Don't delete temp directory in fetch_via_vcs
+
+v0.1.0
+---------
+
+First, initial release.

fetchcode-0.6.0/Makefile

@@ -0,0 +1,56 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/skeleton for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+# Python version can be specified with `$ PYTHON_EXE=python3.x make conf`
+PYTHON_EXE?=python3
+VENV=venv
+ACTIVATE?=. ${VENV}/bin/activate;
+
+dev:
+	@echo "-> Configure the development envt."
+	./configure --dev
+
+isort:
+	@echo "-> Apply isort changes to ensure proper imports ordering"
+	${VENV}/bin/isort --sl -l 100 --skip=src/fetchcode/vcs/pip setup.py src tests
+
+black:
+	@echo "-> Apply black code formatter"
+	${VENV}/bin/black -l 100 --exclude=src/fetchcode/vcs/pip src tests setup.py  
+
+doc8:
+	@echo "-> Run doc8 validation"
+	@${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/
+
+valid: isort black
+
+check:
+	@echo "-> Run pycodestyle (PEP8) validation"
+	@${ACTIVATE} pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,migrations,settings.py,.cache,etc,src/fetchcode/vcs/pip,tests/data/ .
+	@echo "-> Run isort imports ordering validation"
+	@${ACTIVATE} isort --sl --check-only -l 100 --skip=src/fetchcode/vcs/pip setup.py src tests
+	@echo "-> Run black validation"
+	@${ACTIVATE} black --check --check -l 100 --exclude=src/fetchcode/vcs/pip src tests setup.py 
+
+clean:
+	@echo "-> Clean the Python env"
+	./configure --clean
+	rm -rf .venv/ .*cache/ *.egg-info/ build/ dist/
+	find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
+
+test:
+	@echo "-> Run the test suite"
+	${VENV}/bin/pytest -vvs
+
+docs:
+	rm -rf docs/_build/
+	@${ACTIVATE} sphinx-build docs/ docs/_build/
+
+.PHONY: conf dev check valid black isort clean test docs

fetchcode-0.6.0/PKG-INFO

@@ -0,0 +1,119 @@
+Metadata-Version: 2.1
+Name: fetchcode
+Version: 0.6.0
+Summary: fetchcode is a library to reliably fetch code via HTTP, FTP and version control systems.
+Home-page: https://github.com/aboutcode-org/fetchcode
+Author: nexB. Inc. and others
+Author-email: info@aboutcode.org
+License: Apache-2.0
+Keywords: utilities,package manager,dependencies,fetch,scancode,download,downloader
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development
+Classifier: Topic :: Utilities
+Requires-Python: >=3.8
+Description-Content-Type: text/x-rst
+License-File: apache-2.0.LICENSE
+License-File: NOTICE
+License-File: AUTHORS.rst
+License-File: CHANGELOG.rst
+License-File: CODE_OF_CONDUCT.rst
+Requires-Dist: attrs
+Requires-Dist: commoncode
+Requires-Dist: htmllistparse
+Requires-Dist: packageurl-python
+Requires-Dist: requests
+Requires-Dist: python-dateutil
+Requires-Dist: python-dotenv
+Provides-Extra: testing
+Requires-Dist: pytest!=7.0.0,>=6; extra == "testing"
+Requires-Dist: pytest-xdist>=2; extra == "testing"
+Requires-Dist: aboutcode-toolkit>=7.0.2; extra == "testing"
+Requires-Dist: pycodestyle>=2.8.0; extra == "testing"
+Requires-Dist: twine; extra == "testing"
+Requires-Dist: black; extra == "testing"
+Requires-Dist: isort; extra == "testing"
+Provides-Extra: docs
+Requires-Dist: Sphinx>=5.0.2; extra == "docs"
+Requires-Dist: sphinx-rtd-theme>=1.0.0; extra == "docs"
+Requires-Dist: sphinx-reredirects>=0.1.2; extra == "docs"
+Requires-Dist: doc8>=0.11.2; extra == "docs"
+Requires-Dist: sphinx-autobuild; extra == "docs"
+Requires-Dist: sphinx-rtd-dark-mode>=1.3.0; extra == "docs"
+Requires-Dist: sphinx-copybutton; extra == "docs"
+
+FetchCode: Utilities to fetch code from purls, URLs and version control repos.
+================================================================================
+
+FetchCode is a library to reliably fetch code via HTTP, FTP and version control
+systems. It can work using plain HTTP and FTP URLs, as well as
+`Package URLs <https://github.com/package-url>`_ and version control (VCS) URLs
+as used in Python pip and as specified in `SPDX Package Download Location
+<https://spdx.github.io/spdx-spec/3-package-information/#37-package-download-location>`_
+
+Homepage and support: https://github.com/aboutcode-org/fetchcode
+
+
+Why FetchCode?
+--------------
+
+It is surprisingly difficult to have a simple API to consistently fetch code
+from package repositories, version control repositories and APIs: each site
+and each package manager has its own unique and peculiar ways. FetchCode goal
+is to abstract all these details and make it easy to fetch things reliably.
+
+
+Development installation
+--------------------------
+
+Clone the repo::
+
+    git clone https://github.com/aboutcode-org/fetchcode
+
+Then install all the requirements using this command (on POSIX)::
+
+    ./configure --dev
+
+
+Running tests
+----------------
+
+To run test suite use::
+
+    pytest -vvs
+
+
+Usage
+--------
+
+Fetch a code archive and get a ``fetchcode.fetch.Response`` object back::
+
+    >>> from fetchcode import fetch
+    >>> f = fetch('https://github.com/aboutcode-org/fetchcode/archive/ab65b2e645c889887227ea49eb3332d885fd0a54.zip')
+    >>> f.location
+    '/tmp/tmp_cm02xsg'
+    >>> f.content_type
+    'application/zip'
+    >>> f.url
+    'https://github.com/aboutcode-org/fetchcode/archive/ab65b2e645c889887227ea49eb3332d885fd0a54.zip'
+
+Fetch some package metadata and get a ``fetchcode.packagedcode_models.Package`` object back::
+
+    >>> from fetchcode import package
+    >>> list(package.info('pkg:rubygems/files'))
+    [Package(type='rubygems', namespace=None, name='files', version=None)]
+
+
+License
+--------
+
+- SPDX-License-Identifier: Apache-2.0
+
+Copyright (c) nexB Inc. and others.

fetchcode-0.6.0/azure-pipelines.yml

@@ -0,0 +1,62 @@
+################################################################################
+# We use Azure to run the full tests suites on multiple Python 3.x
+# on multiple Windows, macOS and Linux versions all on 64 bits
+# These jobs are using VMs with Azure-provided Python builds
+################################################################################
+
+jobs:
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: ubuntu20_cpython
+          image_name: ubuntu-20.04
+          python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
+          test_suites:
+              all: venv/bin/pytest -n 2 -vvs
+
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: ubuntu22_cpython
+          image_name: ubuntu-22.04
+          python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
+          test_suites:
+              all: venv/bin/pytest -n 2 -vvs
+
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: macos12_cpython
+          image_name: macOS-12
+          python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
+          test_suites:
+              all: venv/bin/pytest -n 2 -vvs
+
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: macos13_cpython
+          image_name: macOS-13
+          python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
+          test_suites:
+              all: venv/bin/pytest -n 2 -vvs
+
+    - template: etc/ci/azure-posix.yml
+      parameters:
+          job_name: macos14_cpython_arm64
+          image_name: macOS-14
+          python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
+          test_suites:
+              all: venv/bin/pytest -n 2 -vvs
+
+    - template: etc/ci/azure-win.yml
+      parameters:
+          job_name: win2019_cpython
+          image_name: windows-2019
+          python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
+          test_suites:
+              all: venv\Scripts\pytest -n 2 -vvs
+
+    - template: etc/ci/azure-win.yml
+      parameters:
+          job_name: win2022_cpython
+          image_name: windows-2022
+          python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12']
+          test_suites:
+              all: venv\Scripts\pytest -n 2 -vvs

fetchcode-0.6.0/docs/Makefile

@@ -0,0 +1,28 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SPHINXAUTOBUILD = sphinx-autobuild
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Run the development server using sphinx-autobuild
+docs:
+	@echo
+	@echo "Starting up the docs server..."
+	@echo
+	$(SPHINXAUTOBUILD) --port 8000 --watch ${SOURCEDIR} $(SOURCEDIR) "$(BUILDDIR)/html" $(SPHINXOPTS) $(O)
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

fetchcode-0.6.0/docs/make.bat

@@ -0,0 +1,47 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+if "%SPHINXAUTOBUILD%" == "" (
+	set SPHINXAUTOBUILD=sphinx-autobuild
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+if "%1" == "" goto help
+
+if "%1" == "docs" goto docs
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:docs
+@echo
+@echo Starting up the docs server...
+@echo
+%SPHINXAUTOBUILD% --port 8000 --watch %SOURCEDIR% %SOURCEDIR% %BUILDDIR%\html %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

fetchcode-0.6.0/docs/source/_static/theme_overrides.css

@@ -0,0 +1,26 @@
+/* this is the container for the pages */
+.wy-nav-content {
+    max-width: 100%;
+    padding: 0px 40px 0px 0px;
+    margin-top: 0px;
+}
+
+.wy-nav-content-wrap {
+    border-right: solid 1px;
+}
+
+div.rst-content {
+    max-width: 1300px;
+    border: 0;
+    padding: 10px 80px 10px 80px;
+    margin-left: 50px;
+}
+
+@media (max-width: 768px) {
+    div.rst-content {
+        max-width: 1300px;
+        border: 0;
+        padding: 0px 10px 10px 10px;
+        margin-left: 0px;
+    }
+}

fetchcode-0.6.0/docs/source/conf.py

@@ -0,0 +1,116 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = "FetchCode"
+copyright = "nexB Inc. and others."
+author = "AboutCode.org authors and contributors"
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx.ext.intersphinx",
+    "sphinx_reredirects",
+    "sphinx_rtd_theme",
+    "sphinx_rtd_dark_mode",
+    "sphinx.ext.extlinks",
+    "sphinx_copybutton",
+]
+
+
+# Redirects for olds pages
+# See https://documatt.gitlab.io/sphinx-reredirects/usage.html
+redirects = {}
+
+# This points to aboutcode.readthedocs.io
+# In case of "undefined label" ERRORS check docs on intersphinx to troubleshoot
+# Link was created at commit - https://github.com/aboutcode-org/aboutcode/commit/faea9fcf3248f8f198844fe34d43833224ac4a83
+
+intersphinx_mapping = {
+    "aboutcode": ("https://aboutcode.readthedocs.io/en/latest/", None),
+    "scancode-workbench": (
+        "https://scancode-workbench.readthedocs.io/en/develop/",
+        None,
+    ),
+}
+
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_rtd_theme"
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+master_doc = "index"
+
+html_context = {
+    "display_github": True,
+    "github_user": "nexB",
+    "github_repo": "fetchcode",
+    "github_version": "develop",  # branch
+    "conf_py_path": "/docs/source/",  # path in the checkout to the docs root
+}
+
+html_css_files = [
+    "theme_overrides.css",
+]
+
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+html_show_sphinx = True
+
+# Define CSS and HTML abbreviations used in .rst files.  These are examples.
+# .. role:: is used to refer to styles defined in _static/theme_overrides.css and is used like this: :red:`text`
+rst_prolog = """
+.. |psf| replace:: Python Software Foundation
+
+.. # define a hard line break for HTML
+.. |br| raw:: html
+
+   <br />
+
+.. role:: red
+
+.. role:: img-title
+
+.. role:: img-title-para
+
+"""
+
+# -- Options for LaTeX output -------------------------------------------------
+
+latex_elements = {"classoptions": ",openany,oneside"}