fetchcode 0.4.0__tar.gz → 0.5.1__tar.gz

{fetchcode-0.4.0 → fetchcode-0.5.1}/.gitignore

@@ -72,3 +72,6 @@ tcl
 
 # Ignore Jupyter Notebook related temp files
 .ipynb_checkpoints/
+
+# Env Files
+.env

fetchcode-0.5.1/CHANGELOG.rst

@@ -0,0 +1,66 @@
+Changelog
+=========
+
+
+v0.5.1
+-------
+- Use authenticated requests for GitHub REST API calls
+
+
+v0.5.0
+-------
+- FetchCode now supports retrieving package info for following generic packages: 
+    * pkg:generic/linux
+    * pkg:generic/mtd-utils
+    * pkg:generic/barebox
+    * pkg:generic/e2fsprogs
+    * pkg:generic/udhcp
+    * pkg:generic/miniupnpc
+    * pkg:generic/miniupnpd
+    * pkg:generic/minissdpd
+    * pkg:generic/erofs-utils
+    * pkg:openssl/openssl
+
+- FetchCode also supports retrieving package info for packages hosted on GitHub specifically.
+    * pkg:github/avahi/avahi
+    * pkg:github/bestouff/genext2fs
+    * pkg:github/dosfstools/dosfstools
+    * pkg:github/google/brotli
+    * pkg:github/hewlettpackard/wireless-tools
+    * pkg:github/inotify-tools/inotify-tools
+    * pkg:github/libbpf/bpftool
+    * pkg:github/llvm/llvm-project
+    * pkg:github/nixos/nix
+    * pkg:github/plougher/squashfs-tools
+    * pkg:github/pupnp/pupnp
+    * pkg:github/python/cpython
+    * pkg:github/rpm-software-management/rpm
+    * pkg:github/shadow-maint/shadow
+    * pkg:github/sqlite/sqlite
+    * pkg:github/u-boot/u-boot
+
+
+v0.4.0
+-------
+- FetchCode now supports retrieving package info for following generic packages: 
+    ``busybox``, ``bzip2``, ``dnsmasq``, ``dropbear``, ``ebtables``, ``hostapd``, ``ipkg``,
+    ``iproute2``, ``iptables``, ``libnl``, ``lighttpd``, ``nftables``, ``openssh``, ``samba``,
+    ``syslinux``, ``toybox``, ``uclibc``, ``uclibc-ng``, ``util-linux`` and ``wpa_supplicant``.
+- FetchCode also supports retrieving package info for GNU packages.
+
+
+v0.3.0
+-------
+- Add `package_versions` for retrieving all released versions of a given package.
+
+
+v0.2.0
+-------
+
+- Don't delete temp directory in fetch_via_vcs
+
+v0.1.0
+---------
+
+First, initial release.
+ 

{fetchcode-0.4.0 → fetchcode-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fetchcode
-Version: 0.4.0
+Version: 0.5.1
 Summary: fetchcode is a library to reliably fetch code via HTTP, FTP and version control systems.
 Home-page: https://github.com/nexB/fetchcode
 Author: nexB. Inc. and others
@@ -31,6 +31,7 @@ Requires-Dist: htmllistparse
 Requires-Dist: packageurl-python
 Requires-Dist: requests
 Requires-Dist: python-dateutil
+Requires-Dist: python-dotenv
 Provides-Extra: testing
 Requires-Dist: pytest!=7.0.0,>=6; extra == "testing"
 Requires-Dist: pytest-xdist>=2; extra == "testing"

{fetchcode-0.4.0 → fetchcode-0.5.1}/setup.cfg

@@ -49,6 +49,7 @@ install_requires =
 	packageurl-python
 	requests
 	python-dateutil
+	python-dotenv
 
 [options.packages.find]
 where = src

{fetchcode-0.4.0 → fetchcode-0.5.1}/src/fetchcode/package.py

@@ -21,13 +21,19 @@ from typing import List
 from urllib.parse import urljoin
 
 import htmllistparse
-import requests
 from packageurl import PackageURL
 from packageurl.contrib.route import NoRouteAvailable
 from packageurl.contrib.route import Router
 
-from fetchcode.ipkg_release_info import IPKG_RELEASES
+from fetchcode.package_util import GITHUB_SOURCE_BY_PACKAGE
+from fetchcode.package_util import IPKG_RELEASES
+from fetchcode.package_util import UDHCP_RELEASES
+from fetchcode.package_util import ErofsUtilsGitHubSource
+from fetchcode.package_util import GitHubSource
+from fetchcode.package_util import MiniupnpPackagesGitHubSource
+from fetchcode.package_util import OpenSSLGitHubSource
 from fetchcode.packagedcode_models import Package
+from fetchcode.utils import get_response
 
 router = Router()
 
@@ -44,17 +50,6 @@ def info(url):
             return
 
 
-def get_response(url):
-    """
-    Generate `Package` object for a `url` string
-    """
-    resp = requests.get(url)
-    if resp.status_code == 200:
-        return resp.json()
-
-    raise Exception(f"Failed to fetch: {url}")
-
-
 def get_pypi_bugtracker_url(project_urls):
     bug_tracking_url = project_urls.get("Tracker")
     if not (bug_tracking_url):
@@ -216,53 +211,72 @@ def get_pypi_data_from_purl(purl):
 @router.route("pkg:github/.*")
 def get_github_data_from_purl(purl):
     """
-    Generate `Package` object from the `purl` string of github type
+    Yield `Package` object from the `purl` string of github type
     """
     purl = PackageURL.from_string(purl)
     name = purl.name
     namespace = purl.namespace
-    base_path = "https://api.github.com/repos"
-    api_url = f"{base_path}/{namespace}/{name}"
-    response = get_response(api_url)
-    homepage_url = response.get("homepage")
-    vcs_url = response.get("git_url")
-    github_url = "https://github.com"
-    bug_tracking_url = f"{github_url}/{namespace}/{name}/issues"
-    code_view_url = f"{github_url}/{namespace}/{name}"
-    license_data = response.get("license") or {}
-    declared_license = license_data.get("spdx_id")
-    primary_language = response.get("language")
-    yield Package(
-        homepage_url=homepage_url,
-        vcs_url=vcs_url,
-        api_url=api_url,
-        bug_tracking_url=bug_tracking_url,
-        code_view_url=code_view_url,
-        declared_license=declared_license,
-        primary_language=primary_language,
-        **purl.to_dict(),
+
+    gh_package = f"{namespace}/{name}"
+    gh_source_class = GITHUB_SOURCE_BY_PACKAGE.get(gh_package, GitHubSource)
+
+    return gh_source_class.get_package_info(purl)
+
+
+@router.route(
+    "pkg:generic/miniupnpc.*",
+    "pkg:generic/miniupnpd.*",
+    "pkg:generic/minissdpd.*",
+)
+def get_github_data_for_miniupnp(purl):
+    """
+    Yield `Package` object for miniupnp packages from GitHub.
+    """
+    generic_purl = PackageURL.from_string(purl)
+    github_repo_purl = PackageURL(
+        type="github",
+        namespace="miniupnp",
+        name="miniupnp",
+        version=generic_purl.version,
     )
-    release_url = f"{api_url}/releases"
-    releases = get_response(release_url)
-    for release in releases:
-        version = release.get("name")
-        version_purl = PackageURL(
-            type=purl.type, namespace=namespace, name=name, version=version
-        )
-        download_url = release.get("tarball_url")
-        code_view_url = f"{github_url}/{namespace}/{name}/tree/{version}"
-        version_vcs_url = f"{vcs_url}@{version}"
-        yield Package(
-            homepage_url=homepage_url,
-            vcs_url=version_vcs_url,
-            api_url=api_url,
-            bug_tracking_url=bug_tracking_url,
-            code_view_url=code_view_url,
-            declared_license=declared_license,
-            primary_language=primary_language,
-            download_url=download_url,
-            **version_purl.to_dict(),
-        )
+
+    return MiniupnpPackagesGitHubSource.get_package_info(
+        gh_purl=github_repo_purl, package_name=generic_purl.name
+    )
+
+
+@router.route(
+    "pkg:openssl/openssl.*",
+)
+def get_github_data_for_openssl(purl):
+    """
+    Yield `Package` object for OpenSSL package from GitHub.
+    """
+    generic_purl = PackageURL.from_string(purl)
+    github_repo_purl = PackageURL(
+        type="github",
+        namespace="openssl",
+        name="openssl",
+        version=generic_purl.version,
+    )
+
+    return OpenSSLGitHubSource.get_package_info(github_repo_purl)
+
+
+@router.route("pkg:generic/erofs-utils.*")
+def get_github_data_for_erofs_utils(purl):
+    """
+    Yield `Package` object for erofs-utils package from GitHub.
+    """
+    generic_purl = PackageURL.from_string(purl)
+    github_repo_purl = PackageURL(
+        type="github",
+        namespace="erofs",
+        name="erofs-utils",
+        version=generic_purl.version,
+    )
+
+    return ErofsUtilsGitHubSource.get_package_info(github_repo_purl)
 
 
 @router.route("pkg:bitbucket/.*")
@@ -340,9 +354,12 @@ def get_gnu_data_from_purl(purl):
     """Generate `Package` object from the `purl` string of gnu type"""
     purl = PackageURL.from_string(purl)
     source_archive_url = f"https://ftp.gnu.org/pub/gnu/{purl.name}/"
-    regex = r"^({}-)([\w.-]*)(.tar.gz)$".format(purl.name)
+    version_regex_template = r"^({}-)(?P<version>[\w.-]*)(.tar.gz)$"
+    version_regex = re.compile(version_regex_template.format(re.escape(purl.name)))
 
-    yield from extract_packages_from_listing(purl, source_archive_url, regex, [])
+    yield from extract_packages_from_listing(
+        purl, source_archive_url, version_regex, []
+    )
 
 
 @dataclasses.dataclass
@@ -356,8 +373,8 @@ class DirectoryListedSource:
             "description": "Flag indicating whether the archives are nested within another directory"
         },
     )
-    source_archive_regex: str = dataclasses.field(
-        default="",
+    source_archive_regex: re.Pattern = dataclasses.field(
+        default=None,
         metadata={
             "description": "Regular expression pattern to match files in the directory listing."
         },
@@ -388,10 +405,43 @@ class DirectoryListedSource:
             )
 
 
+# The udhcp is no longer maintained as a standalone project.
+# It has been fully integrated into busybox.
+class UdhcpDirectoryListedSource(DirectoryListedSource):
+    source_url = (
+        "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source/"
+    )
+
+    @classmethod
+    def get_package_info(cls, package_url):
+
+        version = package_url.version
+        if version and version in UDHCP_RELEASES:
+            archive = UDHCP_RELEASES[version]
+            yield Package(
+                homepage_url=cls.source_url,
+                download_url=archive["url"],
+                release_date=archive["date"],
+                **package_url.to_dict(),
+            )
+
+        else:
+            for version, data in UDHCP_RELEASES.items():
+                purl = PackageURL(type="generic", name="udhcp", version=version)
+                yield Package(
+                    homepage_url=cls.source_url,
+                    download_url=data["url"],
+                    release_date=data["date"],
+                    **purl.to_dict(),
+                )
+
+
 class IpkgDirectoryListedSource(DirectoryListedSource):
     source_url = "https://web.archive.org/web/20090326020239/http://handhelds.org/download/packages/ipkg/"
     is_nested = False
-    source_archive_regex = r"^(ipkg[-_])([\w.-]*)(_arm.ipk|.tar.gz)$"
+    source_archive_regex = re.compile(
+        r"^(ipkg[-_])(?P<version>[\w.-]*)(_arm.ipk|.tar.gz)$"
+    )
     ignored_files_and_dir = []
 
     @classmethod
@@ -408,7 +458,7 @@ class IpkgDirectoryListedSource(DirectoryListedSource):
             )
 
         else:
-            for version, data in archives.items():
+            for version, data in IPKG_RELEASES.items():
                 purl = PackageURL(type="generic", name="ipkg", version=version)
                 yield Package(
                     homepage_url=cls.source_url,
@@ -421,132 +471,165 @@ class IpkgDirectoryListedSource(DirectoryListedSource):
 class UtilLinuxDirectoryListedSource(DirectoryListedSource):
     source_url = "https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/"
     is_nested = True
-    source_archive_regex = r"^(util-linux-)([\w.-]*)(.tar.gz)$"
+    # Source archive ex: util-linux-1.2.3.tar.gz
+    source_archive_regex = re.compile(
+        r"^(util-linux-|util-linux-ng-)(?P<version>[\w.-]*)(.tar.gz)$"
+    )
     ignored_files_and_dir = []
 
 
 class BusyBoxDirectoryListedSource(DirectoryListedSource):
     source_url = "https://www.busybox.net/downloads/"
-    source_archive_regex = r"^(busybox-)([\w.-]*)(.tar.bz2)$"
+    # Source archive ex: busybox-1.2.3.tar.bz2
+    source_archive_regex = re.compile(r"^(busybox-)(?P<version>[\w.-]*)(.tar.bz2)$")
     is_nested = False
     ignored_files_and_dir = []
 
 
 class UclibcDirectoryListedSource(DirectoryListedSource):
     source_url = "https://www.uclibc.org/downloads/"
-    source_archive_regex = r"^(uClibc-)([\w.-]*)(.tar.gz)$"
+    # Source archive ex: uClibc-1.2.3.tar.gz
+    source_archive_regex = re.compile(r"^(uClibc-)(?P<version>[\w.-]*)(.tar.gz)$")
     is_nested = False
     ignored_files_and_dir = []
 
 
 class UclibcNGDirectoryListedSource(DirectoryListedSource):
     source_url = "https://downloads.uclibc-ng.org/releases/"
-    source_archive_regex = r"^(uClibc-ng-)([\w.-]*)(.tar.gz)$"
+    # Source archive ex: uClibc-ng-1.2.3.tar.gz
+    source_archive_regex = re.compile(r"^(uClibc-ng-)(?P<version>[\w.-]*)(.tar.gz)$")
     is_nested = True
     ignored_files_and_dir = []
 
 
 class Bzip2DirectoryListedSource(DirectoryListedSource):
     source_url = "https://sourceware.org/pub/bzip2/"
-    source_archive_regex = r"^(bzip2-)([\w.-]*)(.tar.gz)$"
+    # Source archive ex: bzip2-1.2.3.tar.gz
+    source_archive_regex = re.compile(r"^(bzip2-)(?P<version>[\w.-]*)(.tar.gz)$")
     is_nested = False
     ignored_files_and_dir = []
 
 
 class OpenSSHDirectoryListedSource(DirectoryListedSource):
     source_url = "https://cdn.openbsd.org/pub/OpenBSD/OpenSSH/"
-    source_archive_regex = r"^(openssh-)([\w.-]*)(.tgz|.tar.gz)$"
+    # Source archive ex: openssh-1.2.3.tar.gz
+    source_archive_regex = re.compile(r"^(openssh-)(?P<version>[\w.-]*)(.tgz|.tar.gz)$")
     is_nested = False
     ignored_files_and_dir = []
 
 
 class DnsmasqDirectoryListedSource(DirectoryListedSource):
     source_url = "https://thekelleys.org.uk/dnsmasq/"
-    source_archive_regex = r"^(dnsmasq-)([\w.-]*)(.tar.xz|.tar.gz)$"
+    # Source archive ex: dnsmasq-1.2.3.tar.gz
+    source_archive_regex = re.compile(
+        r"^(dnsmasq-)(?P<version>[\w.-]*)(.tar.xz|.tar.gz)$"
+    )
     is_nested = False
     ignored_files_and_dir = []
 
 
 class EbtablesDirectoryListedSource(DirectoryListedSource):
     source_url = "https://www.netfilter.org/pub/ebtables/"
-    source_archive_regex = r"^(ebtables-)([\w.-]*)(.tar.gz)$"
+    # Source archive ex: ebtables-1.2.3.tar.gz
+    source_archive_regex = re.compile(r"^(ebtables-)(?P<version>[\w.-]*)(.tar.gz)$")
     is_nested = False
     ignored_files_and_dir = []
 
 
 class HostapdDirectoryListedSource(DirectoryListedSource):
     source_url = "https://w1.fi/releases/"
-    source_archive_regex = r"^(hostapd-)([\w.-]*)(.tar.gz)$"
+    # Source archive ex: hostapd-1.2.3.tar.gz
+    source_archive_regex = re.compile(r"^(hostapd-)(?P<version>[\w.-]*)(.tar.gz)$")
     is_nested = False
     ignored_files_and_dir = []
 
 
 class Iproute2DirectoryListedSource(DirectoryListedSource):
     source_url = "https://mirrors.edge.kernel.org/pub/linux/utils/net/iproute2/"
-    source_archive_regex = r"^(iproute2-)([\w.-]*)(.tar.xz|.tar.gz)$"
+    source_archive_regex = re.compile(
+        # Source archive ex: iproute2-1.2.3.tar.gz
+        r"^(iproute2-)(?P<version>[\w.-]*)(.tar.xz|.tar.gz)$"
+    )
     is_nested = False
     ignored_files_and_dir = []
 
 
 class IptablesDirectoryListedSource(DirectoryListedSource):
     source_url = "https://www.netfilter.org/pub/iptables/"
-    source_archive_regex = r"^(iptables-)([\w.-]*)(.tar.bz2)$"
+    # Source archive ex: iptables-1.2.3.tar.bz2
+    source_archive_regex = re.compile(r"^(iptables-)(?P<version>[\w.-]*)(.tar.bz2)$")
     is_nested = False
     ignored_files_and_dir = []
 
 
 class LibnlDirectoryListedSource(DirectoryListedSource):
     source_url = "https://www.infradead.org/~tgr/libnl/files/"
-    source_archive_regex = r"^(libnl-)([\w.-]*)(.tar.gz)$"
+    # Source archive ex: libnl-1.2.3.tar.gz
+    source_archive_regex = re.compile(r"^(libnl-)(?P<version>[\w.-]*)(.tar.gz)$")
     is_nested = False
     ignored_files_and_dir = []
 
 
 class LighttpdDirectoryListedSource(DirectoryListedSource):
     source_url = "https://download.lighttpd.net/lighttpd/releases-1.4.x/"
-    source_archive_regex = r"^(lighttpd-)([\w.-]*)(.tar.gz)$"
+    # Source archive ex: lighttpd-1.2.3.tar.gz
+    source_archive_regex = re.compile(r"^(lighttpd-)(?P<version>[\w.-]*)(.tar.gz)$")
     is_nested = False
     ignored_files_and_dir = []
 
 
 class NftablesDirectoryListedSource(DirectoryListedSource):
     source_url = "https://www.netfilter.org/pub/nftables/"
-    source_archive_regex = r"^(nftables-)([\w.-]*)(.tar.xz|.tar.bz2)$"
+    # Source archive ex: nftables-1.2.3.tar.bz2
+    source_archive_regex = re.compile(
+        r"^(nftables-)(?P<version>[\w.-]*)(.tar.xz|.tar.bz2)$"
+    )
     is_nested = False
     ignored_files_and_dir = []
 
 
 class WpaSupplicantDirectoryListedSource(DirectoryListedSource):
     source_url = "https://w1.fi/releases/"
-    source_archive_regex = r"^(wpa_supplicant-)([\w.-]*)(.tar.gz)$"
+    # Source archive ex: wpa_supplicant-1.2.3.tar.gz
+    source_archive_regex = re.compile(
+        r"^(wpa_supplicant-)(?P<version>[\w.-]*)(.tar.gz)$"
+    )
     is_nested = False
     ignored_files_and_dir = []
 
 
 class SyslinuxDirectoryListedSource(DirectoryListedSource):
     source_url = "https://mirrors.edge.kernel.org/pub/linux/utils/boot/syslinux/"
-    source_archive_regex = r"^(syslinux-)([\w.-]*)(.tar.gz)$"
+    # Source archive ex: syslinux-1.2.3.tar.gz
+    source_archive_regex = re.compile(r"^(syslinux-)(?P<version>[\w.-]*)(.tar.gz)$")
     is_nested = False
     ignored_files_and_dir = []
 
 
 class SyslinuxDirectoryListedSource(DirectoryListedSource):
     source_url = "https://mirrors.edge.kernel.org/pub/linux/utils/boot/syslinux/"
-    source_archive_regex = r"^(syslinux-)([\w.-]*)(.tar.gz)$"
+    # Source archive ex: syslinux-1.2.3.tar.gz
+    source_archive_regex = re.compile(r"^(syslinux-)(?P<version>[\w.-]*)(.tar.gz)$")
     is_nested = False
     ignored_files_and_dir = []
 
 
 class ToyboxDirectoryListedSource(DirectoryListedSource):
     source_url = "http://www.landley.net/toybox/downloads/"
-    source_archive_regex = r"^(toybox-)([\w.-]*)(.tar.gz|.tar.bz2)$"
+    # Source archive ex: toybox-1.2.3.tar.gz
+    source_archive_regex = re.compile(
+        r"^(toybox-)(?P<version>[\w.-]*)(.tar.gz|.tar.bz2)$"
+    )
     is_nested = False
     ignored_files_and_dir = []
 
 
 class DropbearDirectoryListedSource(DirectoryListedSource):
     source_url = "https://matt.ucc.asn.au/dropbear/releases/"
-    source_archive_regex = r"^(dropbear-)([\w.-]*)(.tar.bz2|_i386.deb)$"
+    # Source archive ex: dropbear-1.2.3.tar.bz2
+    source_archive_regex = re.compile(
+        r"^(dropbear-)(?P<version>[\w.-]*)(.tar.bz2|_i386.deb)$"
+    )
     is_nested = False
     ignored_files_and_dir = [
         "dropbear-0.44test1.tar.bz2",
@@ -562,11 +645,58 @@ class DropbearDirectoryListedSource(DirectoryListedSource):
 
 class SambaDirectoryListedSource(DirectoryListedSource):
     source_url = "https://download.samba.org/pub/samba/stable/"
-    source_archive_regex = r"^(samba-)([\w.-]*)(.tar.gz)$"
+    # Source archive ex: samba-1.2.3.tar.gz
+    source_archive_regex = re.compile(r"^(samba-)(?P<version>[\w.-]*)(.tar.gz)$")
+    is_nested = False
+    ignored_files_and_dir = []
+
+
+class MtdUtilsDirectoryListedSource(DirectoryListedSource):
+    source_url = "https://infraroot.at/pub/mtd/"
+    # Source archive ex: mtd-utils-1.2.3.tar.bz2
+    source_archive_regex = re.compile(r"^(mtd-utils-)(?P<version>[\w.-]*)(.tar.bz2)$")
+    is_nested = False
+    ignored_files_and_dir = []
+
+
+class BareboxDirectoryListedSource(DirectoryListedSource):
+    source_url = "https://www.barebox.org/download/"
+    # Source archive ex: barebox-1.2.3.tar.bz2
+    source_archive_regex = re.compile(r"^(barebox-)(?P<version>[\w.-]*)(.tar.bz2)$")
     is_nested = False
     ignored_files_and_dir = []
 
 
+class LinuxDirectoryListedSource(DirectoryListedSource):
+    source_url = "https://mirrors.edge.kernel.org/pub/linux/kernel/"
+    # Source archive ex: linux-1.2.3.tar.gz
+    source_archive_regex = re.compile(r"^(linux-)(?P<version>[\w.-]*)(.tar.gz)$")
+    is_nested = True
+    ignored_files_and_dir = [
+        "Historic/",
+        "SillySounds/",
+        "crypto/",
+        "firmware/",
+        "next/",
+        "people/",
+        "ports/",
+        "projects/",
+        "testing/",
+        "tools/",
+        "uemacs/",
+    ]
+
+
+class E2fsprogsDirectoryListedSource(DirectoryListedSource):
+    source_url = (
+        "https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/"
+    )
+    # Source archive ex: e2fsprogs-1.2.3.tar.gz
+    source_archive_regex = re.compile(r"^(e2fsprogs-)(?P<version>[\w.-]*)(.tar.gz)$")
+    is_nested = True
+    ignored_files_and_dir = ["testing/"]
+
+
 DIR_SUPPORTED_PURLS = [
     "pkg:generic/busybox.*",
     "pkg:generic/bzip2.*",
@@ -589,6 +719,11 @@ DIR_SUPPORTED_PURLS = [
     "pkg:generic/util-linux.*",
     "pkg:generic/wpa_supplicant.*",
     "pkg:generic/ipkg.*",
+    "pkg:generic/mtd-utils.*",
+    "pkg:generic/barebox.*",
+    "pkg:generic/linux.*",
+    "pkg:generic/e2fsprogs.*",
+    "pkg:generic/udhcp.*",
 ]
 
 DIR_LISTED_SOURCE_BY_PACKAGE_NAME = {
@@ -612,6 +747,11 @@ DIR_LISTED_SOURCE_BY_PACKAGE_NAME = {
     "util-linux": UtilLinuxDirectoryListedSource,
     "wpa_supplicant": WpaSupplicantDirectoryListedSource,
     "ipkg": IpkgDirectoryListedSource,
+    "mtd-utils": MtdUtilsDirectoryListedSource,
+    "barebox": BareboxDirectoryListedSource,
+    "linux": LinuxDirectoryListedSource,
+    "e2fsprogs": E2fsprogsDirectoryListedSource,
+    "udhcp": UdhcpDirectoryListedSource,
 }
 
 
@@ -628,18 +768,17 @@ def get_packages_from_listing(purl, source_archive_url, regex, ignored_files_and
     """
     Return list of package data from a directory listing based on the specified regex.
     """
-    pattern = re.compile(regex)
     _, listing = htmllistparse.fetch_listing(source_archive_url)
 
     packages = []
     for file in listing:
-        if not pattern.match(file.name) or file.name in ignored_files_and_dir:
+        match = regex.match(file.name)
+        if not match or file.name in ignored_files_and_dir:
             continue
 
-        match = re.search(regex, file.name)
-        version = match.group(2)
+        version = match.group("version")
         version = version.strip("v").strip()
-        if not version:
+        if not version or not version[0].isdigit():
             continue
 
         modified_time = file.modified
@@ -691,7 +830,7 @@ def extract_package_from_nested_listing(purl, source_url, regex, ignored_files_a
     """
     _, listing = htmllistparse.fetch_listing(source_url)
     for directory in listing:
-        if not directory.name.endswith("/"):
+        if not directory.name.endswith("/") or directory.name in ignored_files_and_dir:
             continue
 
         directory_url = urljoin(source_url, directory.name)