Markdown-Checker 1.0.3__tar.gz → 1.1.0__tar.gz

{markdown_checker-1.0.3 → markdown_checker-1.1.0}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.4
 Name: markdown-checker
-Version: 1.0.3
+Version: 1.1.0
 Summary: A markdown link validation reporting tool.
 Author-email: John Aziz <johnaziz269@gmail.com>
 Maintainer-email: John Aziz <johnaziz269@gmail.com>
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-Expression: MIT
 Classifier: Development Status :: 5 - Production/Stable
@@ -12,7 +12,6 @@ Classifier: Environment :: Console
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13

{markdown_checker-1.0.3 → markdown_checker-1.1.0}/docs/advanced.rst

@@ -285,6 +285,57 @@ Usage examples
   # Short-form alias
   markdown-checker -d . -f check_broken_urls -frd 45
 
+``-mw``, ``--max-workers``
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Type
+  ``Click.IntRange``
+Description
+  Maximum number of concurrent URL-check workers.
+Default
+  ``10``, or the number of available CPUs when running in GitHub Actions
+  (``$GITHUB_ACTIONS=true``).
+Range
+  ``1`` or greater.
+Required
+  No.
+
+Usage examples
+^^^^^^^^^^^^^^
+
+.. code-block:: bash
+
+  # Use a custom worker count
+  markdown-checker -d . -f check_broken_urls --max-workers=20
+
+  # Short-form alias
+  markdown-checker -d . -f check_broken_urls -mw 20
+
+``-phd``, ``--per-host-delay``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Type
+  ``Click.FloatRange``
+Description
+  Minimum delay in seconds between requests to the same host.
+Default
+  ``0.5``
+Range
+  ``0.0-10.0``
+Required
+  No.
+
+Usage examples
+^^^^^^^^^^^^^^
+
+.. code-block:: bash
+
+  # Use a custom per-host delay of 1 second
+  markdown-checker -d . -f check_broken_urls --per-host-delay=1.0
+
+  # Short-form alias
+  markdown-checker -d . -f check_broken_urls -phd 1.0
+
 ``-o``, ``--output-file-name``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

{markdown_checker-1.0.3 → markdown_checker-1.1.0}/docs/api/main.rst

@@ -4,3 +4,4 @@ Main
 .. automodule:: markdown_checker
 	:members:
 	:undoc-members:
+	:noindex:

{markdown_checker-1.0.3 → markdown_checker-1.1.0}/docs/api/models.rst

@@ -9,6 +9,14 @@ Base
 	:undoc-members:
 	:show-inheritance:
 
+Config
+------
+
+.. automodule:: markdown_checker.models.config
+	:members:
+	:undoc-members:
+	:show-inheritance:
+
 Path
 ----
 

markdown_checker-1.1.0/docs/api/patterns.rst

@@ -0,0 +1,7 @@
+Patterns
+========
+
+.. automodule:: markdown_checker.patterns
+	:members:
+	:undoc-members:
+	:show-inheritance:

{markdown_checker-1.0.3 → markdown_checker-1.1.0}/docs/api/utils.rst

@@ -9,6 +9,14 @@ Extract Links
 	:undoc-members:
 	:show-inheritance:
 
+GitHub Environment
+-------------------
+
+.. automodule:: markdown_checker.utils.github_env
+	:members:
+	:undoc-members:
+	:show-inheritance:
+
 List Files
 ----------
 
@@ -24,3 +32,11 @@ Spinner
 	:members:
 	:undoc-members:
 	:show-inheritance:
+
+URL Pipeline
+------------
+
+.. automodule:: markdown_checker.utils.url_pipeline
+	:members:
+	:undoc-members:
+	:show-inheritance:

{markdown_checker-1.0.3 → markdown_checker-1.1.0}/docs/conf.py

@@ -15,6 +15,7 @@ extensions = [
     "sphinx.ext.autodoc",
     "sphinx.ext.extlinks",
     "sphinx.ext.intersphinx",
+    "sphinx.ext.napoleon",
     "sphinxcontrib.log_cabinet",
     "sphinx_tabs.tabs",
     "sphinx_copybutton",
@@ -23,6 +24,9 @@ extensions = [
 autodoc_member_order = "bysource"
 autodoc_typehints = "description"
 autodoc_preserve_defaults = True
+napoleon_google_docstring = True
+napoleon_numpy_docstring = False
+napoleon_use_rtype = False
 extlinks = {
     "issue": ("https://github.com/john0isaac/markdown-checker/issues/%s", "#%s"),
     "pr": ("https://github.com/john0isaac/markdown-checker/pull/%s", "#%s"),

{markdown_checker-1.0.3 → markdown_checker-1.1.0}/docs/index.rst

@@ -24,7 +24,7 @@ Requirements
 
 You will need the following prerequisites in order to use this library:
 
-- Python >= 3.10
+- Python >= 3.11
 - `httpx2 <https://pypi.org/project/httpx2/>`_
 - `click <https://pypi.org/project/click/>`_
 

{markdown_checker-1.0.3 → markdown_checker-1.1.0}/pyproject.toml

@@ -1,14 +1,14 @@
 [project]
 name = "markdown-checker"
 description= "A markdown link validation reporting tool."
-version = "1.0.3"
+version = "1.1.0"
 authors = [{ name = "John Aziz", email = "johnaziz269@gmail.com" }]
 maintainers = [{ name = "John Aziz", email = "johnaziz269@gmail.com" }]
 license = "MIT"
 license-files = ["LICENSE"]
 readme = "README.md"
 
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dependencies = [
     "click>=8.3.1",
     "httpx2>=2.0.0",
@@ -19,7 +19,6 @@ classifiers = [
     "Operating System :: OS Independent",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
@@ -111,7 +110,7 @@ order-by-type = false
 known-first-party = ["markdown_checker"]
 
 [tool.mypy]
-python_version = '3.10'
+python_version = '3.11'
 files = ["src"]
 show_error_codes = true
 pretty = true
@@ -147,7 +146,7 @@ exclude_also = [
 
 [tool.tox]
 env_list = [
-    "py3.14", "py3.13", "py3.12", "py3.11", "py3.10",
+    "py3.14", "py3.13", "py3.12", "py3.11",
     "style",
     "typing",
     "docs",

{markdown_checker-1.0.3 → markdown_checker-1.1.0}/src/markdown_checker/__init__.py

@@ -1,8 +1,8 @@
 """
-markdown-checker — a markdown link validation reporting tool.
+markdown-checker - a markdown link validation reporting tool.
 """
 
-__version__ = "1.0.3"
+__version__ = "1.1.0"
 
 from markdown_checker.cli import main
 from markdown_checker.models import Config

markdown_checker-1.1.0/src/markdown_checker/checker.py

@@ -0,0 +1,144 @@
+from collections import deque
+from collections.abc import Callable
+from dataclasses import dataclass
+from dataclasses import field
+from pathlib import Path
+
+from markdown_checker.checks import REGISTRY
+from markdown_checker.models import Config
+from markdown_checker.models import MarkdownLinkBase
+from markdown_checker.utils import get_links_from_md_file
+from markdown_checker.utils.url_pipeline import URLCheckService
+
+# Bounds on how many files may have their check submitted (but not yet collected) at
+# once. The window drains early once enough links are in flight to keep every worker
+# fed (target ~2x max_workers), so link-dense files don't balloon memory; it never
+# drains before _MIN_FILE_WINDOW files, so link-sparse files (few links per file) still
+# get real cross-file overlap; _MAX_FILE_WINDOW is an absolute cap regardless of
+# link density.
+_MIN_FILE_WINDOW = 4
+_MAX_FILE_WINDOW = 64
+
+
+@dataclass
+class CheckResult:
+    """Result of running a check on one or more files."""
+
+    issues: list[tuple[Path, list[MarkdownLinkBase]]] = field(default_factory=list)
+    links_checked: int = 0
+
+
+def detect_issues(
+    func: str,
+    file_path: Path,
+    config: Config,
+    service: URLCheckService | None = None,
+) -> tuple[list[MarkdownLinkBase], int]:
+    """
+    Detect issues in a single markdown file using the named check.
+
+    Args:
+        func (str): Name of the check to run (must be a key in REGISTRY).
+        file_path (Path): Path to the markdown file to check.
+        config (Config): Runtime configuration for the check.
+        service: Optional shared URL-checking pipeline, reused across files
+            in a run so duplicate URLs are only checked once.
+
+    Returns:
+        A tuple of (detected_issues, links_checked_count).
+
+    Raises:
+        ValueError: If func is not a registered check name.
+    """
+    check = REGISTRY.get(func)
+    if check is None:
+        raise ValueError(f"Unknown check function: {func!r}. Available: {list(REGISTRY)}")
+
+    all_links = get_links_from_md_file(file_path)
+    if not all_links.paths and not all_links.urls:
+        return [], 0
+
+    detected_issues = check.run(links=all_links, config=config, service=service)
+
+    links_count = len(all_links.paths) if check.link_type == "paths" else len(all_links.urls)
+    return detected_issues, links_count
+
+
+def run_check_on_files(
+    func: str,
+    files_paths: list[Path],
+    config: Config,
+    progress_callback: Callable[[], None],
+) -> CheckResult:
+    """
+    Run a named check across multiple files.
+
+    Files are processed through an adaptive window: a file's check is submitted
+    (but not necessarily collected) as soon as it's reached, so link extraction
+    for upcoming files overlaps with the network wait of files already submitted.
+    The window drains early once roughly ``2 * config.max_workers`` links are in
+    flight - enough to keep every worker fed without link-dense files ballooning
+    memory - but never before ``_MIN_FILE_WINDOW`` files, so link-sparse files
+    still get real overlap; ``_MAX_FILE_WINDOW`` is an absolute cap either way.
+    ``progress_callback`` fires as each file is submitted rather than once its
+    results are collected, so the bar advances smoothly instead of lagging
+    behind in bursts.
+
+    Args:
+        func: Name of the check to run (must be a key in REGISTRY).
+        files_paths: List of markdown file paths to check.
+        config: Runtime configuration for the check.
+        progress_callback: Callback invoked once each file has been submitted.
+
+    Returns:
+        A CheckResult with per-file issues and total links checked.
+    """
+    check = REGISTRY.get(func)
+    if check is None:
+        raise ValueError(f"Unknown check function: {func!r}. Available: {list(REGISTRY)}")
+
+    result = CheckResult()
+    # Shared across all files so duplicate URLs are checked once, requests to the
+    # same host are paced, and a rate limit pauses that host instead of retrying
+    # it file-by-file.
+    service = URLCheckService(config) if check.link_type == "urls" else None
+    window: deque[tuple[Path, object, int]] = deque()
+    window_links = 0
+    target_links_in_flight = max(config.max_workers * 2, 1)
+
+    def drain_one() -> int:
+        """Collect the oldest submitted file and return its link count."""
+        file_path, pending, links_count = window.popleft()
+        detected_issues = check.collect(pending)
+        result.links_checked += links_count
+        if detected_issues:
+            result.issues.append((file_path, detected_issues))
+        return links_count
+
+    def window_full() -> bool:
+        if len(window) >= _MAX_FILE_WINDOW:
+            return True
+        return len(window) >= _MIN_FILE_WINDOW and window_links >= target_links_in_flight
+
+    try:
+        for file_path in files_paths:
+            all_links = get_links_from_md_file(file_path)
+            if not all_links.paths and not all_links.urls:
+                progress_callback()
+                continue
+
+            pending = check.submit(links=all_links, config=config, service=service)
+            links_count = len(all_links.paths) if check.link_type == "paths" else len(all_links.urls)
+            window.append((file_path, pending, links_count))
+            window_links += links_count
+            progress_callback()
+            if window_full():
+                window_links -= drain_one()
+
+        while window:
+            drain_one()
+    finally:
+        if service is not None:
+            service.close()
+
+    return result

markdown_checker-1.1.0/src/markdown_checker/checks/base.py

@@ -0,0 +1,82 @@
+from abc import ABC
+from abc import abstractmethod
+from typing import cast
+from typing import Generic
+from typing import Literal
+from typing import TypeVar
+
+from markdown_checker.models import Config
+from markdown_checker.models import MarkdownLinkBase
+from markdown_checker.utils.extract_links import MarkdownLinks
+from markdown_checker.utils.url_pipeline import URLCheckService
+
+TLink = TypeVar("TLink", bound=MarkdownLinkBase, covariant=True)
+
+
+class BaseCheck(Generic[TLink], ABC):
+    """Abstract base class for all link checks."""
+
+    name: str
+    link_type: Literal["paths", "urls"]
+
+    @abstractmethod
+    def run(
+        self,
+        links: MarkdownLinks,
+        config: Config | None = None,
+        service: URLCheckService | None = None,
+    ) -> list[TLink]:
+        """
+        Run the check against the extracted links.
+
+        Args:
+            links (MarkdownLinks): Extracted URLs and paths from a file.
+            config: Runtime configuration for the check run.
+            service: Optional shared URL-checking pipeline (dedupe memo,
+                per-host pacing, circuit breaker). Only used by checks with
+                link_type "urls"; ignored otherwise. When None, URL checks
+                fall back to a private, run-scoped service instance.
+
+        Returns:
+            List of links that failed the check, each with an .issue set.
+        """
+        raise NotImplementedError
+
+    def submit(
+        self,
+        links: MarkdownLinks,
+        config: Config | None = None,
+        service: URLCheckService | None = None,
+    ) -> object:
+        """
+        Begin checking without blocking the caller, returning an opaque
+        "pending" token to be finished later via ``collect()``.
+
+        The default implementation simply runs the check synchronously and
+        hands the finished result to ``collect()`` unchanged - appropriate
+        for checks with no network I/O to overlap with other files. Checks
+        that can hand work off to a shared service (e.g. URL checks) should
+        override both ``submit()`` and ``collect()`` to split the non-blocking
+        submission from the blocking wait for results.
+
+        Args:
+            links (MarkdownLinks): Extracted URLs and paths from a file.
+            config: Runtime configuration for the check run.
+            service: Optional shared URL-checking pipeline.
+
+        Returns:
+            An opaque token to pass to ``collect()``.
+        """
+        return self.run(links=links, config=config, service=service)
+
+    def collect(self, pending: object) -> list[TLink]:
+        """
+        Resolve a pending token from ``submit()`` into final results.
+
+        Args:
+            pending: The token returned by a matching ``submit()`` call.
+
+        Returns:
+            List of links that failed the check, each with an .issue set.
+        """
+        return cast(list[TLink], pending)

{markdown_checker-1.0.3 → markdown_checker-1.1.0}/src/markdown_checker/checks/broken_paths.py

@@ -2,6 +2,7 @@ from markdown_checker.checks.base import BaseCheck
 from markdown_checker.models import Config
 from markdown_checker.models import MarkdownPath
 from markdown_checker.utils.extract_links import MarkdownLinks
+from markdown_checker.utils.url_pipeline import URLCheckService
 
 
 class BrokenPathsCheck(BaseCheck[MarkdownPath]):
@@ -14,6 +15,7 @@ class BrokenPathsCheck(BaseCheck[MarkdownPath]):
         self,
         links: MarkdownLinks,
         config: Config | None = None,
+        service: URLCheckService | None = None,
     ) -> list[MarkdownPath]:
         detected_issues: list[MarkdownPath] = []
         for path in links.paths:

markdown_checker-1.1.0/src/markdown_checker/checks/broken_urls.py

@@ -0,0 +1,88 @@
+from concurrent.futures import Future
+from typing import cast
+
+from markdown_checker.checks.base import BaseCheck
+from markdown_checker.models import Config
+from markdown_checker.models import MarkdownURL
+from markdown_checker.models.url import URLCheckResult
+from markdown_checker.utils.extract_links import MarkdownLinks
+from markdown_checker.utils.url_pipeline import URLCheckService
+
+# Domains known to block automated requests (e.g. Cloudflare 403);
+# always skipped for URL checks.
+_BUILTIN_SKIP_DOMAINS: list[str] = []
+
+# Opaque token handed from submit() to collect(): the submitted jobs, plus the
+# service to close afterwards if this check created a private one for itself.
+_Pending = tuple[list[tuple[MarkdownURL, "Future[URLCheckResult]"]], URLCheckService | None]
+
+
+def _to_issue(url: MarkdownURL, result: URLCheckResult) -> MarkdownURL | None:
+    """Map a URLCheckResult onto a MarkdownURL's issue fields."""
+    if result.status == "alive":
+        return None
+    if result.status == "broken":
+        url.issue = "is broken"
+        url.issue_level = "error"
+        return url
+    if result.status == "rate_limited":
+        url.issue = "was skipped due to rate limiting"
+    elif result.status == "unverifiable":
+        url.issue = "could not be verified (access was forbidden by the server)"
+    else:
+        # transient_error
+        url.issue = "could not be reached due to a network error"
+    url.issue_level = "warning"
+    return url
+
+
+class BrokenURLsCheck(BaseCheck[MarkdownURL]):
+    """Check for URLs in markdown files that return non-2xx responses."""
+
+    name = "check_broken_urls"
+    link_type = "urls"
+
+    def submit(
+        self,
+        links: MarkdownLinks,
+        config: Config | None = None,
+        service: URLCheckService | None = None,
+    ) -> object:
+        config = config or Config()
+        effective_skip = [domain.lower() for domain in [*config.skip_domains, *_BUILTIN_SKIP_DOMAINS]]
+        skip_urls_containing = config.skip_urls_containing
+
+        # Standalone fallback (direct calls, tests): own a private service for this run only.
+        owns_service = service is None
+        _service = service or URLCheckService(config)
+
+        pending: list[tuple[MarkdownURL, Future[URLCheckResult]]] = []
+        for url in links.urls:
+            hostname = url.host_name().lower()
+            if any(domain in hostname for domain in effective_skip) or any(
+                substring in url.link for substring in skip_urls_containing
+            ):
+                continue
+            pending.append((url, _service.submit(url)))
+        return (pending, _service if owns_service else None)
+
+    def collect(self, pending: object) -> list[MarkdownURL]:
+        jobs, owned_service = cast(_Pending, pending)
+        try:
+            results: list[MarkdownURL] = []
+            for url, future in jobs:
+                issue = _to_issue(url, future.result())
+                if issue is not None:
+                    results.append(issue)
+            return results
+        finally:
+            if owned_service is not None:
+                owned_service.close()
+
+    def run(
+        self,
+        links: MarkdownLinks,
+        config: Config | None = None,
+        service: URLCheckService | None = None,
+    ) -> list[MarkdownURL]:
+        return self.collect(self.submit(links, config=config, service=service))

{markdown_checker-1.0.3 → markdown_checker-1.1.0}/src/markdown_checker/checks/locale.py

@@ -2,6 +2,7 @@ from markdown_checker.checks.base import BaseCheck
 from markdown_checker.models import Config
 from markdown_checker.models import MarkdownURL
 from markdown_checker.utils.extract_links import MarkdownLinks
+from markdown_checker.utils.url_pipeline import URLCheckService
 
 # Domains where locale is required in the URL path; always skipped for locale checks.
 _BUILTIN_SKIP_DOMAINS: list[str] = [
@@ -19,6 +20,7 @@ class URLsLocaleCheck(BaseCheck[MarkdownURL]):
         self,
         links: MarkdownLinks,
         config: Config | None = None,
+        service: URLCheckService | None = None,
     ) -> list[MarkdownURL]:
         config = config or Config()
         effective_skip = [*config.skip_domains, *_BUILTIN_SKIP_DOMAINS]

{markdown_checker-1.0.3 → markdown_checker-1.1.0}/src/markdown_checker/checks/tracking.py

@@ -3,6 +3,7 @@ from markdown_checker.models import Config
 from markdown_checker.models import MarkdownPath
 from markdown_checker.models import MarkdownURL
 from markdown_checker.utils.extract_links import MarkdownLinks
+from markdown_checker.utils.url_pipeline import URLCheckService
 
 
 class URLsTrackingCheck(BaseCheck[MarkdownURL]):
@@ -15,6 +16,7 @@ class URLsTrackingCheck(BaseCheck[MarkdownURL]):
         self,
         links: MarkdownLinks,
         config: Config | None = None,
+        service: URLCheckService | None = None,
     ) -> list[MarkdownURL]:
         config = config or Config()
         skip_domains = config.skip_domains
@@ -44,6 +46,7 @@ class PathsTrackingCheck(BaseCheck[MarkdownPath]):
         self,
         links: MarkdownLinks,
         config: Config | None = None,
+        service: URLCheckService | None = None,
     ) -> list[MarkdownPath]:
         detected_issues: list[MarkdownPath] = []
         for path in links.paths:

{markdown_checker-1.0.3 → markdown_checker-1.1.0}/src/markdown_checker/cli.py

@@ -8,6 +8,7 @@ from markdown_checker.checker import run_check_on_files
 from markdown_checker.models.config import Config
 from markdown_checker.reports.format_output import format_issues_table
 from markdown_checker.reports.markdown import MarkdownGenerator
+from markdown_checker.utils.github_env import get_github_repo_blob_url
 from markdown_checker.utils.list_files import get_files_paths_list
 
 
@@ -144,6 +145,23 @@ class ListOfStrings(click.Option):
     help="Fallback delay in seconds when a 429 response has no Retry-After header.",
     required=False,
 )
+@click.option(
+    "-mw",
+    "--max-workers",
+    type=click.IntRange(min=1),
+    default=None,
+    help="Maximum number of concurrent URL-check workers. Defaults to 10, "
+    "or the number of available CPUs when running in GitHub Actions.",
+    required=False,
+)
+@click.option(
+    "-phd",
+    "--per-host-delay",
+    type=click.FloatRange(0.0, 10.0),
+    default=0.5,
+    help="Minimum delay in seconds between requests to the same host.",
+    required=False,
+)
 @click.option(
     "-o",
     "--output-file-name",
@@ -171,9 +189,17 @@ def main(
     retries: int,
     retry_on_429: bool,
     fallback_retry_delay: int,
+    max_workers: int | None,
+    per_host_delay: float,
     output_file_name: str,
 ) -> None:
     """A markdown link validation reporting tool."""
+    if max_workers is None:
+        if os.getenv("GITHUB_ACTIONS") == "true":
+            max_workers = max(1, os.cpu_count() or 10)
+        else:
+            max_workers = 10
+
     if src:
         files_paths = []
         for p in src:
@@ -198,8 +224,11 @@ def main(
         retries=retries,
         retry_on_429=retry_on_429,
         fallback_retry_delay=fallback_retry_delay,
+        max_workers=max_workers,
+        per_host_delay=per_host_delay,
         output_mode="ci" if os.getenv("CI", "false") == "true" else "local",
     )
+    repo_url = get_github_repo_blob_url() if config.output_mode == "ci" else None
 
     with click.progressbar(length=len(files_paths), label="Checking", hidden=config.output_mode == "ci") as bar:
         check_result = run_check_on_files(
@@ -251,7 +280,7 @@ def main(
                 error_by_file.append((path, file_errors))
 
         if error_count:
-            formatted_output = format_issues_table(error_by_file, config.output_mode)
+            formatted_output = format_issues_table(error_by_file, config.output_mode, repo_url)
             generator = MarkdownGenerator(contributing_guide_url=guide_url, output_file_name=output_file_name)
             generator.generate(func, formatted_output)
             click.echo(click.style(f"😭 Found {error_count} issues in the following files:", fg="red"), err=True)