Markdown-Checker 0.1.5__tar.gz → 0.2.1__tar.gz

{markdown_checker-0.1.5/src/markdown_checker.egg-info → markdown_checker-0.2.1}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: markdown-checker
-Version: 0.1.5
-Summary: A markdown link validation reporting tool
+Version: 0.2.1
+Summary: A markdown link validation reporting tool.
 Author-email: John Aziz <johnaziz269@gmail.com>
 Maintainer-email: John Aziz <johnaziz269@gmail.com>
 License: MIT License
@@ -32,26 +32,36 @@ Project-URL: Issues, https://github.com/john0isaac/markdown-checker/issues
 Project-URL: Contributing, https://github.com/john0isaac/markdown-checker/pulls
 Project-URL: Releases, https://github.com/john0isaac/markdown-checker/releases
 Classifier: Development Status :: 5 - Production/Stable
+Classifier: Environment :: Console
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Python: >=3.8
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: click
 Requires-Dist: requests
-Requires-Dist: types-requests
 Provides-Extra: dev
-Requires-Dist: pytest; extra == "dev"
-Requires-Dist: pytest-cov; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
-Requires-Dist: ruff; extra == "dev"
-Requires-Dist: black; extra == "dev"
 Requires-Dist: mypy; extra == "dev"
+Requires-Dist: types-requests; extra == "dev"
+Provides-Extra: lint
+Requires-Dist: ruff; extra == "lint"
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+Provides-Extra: docs
+Requires-Dist: mkdocs; extra == "docs"
+Requires-Dist: mkdocs-material; extra == "docs"
+Requires-Dist: mkdocstrings[python]; extra == "docs"
+Requires-Dist: pymdown-extensions; extra == "docs"
 
 [![PyPi](https://img.shields.io/pypi/v/markdown-checker)](https://pypi.org/project/markdown-checker/)
 [![Documentation Status](https://readthedocs.org/projects/markdown-checker/badge/?version=latest)](https://markdown-checker.readthedocs.io/en/latest/?badge=latest)
@@ -78,7 +88,9 @@ pip install markdown-checker
 
 1. Run `pip install markdown-checker`.
 2. Run `markdown-checker -d {src} -f {func} -gu {url}`. Replace `{src}` with the directory you want to analyze, `{func}` with the available functions like `check_broken_paths`, `{gu}` with your contribution guidance full URL.
-3. The output will be displayed in the terminal and in a `comments.md` file.
+3. The output will be displayed in the terminal and in a `comment.md` file.
+
+For more customizations read the docs.
 
 ## Using `markdown-checker` in GitHub Actions
 

{markdown_checker-0.1.5 → markdown_checker-0.2.1}/README.md

@@ -23,7 +23,9 @@ pip install markdown-checker
 
 1. Run `pip install markdown-checker`.
 2. Run `markdown-checker -d {src} -f {func} -gu {url}`. Replace `{src}` with the directory you want to analyze, `{func}` with the available functions like `check_broken_paths`, `{gu}` with your contribution guidance full URL.
-3. The output will be displayed in the terminal and in a `comments.md` file.
+3. The output will be displayed in the terminal and in a `comment.md` file.
+
+For more customizations read the docs.
 
 ## Using `markdown-checker` in GitHub Actions
 

{markdown_checker-0.1.5 → markdown_checker-0.2.1}/pyproject.toml

@@ -1,32 +1,31 @@
-[build-system]
-requires = ["setuptools", "wheel"]
-build-backend = "setuptools.build_meta"
-
 [project]
 name = "markdown-checker"
-description= "A markdown link validation reporting tool"
-version = "0.1.5"
+description= "A markdown link validation reporting tool."
+version = "0.2.1"
 authors = [{ name = "John Aziz", email = "johnaziz269@gmail.com" }]
 maintainers = [{ name = "John Aziz", email = "johnaziz269@gmail.com" }]
 license = {file = "LICENSE"}
 readme = "README.md"
 
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 dependencies = [
+    "click",
     "requests",
-    "types-requests",
-    ]
+]
 classifiers = [
     "Development Status :: 5 - Production/Stable",
+    "Environment :: Console",
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
-    "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3 :: Only",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
-    ]
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Software Development :: Quality Assurance",
+]
 
 [project.urls]
 Homepage = "https://github.com/john0isaac/markdown-checker"
@@ -40,26 +39,40 @@ Releases = "https://github.com/john0isaac/markdown-checker/releases"
 
 [project.optional-dependencies]
 dev = [
-    "pytest",
-    "pytest-cov",
     "pre-commit",
-    "ruff",
-    "black",
     "mypy",
+    "types-requests",
+]
+lint = [
+    "ruff"
+]
+test = [
+    "pytest",
+    "pytest-cov"
+]
+docs = [
+    "mkdocs",
+    "mkdocs-material",
+    "mkdocstrings[python]",
+    "pymdown-extensions"
 ]
 
 [tool.ruff]
+lint.select = ["E", "F", "I", "UP"]
 line-length = 120
-target-version = "py38"
+target-version = "py39"
 src = ["src"]
+lint.isort.known-first-party = ["markdown_checker"]
 output-format = "full"
 
-[tool.ruff.lint]
-select = ["E", "F", "I", "UP"]
-
-[tool.black]
-line-length = 120
-target-version = ["py38"]
+[tool.mypy]
+check_untyped_defs = true
+python_version = 3.9
+exclude = [".venv/*"]
 
 [project.scripts]
-markdown-checker = "markdown_checker.validate_markdown:main"
+markdown-checker = "markdown_checker:main_with_spinner"
+
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"

markdown_checker-0.2.1/src/markdown_checker/__init__.py

@@ -0,0 +1,373 @@
+#!/usr/bin/env python3
+"""
+Module providing automatic checks functionality to markdown files
+following some Guidelines
+"""
+
+import concurrent.futures
+import os
+import platform
+import sys
+from pathlib import Path
+from typing import Union
+
+import click
+
+from markdown_checker.paths import MarkdownPath
+from markdown_checker.reports.md_reports.generator import MarkdownGenerator
+from markdown_checker.urls import MarkdownURL
+from markdown_checker.utils.extract_links import get_links_from_md_file
+from markdown_checker.utils.format_output import format_links
+from markdown_checker.utils.list_files import get_files_paths_list
+from markdown_checker.utils.spinner import spinner
+
+
+def check_url(
+    url: MarkdownURL, skip_domains: list[str], skip_urls_containing: list[str], timeout: int, retries: int
+) -> Union[None, MarkdownURL]:
+    if any(url.host_name().lower() in domain.lower() for domain in skip_domains) or any(
+        url.link in substring for substring in skip_urls_containing
+    ):
+        return None
+    if not url.is_alive(timeout=timeout, retries=retries):
+        url.issue = "is broken"
+        return url
+    return None
+
+
+def detect_issues(
+    func: str,
+    file_path: Path,
+    skip_urls_containing: list[str],
+    skip_domains: list[str],
+    tracking_domains: list[str],
+    timeout: int,
+    retries: int,
+) -> tuple[list[Union[MarkdownPath, MarkdownURL]], int]:
+    """
+    Function to detect issues in the markdown file based on the function
+
+    Args:
+        func (str): Function to be executed
+        file_path (Path): Path to the markdown file
+        skip_urls_containing (list[str]): List of urls to skip check
+        skip_domains (list[str]): List of domains to skip check
+        tracking_domains (list[str]): List of tracking domains to check
+        timeout (int): Timeout in seconds for the requests
+        retries (int): Number of retries for the requests
+
+    Returns:
+        Detected issues and links count.
+    """
+    detected_issues: list[Union[MarkdownPath, MarkdownURL]] = []
+    links_count = 0
+    # Step 1: Extract all Links from file path
+    all_links = get_links_from_md_file(file_path)
+    if len(all_links.paths) == 0 and len(all_links.urls) == 0:
+        return detected_issues, links_count
+    # Step 2: Check for issues based on the function
+    if func == "check_broken_paths":
+        for path in all_links.paths:
+            if not path.exists():
+                path.issue = "is broken"
+                detected_issues.append(path)
+        links_count += len(all_links.paths)
+    elif func == "check_broken_urls":
+        # currently these domains are known to have restrictions on the requests
+        skip_domains.extend(
+            [
+                "platform.openai.com",
+                "help.openai.com",
+                "beta.openai.com",
+                "marketplace.visualstudio.com",
+                "huggingface.co",
+                "en.wikipedia.org",
+                "twitter.com",
+                "www.linkedin.com",
+                "make.powerautomate.com",
+                "make.powerapps.com",
+                "www.midjourney.com",
+                "vscode.dev",
+                "rodtrent.substack.com",
+                "example.com",
+                "www.nuget.org",
+                "www.docker.com",
+                "build.nvidia.com",
+                "dotnet.microsoft.com",
+                "www.gemini.com",
+                "upload.wikimedia.org",
+                "medium.com",
+                "blogs.nvidia.com",
+                "blog.gopenai.com",
+                "towardsdatascience.com",
+                "code.visualstudio.com",
+            ]
+        )
+        with concurrent.futures.ProcessPoolExecutor() as executor:
+            results = list(
+                executor.map(
+                    check_url,
+                    all_links.urls,
+                    [skip_domains] * len(all_links.urls),
+                    [skip_urls_containing] * len(all_links.urls),
+                    [timeout] * len(all_links.urls),
+                    [retries] * len(all_links.urls),
+                )
+            )
+        detected_issues.extend(filter(None, results))
+        links_count += len(all_links.urls)
+    elif func == "check_urls_tracking":
+        for url in all_links.urls:
+            if any(url.host_name().lower() in domain.lower() for domain in skip_domains) or any(
+                url.link in substring for substring in skip_urls_containing
+            ):
+                continue
+            if any(url.host_name().lower() in domain.lower() for domain in tracking_domains) and not url.has_tracking():
+                url.issue = "is missing tracking id"
+                detected_issues.append(url)
+        links_count += len(all_links.urls)
+    elif func == "check_paths_tracking":
+        for path in all_links.paths:
+            if not path.has_tracking():
+                path.issue = "is missing tracking id"
+                detected_issues.append(path)
+        links_count += len(all_links.paths)
+    elif func == "check_urls_locale":
+        # if you remove locale from these domains, it will fail
+        skip_domains.extend(
+            [
+                "www.nvidia.com",
+            ]
+        )
+        for url in all_links.urls:
+            if any(url.host_name().lower() in domain.lower() for domain in skip_domains) or any(
+                url.link in substring for substring in skip_urls_containing
+            ):
+                continue
+            if url.has_locale():
+                url.issue = "has locale"
+                detected_issues.append(url)
+        links_count += len(all_links.urls)
+    elif func == "check_paths_locale":
+        for path in all_links.paths:
+            if path.has_locale():
+                path.issue = "has locale"
+                detected_issues.append(path)
+        links_count += len(all_links.paths)
+    return detected_issues, links_count
+
+
+class ListOfStrings(click.Option):
+    """
+    Helper class to parse a list of strings from the command line.
+
+    Ref: https://stackoverflow.com/questions/47631914/how-to-pass-several-list-of-arguments-to-click-option
+    """
+
+    def type_cast_value(self, ctx, value):
+        try:
+            if isinstance(value, str):
+                # split the string by comma and remove empty strings
+                return list(filter(None, value.split(",")))
+            elif isinstance(value, list):
+                return value
+            else:
+                raise Exception
+        except Exception:
+            raise click.BadParameter(value)
+
+
+@click.command()
+@click.option(
+    "-d",
+    "--dir",
+    type=click.Path(path_type=Path, exists=True, file_okay=False, dir_okay=True, readable=True),
+    help="Path to the root directory to check.",
+    required=True,
+)
+@click.option(
+    "-f",
+    "--func",
+    type=click.Choice(
+        [
+            "check_broken_paths",
+            "check_broken_urls",
+            "check_paths_tracking",
+            "check_urls_tracking",
+            "check_urls_locale",
+        ]
+    ),
+    help="Function to be executed.",
+    required=True,
+)
+@click.option(
+    "-ext",
+    "--extensions",
+    cls=ListOfStrings,
+    type=list[str],
+    default=[".md", ".ipynb"],
+    help="File extensions to filter the files.",
+    required=False,
+)
+@click.option(
+    "-td",
+    "--tracking-domains",
+    cls=ListOfStrings,
+    type=list[str],
+    default=["github.com", "microsoft.com", "visualstudio.com", "aka.ms", "azure.com"],
+    help="List of tracking domains to check.",
+    required=False,
+)
+@click.option(
+    "-sf",
+    "--skip-files",
+    cls=ListOfStrings,
+    type=list[str],
+    default=[
+        "CODE_OF_CONDUCT.md",
+        "SECURITY.md",
+    ],
+    help="List of file names to skip check.",
+    required=False,
+)
+@click.option(
+    "-sd",
+    "--skip-domains",
+    cls=ListOfStrings,
+    type=list[str],
+    default=[],
+    help="List of domains to skip check.",
+    required=False,
+)
+@click.option(
+    "-suc",
+    "--skip-urls-containing",
+    cls=ListOfStrings,
+    type=list[str],
+    default=["https://www.microsoft.com/en-us/security/blog", "video-embed.html"],
+    help="List of urls to skip check.",
+    required=False,
+)
+@click.option(
+    "-gu",
+    "--guide-url",
+    type=str,
+    help="Full url of your contributing guide.",
+    required=False,
+)
+@click.option(
+    "-to",
+    "--timeout",
+    type=click.IntRange(0, 50),
+    default=10,
+    help="Timeout in seconds for the requests.",
+    required=False,
+)
+@click.option(
+    "-rt",
+    "--retries",
+    type=click.IntRange(0, 10),
+    default=3,
+    help="Number of retries for the requests.",
+    required=False,
+)
+@click.option(
+    "-o",
+    "--output-file-name",
+    type=str,
+    default="comment",
+    help="Name of the output file.",
+    required=False,
+)
+@click.argument(
+    "src",
+    nargs=-1,
+    type=click.Path(path_type=Path, exists=True, file_okay=True, dir_okay=True, readable=True),
+    is_eager=True,
+    metavar="SRC ...",
+    required=False,
+)
+@click.version_option(
+    message=(f"%(prog)s, %(version)s\n" f"Python ({platform.python_implementation()}) {platform.python_version()}"),
+)
+def main(
+    src: tuple[Path, ...],
+    dir: Path,
+    func: str,
+    guide_url: Union[str, None],
+    extensions: list[str],
+    skip_files: list[str],
+    skip_domains: list[str],
+    skip_urls_containing: list[str],
+    tracking_domains: list[str],
+    timeout: int,
+    retries: int,
+    output_file_name: str,
+) -> None:
+    """A markdown link validation reporting tool."""
+    _ = src or (Path("./"),)  # default to current directory
+    _, files_paths = get_files_paths_list(dir, extensions)
+
+    # remove files from skip_files list
+    files_paths = [file_path for file_path in files_paths if file_path.name not in skip_files]
+
+    formatted_output = ""
+    all_files_issues: list[Union[MarkdownPath, MarkdownURL]] = []
+    links_checked_count = 0
+    github_ci = os.getenv("CI", "false")
+
+    # iterate over the files to validate the content
+    for file_path in files_paths:
+        detected_issues, links_count = detect_issues(
+            func=func,
+            file_path=file_path,
+            skip_urls_containing=skip_urls_containing,
+            skip_domains=skip_domains,
+            tracking_domains=tracking_domains,
+            timeout=timeout,
+            retries=retries,
+        )
+        links_checked_count += links_count
+        if len(detected_issues) > 0:
+            if github_ci == "true":
+                formatted_output += f"| `{file_path}` |" + format_links(detected_issues)
+            else:
+                formatted_output += f"| [`{file_path}`]({file_path}) |" + format_links(detected_issues)
+            all_files_issues.extend(detected_issues)
+    click.echo(
+        click.style(f"\n🔍 Checked {links_checked_count} links in {len(files_paths)} files.", fg="blue"), err=False
+    )
+    if len(all_files_issues) > 0:
+        formatted_output = "| File Full Path | Issues |\n|--------|--------|\n" + formatted_output
+        generator = MarkdownGenerator(contributing_guide_url=guide_url, output_file_name=output_file_name)
+        generator.generate(func, formatted_output)
+        click.echo(click.style(f"😭 Found {len(all_files_issues)} issues in the following files:", fg="red"), err=True)
+        for markdown_path in all_files_issues:
+            if github_ci == "true":
+                # Ref: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-a-warning-message
+                click.echo(
+                    click.style(
+                        f"::error file={markdown_path.file_path},line={markdown_path.line_number}::"
+                        f"File {markdown_path.file_path}, line {markdown_path.line_number}, "
+                        f"Link {markdown_path} {markdown_path.issue}.",
+                        fg="red",
+                    ),
+                    err=True,
+                )
+            else:
+                click.echo(
+                    click.style(
+                        f"\tFile '{markdown_path.file_path.resolve()}', line {markdown_path.line_number}"
+                        f"\n{markdown_path} {markdown_path.issue}.\n",
+                        fg="red",
+                    ),
+                    err=True,
+                )
+        sys.exit(1)
+    click.echo(click.style("All files are compliant with the guidelines. 🎉", fg="green"), err=False)
+    sys.exit(0)
+
+
+def main_with_spinner() -> None:
+    with spinner():
+        main()

markdown_checker-0.2.1/src/markdown_checker/markdown_link_base.py

@@ -0,0 +1,42 @@
+import re
+from abc import ABC
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class MarkdownLinkBase(ABC):
+    """Base class for markdown links"""
+
+    link: str
+    line_number: int
+    file_path: Path
+    issue: str = ""
+
+    def has_locale(self) -> bool:
+        """
+        Check if the link has a locale
+
+        Returns:
+            True if the link has a locale, False otherwise
+        """
+        locale_pattern = re.compile(r"\/[a-z]{2}-[a-z]{2}\/")
+        matches = re.findall(locale_pattern, self.link)
+        return bool(matches)
+
+    def has_tracking(self) -> bool:
+        """
+        Check if the link has a tracking ID
+
+        Returns:
+            True if the link has a tracking ID, False otherwise
+        """
+        tracking_pattern = re.compile(r"(\?|\&)(WT|wt)\.mc_id=")
+        matches = re.findall(tracking_pattern, self.link)
+        return bool(matches)
+
+    def __str__(self) -> str:
+        return self.link
+
+    def __repr__(self) -> str:
+        return self.link

markdown_checker-0.2.1/src/markdown_checker/paths.py

@@ -0,0 +1,92 @@
+import os
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from markdown_checker.markdown_link_base import MarkdownLinkBase
+
+
+@dataclass
+class MarkdownPath(MarkdownLinkBase):
+    """Dataclass to store info about a path"""
+
+    @property
+    def path_without_fragments(self) -> Path:
+        """
+        Get the path without the fragment
+
+        Returns:
+            The path without the fragment
+        """
+        return Path(self.remove_fragments())
+
+    def remove_fragments(self) -> str:
+        """
+        Remove the fragments from a path
+
+        Returns:
+            The path without the fragment
+        """
+        # Find the last occurrence of the dot
+        dot_index = self.link.rfind(".")
+        self.link = re.sub(r"(\?|\&)(WT|wt)\.mc_id=.*", "", self.link)
+
+        # If a dot is found, slice the string up to the end of the extension
+        if dot_index != -1:
+            # Find the next slash or fragment after the dot
+            slash_index = self.link.find("/", dot_index)
+            fragment_index = self.link.find("#", dot_index)
+
+            # Determine the earliest occurrence of either slash or fragment or query
+            end_index = min(
+                slash_index if slash_index != -1 else len(self.link),
+                fragment_index if fragment_index != -1 else len(self.link),
+            )
+            return self.link[:end_index]
+        else:
+            return self.link
+
+    def get_full_path(self) -> Path:
+        """
+        Get the full path of the file by resolving the path without fragments
+
+        Returns:
+            The full path of the file
+        """
+        try:
+            return self.path_without_fragments.resolve()
+        except (FileNotFoundError, RuntimeError):
+            return Path(self.get_full_path_relative())
+
+    def get_full_path_relative(self) -> str:
+        """
+        Get the full path of the file by resolving the path without fragments
+
+        Returns:
+            The full path of the file
+        """
+        return os.path.normpath(os.path.join(os.path.dirname(self.file_path), self.remove_fragments()))
+
+    def exists(self) -> bool:
+        """
+        Check if the path exists
+
+        Returns:
+            True if the path exists, False otherwise
+        """
+        # Paths starting with / are considered absolute and not resolved
+        # so we need to remove the / and check if the path exists
+        changed = False
+        if self.link.startswith("/"):
+            self.link = self.link[1:]
+            changed = True
+        # Check if the path exists using the relative path resolution
+        if os.path.exists(self.get_full_path_relative()):
+            return True
+        # Check if the path exists using the full path resolution recursively
+        if self.get_full_path().exists():
+            return True
+        # Change the link back to the original value
+        if changed:
+            self.link = "/" + self.link
+        return False