troubadix 24.9.4__tar.gz → 24.10.1__tar.gz

{troubadix-24.9.4 → troubadix-24.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: troubadix
-Version: 24.9.4
+Version: 24.10.1
 Summary: A linting and QA check tool for NASL files
 Home-page: https://github.com/greenbone/troubadix
 License: GPL-3.0-or-later
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Dist: chardet (>=4,<6)
 Requires-Dist: charset-normalizer (>=3.2.0,<4.0.0)
@@ -23,7 +24,7 @@ Requires-Dist: codespell (>=2.0.0,<3.0.0)
 Requires-Dist: gitpython (>=3.1.31,<4.0.0)
 Requires-Dist: pontos (>=22.7,<25.0)
 Requires-Dist: python-magic (>=0.4.25,<0.5.0)
-Requires-Dist: validators (==0.20.0)
+Requires-Dist: validators (>=0.34.0,<0.35.0)
 Project-URL: Repository, https://github.com/greenbone/troubadix
 Description-Content-Type: text/markdown
 

{troubadix-24.9.4 → troubadix-24.10.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "troubadix"
-version = "24.9.4"
+version = "24.10.1"
 description = "A linting and QA check tool for NASL files"
 authors = ["Greenbone <info@greenbone.net>"]
 license = "GPL-3.0-or-later"
@@ -29,14 +29,14 @@ pontos = ">=22.7,<25.0"
 codespell = "^2.0.0"
 python-magic = "^0.4.25"
 chardet = ">=4,<6"
-validators = "0.20.0"
+validators = "^0.34.0"
 gitpython = "^3.1.31"
 charset-normalizer = "^3.2.0"
 
 [tool.poetry.group.dev.dependencies]
 autohooks = ">=21.7.0"
 autohooks-plugin-black = ">=21.12.0"
-ruff = ">=0.5.6,<0.7.0"
+ruff = ">=0.5.6,<0.8.0"
 autohooks-plugin-ruff = "^24.1.0"
 
 [tool.black]

troubadix-24.10.1/tests/plugins/test_spaces_before_dots.py

@@ -0,0 +1,103 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2024 Greenbone AG
+
+from pathlib import Path
+
+from tests.plugins import PluginTestCase
+from troubadix.helper import CURRENT_ENCODING
+from troubadix.plugin import LinterFix
+from troubadix.plugins.spaces_before_dots import CheckSpacesBeforeDots
+
+
+class TestSpacesBeforeDots(PluginTestCase):
+
+    def test_ok(self):
+        nasl_file = Path("/some/fake/directory/test.nasl")
+        content = """
+              script_tag(name:"summary", value:"Foo Bar.");
+              script_tag(name:"solution", value:"Foo .NET.");
+              script_tag(name:"insight", value:"Foo Bar ...");
+            """
+        fake_context = self.create_file_plugin_context(
+            nasl_file=nasl_file, file_content=content
+        )
+        plugin = CheckSpacesBeforeDots(fake_context)
+        results = list(plugin.run())
+        self.assertEqual(len(results), 0)
+
+    def test_fail(self):
+        nasl_file = Path("/some/fake/directory/test.nasl")
+        content = (
+            """
+              script_tag(name:"summary", value:"Foo Bar .");
+              script_tag(name:"vuldetect", value:"Foo Bar .");
+              script_tag(name:"insight", value:"Foo Bar .");
+              script_tag(name:"impact", value:"Foo . Bar . Foo");
+            """
+            'script_tag(name:"affected", value:"Foo\n.\nBar.");'
+            'script_tag(name:"solution", value:"Foo Bar\n.");'
+        )
+        fake_context = self.create_file_plugin_context(
+            nasl_file=nasl_file, file_content=content
+        )
+        plugin = CheckSpacesBeforeDots(fake_context)
+        results = list(plugin.run())
+        self.assertEqual(len(results), 7)
+        self.assertEqual(
+            results[0].message,
+            "value of script_tag summary has at least one occurence of excess"
+            " whitespace before a dot:\n"
+            " 'script_tag(name:"
+            '"summary", value:"Foo Bar .");'
+            "'",
+        )
+
+    def test_ignore(self):
+        nasl_file = Path("/some/fake/directory/test.inc")
+        fake_context = self.create_file_plugin_context(nasl_file=nasl_file)
+        plugin = CheckSpacesBeforeDots(fake_context)
+
+        results = list(plugin.run())
+
+        self.assertEqual(len(results), 0)
+
+    def test_fix(self):
+        with self.create_directory() as tempdir:
+            path = tempdir / "file.nasl"
+            content = (
+                """
+                  script_tag(name:"summary", value:"Foo Bar .");
+                  script_tag(name:"vuldetect", value:"Foo Bar .");
+                  script_tag(name:"insight", value:"Foo .Net Bar .");
+                  script_tag(name:"impact", value:"Foo . Bar . Foo");
+                """
+                'script_tag(name:"affected", value:"Foo\n.\nBar.");'
+                'script_tag(name:"solution", value:"Foo Bar\n.");'
+            )
+            expected_modified_content = (
+                """
+                  script_tag(name:"summary", value:"Foo Bar.");
+                  script_tag(name:"vuldetect", value:"Foo Bar.");
+                  script_tag(name:"insight", value:"Foo .Net Bar.");
+                  script_tag(name:"impact", value:"Foo. Bar. Foo");
+                """
+                'script_tag(name:"affected", value:"Foo.\nBar.");'
+                'script_tag(name:"solution", value:"Foo Bar.");'
+            )
+            path.write_text(content, encoding=CURRENT_ENCODING)
+
+            fake_context = self.create_file_plugin_context(
+                nasl_file=path, file_content=content
+            )
+
+            plugin = CheckSpacesBeforeDots(fake_context)
+
+            # keep list() to consume the iterator
+            list(plugin.run())
+
+            results = list(plugin.fix())
+            self.assertEqual(len(results), 1)
+
+            self.assertIsInstance(results[0], LinterFix)
+            modified_content = path.read_text(encoding=CURRENT_ENCODING)
+            self.assertEqual(modified_content, expected_modified_content)

{troubadix-24.9.4 → troubadix-24.10.1}/tests/standalone_plugins/test_deprecate_vts.py

@@ -11,9 +11,9 @@ from troubadix.standalone_plugins.deprecate_vts import (
     _finalize_content,
     _get_summary,
     deprecate,
-    filter_files,
     get_files_from_path,
     parse_args,
+    parse_files,
     update_summary,
 )
 
@@ -26,7 +26,7 @@ class ParseArgsTestCase(unittest.TestCase):
 
         args = parse_args(
             [
-                "--files",
+                "--file",
                 testfile,
                 "--output-path",
                 output_path,
@@ -34,7 +34,7 @@ class ParseArgsTestCase(unittest.TestCase):
                 reason,
             ]
         )
-        self.assertEqual(args.files, [Path(testfile)])
+        self.assertEqual(args.file, [Path(testfile)])
         self.assertEqual(args.output_path, Path(output_path))
         self.assertEqual(args.deprecation_reason, reason)
 
@@ -47,7 +47,7 @@ class ParseArgsTestCase(unittest.TestCase):
         with self.assertRaises(SystemExit):
             parse_args(
                 [
-                    "--files",
+                    "--file",
                     testfile,
                     "--output-path",
                     output_path,
@@ -87,6 +87,28 @@ class ParseArgsTestCase(unittest.TestCase):
                 ]
             )
 
+    def test_parse_from_file(self):
+        output_path = "attic/"
+        with TemporaryDirectory() as tempdir:
+            from_file = tempdir / "from_file.txt"
+            testfile1 = tempdir / "testfile1.nasl"
+            testfile2 = tempdir / "testfile2.nasl"
+
+            from_file.write_text(f"{testfile1}\n{testfile2}\n", encoding="utf8")
+
+            args = parse_args(
+                [
+                    "--from-file",
+                    str(from_file),
+                    "--output-path",
+                    output_path,
+                    "-r",
+                    "NOTUS",
+                ]
+            )
+
+            self.assertEqual(args.from_file, from_file)
+
 
 NASL_CONTENT = (
     '...if(description)\n{\n  script_oid("1.3.6.1.4.1.25623.1.0.910673");'
@@ -190,16 +212,12 @@ class DeprecateVTsTestCase(unittest.TestCase):
 
             self.assertEqual(result, expected)
 
-    def test_filter_files(self):
+    def test_parse_files(self):
         with TemporaryDirectory() as in_dir:
             testfile1 = in_dir / "gb_rhsa_2021_8383_8383.nasl"
             testfile1.write_text(NASL_CONTENT, encoding="utf8")
-            testfile2 = in_dir / "gb_rhsa_2020_8383_8383.nasl"
-            testfile2.write_text(NASL_CONTENT, encoding="utf8")
 
-            result = filter_files(
-                files=[testfile1, testfile2], filename_prefix="gb_rhsa_2021"
-            )
+            result = parse_files(files=[testfile1])
             expected = [
                 DeprecatedFile(
                     name="gb_rhsa_2021_8383_8383.nasl",
@@ -209,15 +227,3 @@ class DeprecateVTsTestCase(unittest.TestCase):
             ]
 
             self.assertEqual(result, expected)
-
-    def test_filter_files_out(self):
-        with TemporaryDirectory() as in_dir:
-            testfile1 = in_dir / "gb_rhsa_2021_8383_8383.nasl"
-            testfile1.write_text(NASL_CONTENT, encoding="utf8")
-
-            result = filter_files(
-                files=[testfile1], filename_prefix="gb_rhsa_2020"
-            )
-            expected = []
-
-            self.assertEqual(result, expected)

{troubadix-24.9.4 → troubadix-24.10.1}/troubadix/__version__.py

@@ -2,4 +2,4 @@
 
 # THIS IS AN AUTOGENERATED FILE. DO NOT TOUCH!
 
-__version__ = "24.9.4"
+__version__ = "24.10.1"

{troubadix-24.9.4 → troubadix-24.10.1}/troubadix/plugins/__init__.py

@@ -18,6 +18,7 @@
 from typing import Iterable, List
 
 from troubadix.plugin import FilePlugin, FilesPlugin, Plugin
+from troubadix.plugins.spaces_before_dots import CheckSpacesBeforeDots
 
 from .badwords import CheckBadwords
 from .copyright_text import CheckCopyrightText
@@ -140,6 +141,7 @@ _FILE_PLUGINS = [
     CheckVTFilePermissions,
     CheckVTPlacement,
     CheckWrongSetGetKBCalls,
+    CheckSpacesBeforeDots,
 ]
 
 # plugins checking all files

{troubadix-24.9.4 → troubadix-24.10.1}/troubadix/plugins/script_xref_url.py

@@ -64,7 +64,7 @@ class CheckScriptXrefUrl(FileContentPlugin):
         for match in matches:
             if match:
                 if match.group("value") not in ALLOWED_URLS and not url(
-                    match.group("value")
+                    match.group("value"), strict_query=False
                 ):
                     yield LinterError(
                         f"{match.group(0)}: Invalid URL value",

troubadix-24.10.1/troubadix/plugins/spaces_before_dots.py

@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2024 Greenbone AG
+import re
+from collections.abc import Iterator
+from pathlib import Path
+
+from troubadix.helper import CURRENT_ENCODING
+from troubadix.helper.helper import is_ignore_file
+from troubadix.helper.patterns import (
+    ScriptTag,
+    get_script_tag_pattern,
+)
+from troubadix.plugin import (
+    FileContentPlugin,
+    LinterFix,
+    LinterResult,
+    LinterWarning,
+)
+
+TAGS = [
+    ScriptTag.SUMMARY,
+    ScriptTag.VULDETECT,
+    ScriptTag.INSIGHT,
+    ScriptTag.IMPACT,
+    ScriptTag.AFFECTED,
+    ScriptTag.SOLUTION,
+]
+
+# Regex pattern to match:
+# 1. A dot preceded and/or followed by any whitespace character (floating between words)
+# 2. A dot preceded by any whitespace character at the end of the string
+PATTERN = re.compile(r"\s+\.(\s|$)")
+IGNORE = [
+    # 21.04 and 22.04 are generated and should not be touched manually
+    "21.04/",
+    "22.04/",
+    # uses dots for beginning of entry in enumeration
+    "common/2008/debian/deb_246.nasl",
+    "common/2008/debian/deb_266.nasl",
+    "common/2008/freebsd/freebsd_5e92e8a2.nasl",
+    "common/2008/freebsd/freebsdsa_cpio.nasl",
+    "common/2008/freebsd/freebsdsa_cvs2.nasl",
+    "common/2009/osc_photoGallery_sql_injection.nasl",
+    "common/2009/secpod_novell_edir_mult_vuln_jul09_lin.nasl",
+    "common/2009/secpod_novell_edir_mult_vuln_jul09_win.nasl",
+    "common/2010/freebsd/freebsd_3a7c5fc4.nasl",
+    "common/2012/freebsd/freebsd_a4a809d8.nasl",
+    "common/2015/amazon/alas-2014-455.nasl",
+    "common/2015/gb_mozilla_firefox_mult_vuln01_mar15_macosx.nasl",
+    "common/2015/gb_mozilla_firefox_mult_vuln01_mar15_win.nasl",
+    "common/2015/oracle/ELSA-2009-1619.nasl",
+    "common/2015/oracle/ELSA-2011-0586.nasl",
+    "common/2016/gb_perl_privilege_escalation_vuln_win.nasl",
+    "common/2021/dropbear/gb_dropbear_ssh_filename_vuln_may20.nasl",
+    "common/2021/eclipse/gb_jetty_GHSA-v7ff-8wcx-gmc5_lin.nasl",
+    "common/2021/eclipse/gb_jetty_GHSA-v7ff-8wcx-gmc5_win.nasl",
+    "common/gsf/2009/mandriva/gb_mandriva_MDVSA_2008_140.nasl",
+    "common/gsf/2009/mandriva/gb_mandriva_MDVSA_2008_141.nasl",
+    "common/gsf/2010/mandriva/gb_mandriva_MDVA_2010_173.nasl",
+    "common/gsf/2010/mandriva/gb_mandriva_MDVSA_2010_155.nasl",
+    "common/gsf/2010/mandriva/gb_mandriva_MDVSA_2010_155_1.nasl",
+    "common/gsf/2010/mandriva/gb_mandriva_MDVSA_2010_167.nasl",
+    "common/gsf/2020/f5/gb_f5_big_ip_K11315080.nasl",
+    "common/gsf/2020/f5/gb_f5_big_iq_K11315080.nasl",
+    "common/2022/opensuse/gb_opensuse_2022_1548_1.nasl",
+    "common/2024/opensuse/gb_opensuse_2023_3247_1.nasl",
+    "common/attic/debian/deb_232_1.nasl",
+]
+
+
+class CheckSpacesBeforeDots(FileContentPlugin):
+    name = "check_spaces_before_dots"
+
+    def check_content(
+        self, nasl_file: Path, file_content: str
+    ) -> Iterator[LinterResult]:
+        """
+        This plugin checks for excess whitespace before a dot
+        in script_tags that have full sentence values
+        """
+        self.matches = []
+        if nasl_file.suffix == ".inc" or is_ignore_file(nasl_file, IGNORE):
+            return
+        for tag in TAGS:
+            pattern = get_script_tag_pattern(tag)
+            match = pattern.search(file_content)
+            if not match:
+                continue
+
+            value = match.group("value")
+            value_start = match.start("value")
+
+            for excess_match in PATTERN.finditer(value):
+                whitespace_pos = excess_match.start() + value_start
+                self.matches.append((whitespace_pos, excess_match.group()))
+                fullmatch = match.group()
+                yield LinterWarning(
+                    f"value of script_tag {match.group('name')} has at least"
+                    " one occurence of excess whitespace before a dot:"
+                    f"\n '{fullmatch}'",
+                    file=nasl_file,
+                    plugin=self.name,
+                )
+
+    def fix(self) -> Iterator[LinterResult]:
+
+        if not self.matches:
+            return
+
+        # Sort matches by position, descending order to avoid messing up indices during replacement
+        self.matches.sort(reverse=True)
+
+        file_content = self.context.file_content
+        for pos, match_str in self.matches:
+            # Replace the match by removing the excess whitespace before the dot
+            fixed_str = re.sub(r"\s+\.", ".", match_str)
+            file_content = (
+                file_content[:pos]
+                + fixed_str
+                + file_content[pos + len(match_str) :]
+            )
+
+        with open(self.context.nasl_file, "w", encoding=CURRENT_ENCODING) as f:
+            f.write(file_content)
+
+        yield LinterFix(
+            "Excess spaces were removed",
+            file=self.context.nasl_file,
+            plugin=self.name,
+        )

{troubadix-24.9.4 → troubadix-24.10.1}/troubadix/standalone_plugins/deprecate_vts.py

@@ -2,12 +2,14 @@
 # SPDX-FileCopyrightText: 2024 Greenbone AG
 
 import re
-from argparse import ArgumentParser, Namespace
+from argparse import ArgumentParser, ArgumentTypeError, Namespace
 from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path
 from typing import Iterable, Optional
 
+from pontos.terminal.terminal import ConsoleTerminal
+
 from troubadix.argparser import directory_type, file_type
 from troubadix.helper.patterns import (
     ScriptTag,
@@ -15,6 +17,7 @@ from troubadix.helper.patterns import (
     get_script_tag_pattern,
     get_special_script_tag_pattern,
 )
+from troubadix.troubadix import from_file
 
 
 class Deprecations(Enum):
@@ -34,6 +37,15 @@ class DeprecatedFile:
 KB_ITEMS_PATTERN = re.compile(r"set_kb_item\(.+\);")
 
 
+def existing_file_type(string: str) -> Path:
+    file_path = Path(string)
+    if not file_path.exists():
+        raise ArgumentTypeError(f'File "{string}" does not exist.')
+    if not file_path.is_file():
+        raise ArgumentTypeError(f'"{string}" is not a file.')
+    return file_path
+
+
 def update_summary(file: DeprecatedFile, deprecation_reason: str) -> str:
     """Update the summary of the nasl script by adding the information
     that the script has been deprecated, and if possible, the oid of
@@ -83,26 +95,18 @@ def get_files_from_path(dir_path: Path = None) -> list:
     return [file for file in dir_path.glob("**/*")]
 
 
-def filter_files(
-    files: list, filename_prefix: str = None
-) -> list[DeprecatedFile]:
-    """Filter the files based on a provided prefix and convert them into
+def parse_files(files: list) -> list[DeprecatedFile]:
+    """Convert filepaths into
     DeprecatedFile objects
 
     Args:
         files: a list of files to deprecate, should be .nasl VTs
-        filename_prefix: an optional prefix to filter only specific files
 
     Returns:
         List of DeprecatedFile objects
 
     """
     to_deprecate = []
-    if filename_prefix:
-        filename_filter = re.compile(rf"{filename_prefix}")
-        files[:] = [
-            file for file in files if re.match(filename_filter, file.name)
-        ]
 
     for file in files:
         with file.open("r", encoding="latin-1") as fh:
@@ -193,16 +197,6 @@ def parse_args(args: Iterable[str] = None) -> Namespace:
         required=True,
         help="Path where the deprecated files should be written to.",
     )
-    parser.add_argument(
-        "-p",
-        "--filename-prefix",
-        metavar="<filename_prefix>",
-        nargs="?",
-        default=None,
-        type=str,
-        help="The prefix of the files you would like to deprecate,"
-        "for example 'gb_rhsa_2021' to filter on the year",
-    )
     parser.add_argument(
         "-r",
         "--deprecation-reason",
@@ -214,16 +208,17 @@ def parse_args(args: Iterable[str] = None) -> Namespace:
         "been merged with another still active VT, 'duplicate': The VT has"
         "a still active duplicate, 'defunct': The VT is no longer "
         "functional.",
+        required=True,
     )
     group = parser.add_mutually_exclusive_group(required=True)
     group.add_argument(
         "-f",
-        "--files",
-        metavar="<files>",
+        "--file",
+        metavar="<file>",
         nargs="+",
         default=None,
         type=file_type,
-        help="Files to deprecate",
+        help="File to deprecate",
     )
     group.add_argument(
         "-i",
@@ -231,20 +226,36 @@ def parse_args(args: Iterable[str] = None) -> Namespace:
         metavar="<input_path>",
         default=None,
         type=directory_type,
-        help="Path to the existing nasl script directory",
+        help="Path to a directory where all files should be deprecated.",
+    )
+    group.add_argument(
+        "--from-file",
+        metavar="<from_file>",
+        default=None,
+        type=existing_file_type,
+        help=(
+            "Path to a single file that contains a list of files "
+            "to be deprecated, separated by new lines."
+        ),
     )
     return parser.parse_args(args)
 
 
 def main():
     args = parse_args()
+    terminal = ConsoleTerminal()
     input_path = args.input_path if args.input_path else None
-    filename_prefix = args.filename_prefix
+    files = []
 
-    files = args.files or get_files_from_path(input_path)
-    filtered_files = filter_files(files, filename_prefix)
+    if input_path:
+        files = get_files_from_path(input_path)
+    elif args.from_file:
+        files = from_file(include_file=args.from_file, term=terminal)
+    elif args.file:
+        files = args.file
 
-    deprecate(args.output_path, filtered_files, args.deprecation_reason)
+    to_be_deprecated = parse_files(files)
+    deprecate(args.output_path, to_be_deprecated, args.deprecation_reason)
 
 
 if __name__ == "__main__":