envoy.code.check 0.5.7__tar.gz → 0.5.9__tar.gz

{envoy.code.check-0.5.7 → envoy.code.check-0.5.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: envoy.code.check
-Version: 0.5.7
+Version: 0.5.9
 Summary: "Code checker used in Envoy proxy's CI"
 Home-page: https://github.com/envoyproxy/toolshed/tree/main/envoy.code.check
 Author: Ryan Northey

{envoy.code.check-0.5.7 → envoy.code.check-0.5.9}/envoy/code/check/abstract/checker.py

@@ -58,6 +58,7 @@ class ACodeChecker(
         "changelog",
         "extensions_fuzzed",
         "extensions_metadata",
+        "extensions_owners",
         "extensions_registered",
         "glint",
         "gofmt",
@@ -135,6 +136,8 @@ class ACodeChecker(
         """Extensions checker."""
         return self.extensions_class(
             self.directory,
+            owners=self.args.owners,
+            codeowners=self.args.codeowners,
             extensions_build_config=self.args.extensions_build_config,
             extensions_fuzzed_count=self.args.extensions_fuzzed_count,
             **self.check_kwargs)
@@ -260,6 +263,8 @@ class ACodeChecker(
         parser.add_argument("-x", "--excluding", action="append")
         parser.add_argument("-b", "--binary", action="append")
         parser.add_argument("-s", "--since")
+        parser.add_argument("--codeowners")
+        parser.add_argument("--owners")
         parser.add_argument("--extensions_build_config")
         parser.add_argument("--extensions_fuzzed_count")
 
@@ -294,6 +299,15 @@ class ACodeChecker(
             else:
                 self.succeed("extensions_metadata", [f"{extension}"])
 
+    async def check_extensions_owners(self) -> None:
+        """Check for glint issues."""
+        checks = await self.extensions.owners_errors
+        for extension, errors in sorted(checks.items()):
+            if errors:
+                self.error("extensions_owners", errors)
+            else:
+                self.succeed("extensions_owners", [f"{extension}"])
+
     async def check_extensions_registered(self) -> None:
         """Check for glint issues."""
         if errors := await self.extensions.registration_errors:
@@ -362,6 +376,11 @@ class ACodeChecker(
         self.extensions.extensions_schema
         self.log.debug(f"Preloaded extensions ({len(metadata)})")
 
+    @checker.preload(when=["extensions_owners"])
+    async def preload_extensions_owners(self) -> None:
+        await self.extensions.owners_errors
+        self.log.debug("Preloaded extensions owners")
+
     @checker.preload(
         when=["python_flake8"],
         catches=[subprocess.exceptions.OSCommandError])

{envoy.code.check-0.5.7 → envoy.code.check-0.5.9}/envoy/code/check/abstract/extensions.py

@@ -4,10 +4,11 @@ import itertools
 import json
 import logging
 import pathlib
+import re
 from functools import cached_property
 from typing import (
     Any, cast, Dict, List,
-    Optional, Set, Tuple, Type, Union)
+    Optional, Pattern, Set, Tuple, Type, Union)
 
 import yaml as _yaml
 
@@ -21,6 +22,10 @@ from envoy.code.check import abstract, exceptions, interface, typing
 
 logger = logging.getLogger(__name__)
 
+CODEOWNER_RE = r"@\S+"
+CODEOWNERS_CONTRIB_RE = r"(/contrib/[^@]*\s+)(@.*)"
+CODEOWNERS_EXTENSIONS_RE = r".*(extensions[^@]*\s+)(@.*)"
+
 FUZZ_TEST_PATH = (
     "test/extensions/filters/network/common/fuzz/config.bzl")
 METADATA_PATH = "source/extensions/extensions_metadata.yaml"
@@ -29,6 +34,12 @@ METADATA_ONLY_EXTENSIONS = (
 CONTRIB_METADATA_PATH = "contrib/extensions_metadata.yaml"
 EXTENSIONS_SCHEMA = "tools/extensions/extensions_schema.yaml"
 
+MAINTAINERS_RE = r".*github.com.(.*)\)\)"
+OWNERS_MIN_DEFAULT = 2
+TRACKED_OWNERSHIP_RE = (
+    r"^source/extensions/[^/]+/[^/]+/.*"
+    r"|^contrib/[^/]+/[^/]+/.*")
+
 # TODO(phlax): remove this workaround if/when per-category status is added
 UPSTREAM_EXTENSION_CATEGORY = "envoy.filters.http.upstream"
 
@@ -36,11 +47,14 @@ UPSTREAM_EXTENSION_CATEGORY = "envoy.filters.http.upstream"
 @abstracts.implementer(interface.IExtensionsCheck)
 class AExtensionsCheck(abstract.ACodeCheck, metaclass=abstracts.Abstraction):
     """Extensions check."""
+    _owners_min_default: int = OWNERS_MIN_DEFAULT
 
     def __init__(
             self,
             *args,
             **kwargs) -> None:
+        self._owners = kwargs.pop("owners")
+        self._codeowners = kwargs.pop("codeowners")
         self.extensions_build_config = kwargs.pop("extensions_build_config")
         self._fuzzed_count = kwargs.pop("extensions_fuzzed_count", None)
         super().__init__(*args, **kwargs)
@@ -64,6 +78,22 @@ class AExtensionsCheck(abstract.ACodeCheck, metaclass=abstracts.Abstraction):
     def builtin_extensions(self) -> typing.ExtensionsSchemaBuiltinList:
         return self.extensions_schema["builtin"]
 
+    @cached_property
+    def codeowner_re(self) -> Pattern[str]:
+        return re.compile(CODEOWNER_RE)
+
+    @cached_property
+    def codeowners_contrib_re(self) -> Pattern[str]:
+        return re.compile(CODEOWNERS_CONTRIB_RE)
+
+    @cached_property
+    def codeowners_extensions_re(self) -> Pattern[str]:
+        return re.compile(CODEOWNERS_EXTENSIONS_RE)
+
+    @cached_property
+    def codeowners_path(self) -> pathlib.Path:
+        return pathlib.Path(self._codeowners)
+
     @cached_property
     def configured_extensions(self) -> typing.ConfiguredExtensionsDict:
         return cast(
@@ -120,6 +150,21 @@ class AExtensionsCheck(abstract.ACodeCheck, metaclass=abstracts.Abstraction):
             if self._fuzzed_count is not None
             else None)
 
+    @cached_property
+    def maintainers(self) -> set[str]:
+        maintainers = {"@UNOWNED"}
+        with self.owners_path.open() as f:
+            for line in f:
+                try:
+                    maintainers |= self._maintainer_line_parse(line)
+                except StopIteration:
+                    break
+        return maintainers
+
+    @cached_property
+    def maintainers_re(self) -> Pattern[str]:
+        return re.compile(MAINTAINERS_RE)
+
     @async_property(cache=True)
     async def metadata(self) -> typing.ExtensionsMetadataDict:
         return dict(**await self.metadata_core, **await self.metadata_contrib)
@@ -164,6 +209,39 @@ class AExtensionsCheck(abstract.ACodeCheck, metaclass=abstracts.Abstraction):
     def metadata_only_extensions(self) -> Set[str]:
         return set(METADATA_ONLY_EXTENSIONS)
 
+    @cached_property
+    def owned(self):
+        owned = dict(contrib={}, core={})
+        with self.codeowners_path.open() as f:
+            for line in f:
+                owned["core"].update(
+                    self._owners_extension_match_line(line))
+                owned["contrib"].update(
+                    self._owners_extension_match_line(
+                        line,
+                        matcher=self.codeowners_contrib_re))
+        return owned
+
+    @async_property(cache=True)
+    async def owners_errors(self) -> Dict[str, Tuple[str, ...]]:
+        return (
+            await self._owners_tracked
+            | await self._owners_found)
+
+    @cached_property
+    def owners_path(self) -> pathlib.Path:
+        return pathlib.Path(self._owners)
+
+    @cached_property
+    def ownership_exceptions(self) -> dict[str, dict[str, int]]:
+        # TODO(phlax): Put this to config
+        return {
+            "extensions/filters/http/composite": dict(owners=1),
+            "contrib/config/source": dict(owners=0),
+            "contrib/config/test": dict(owners=0),
+            "contrib/common/sqlutils/": dict(owners=1),
+            "contrib/language/": dict(owners=1)}
+
     @async_property
     async def registration_errors(self) -> List[str]:
         return [
@@ -185,6 +263,24 @@ class AExtensionsCheck(abstract.ACodeCheck, metaclass=abstracts.Abstraction):
                 and (data["security_posture"]
                      == "robust_to_untrusted_downstream"))])
 
+    @async_property
+    async def tracked_directories(self) -> set[str]:
+        return set(
+            str(pathlib.Path(path).parent)
+            for path
+            in await self.directory.files
+            if self.tracked_ownership_re.match(path))
+
+    @cached_property
+    def tracked_ownership(self) -> tuple[str, ...]:
+        return (
+            tuple(f"source/{p}" for p in self.owned['core'].keys())
+            + tuple(self.owned["contrib"].keys()))
+
+    @cached_property
+    def tracked_ownership_re(self) -> Pattern[str]:
+        return re.compile(TRACKED_OWNERSHIP_RE)
+
     async def check_metadata(self, extension: str) -> Tuple[str, ...]:
         return tuple(
             itertools.chain.from_iterable(
@@ -194,6 +290,32 @@ class AExtensionsCheck(abstract.ACodeCheck, metaclass=abstracts.Abstraction):
                     self._check_metadata_status(extension),
                     self._check_metadata_status_upstream(extension))))
 
+    @async_property(cache=True)
+    async def _owners_found(self) -> dict[str, tuple[str, ...]]:
+        return {
+            directory: self._owners_error_matches(directory)
+            for directory
+            in await self.tracked_directories}
+
+    def _owners_less_than_min(self, extension, data) -> int:
+        min_owners = self.ownership_exceptions.get(
+            extension, {}).get("owners", self._owners_min_default)
+        return (
+            min_owners
+            if (len(data["owners"]) < min_owners
+                and data["owners"] != {"@UNOWNED"})
+            else 0)
+
+    @async_property(cache=True)
+    async def _owners_tracked(self) -> dict[str, tuple[str, ...]]:
+        return {
+            extension: self._owners_error_tracked(
+                extension,
+                extension_type,
+                data)
+            for extension_type, extensions in self.owned.items()
+            for extension, data in extensions.items()}
+
     async def _check_metadata_categories(
             self, extension: str) -> Tuple[str, ...]:
         categories = (await self.metadata)[extension].get("categories", ())
@@ -279,6 +401,14 @@ class AExtensionsCheck(abstract.ACodeCheck, metaclass=abstracts.Abstraction):
             logger.warning(warn_message.format(path=path, e=e))
             return e.value
 
+    def _maintainer_line_parse(self, line: str) -> set[str]:
+        if "Senior extension maintainers" in line:
+            raise StopIteration()
+        return (
+            {f"@{m.group(1).lower()}"}
+            if (m := self.maintainers_re.search(line)) is not None
+            else set())
+
     async def _metadata(self, path) -> typing.ExtensionsMetadataDict:
         errors = (functional.exceptions.TypeCastingError, FileNotFoundError)
         try:
@@ -290,3 +420,56 @@ class AExtensionsCheck(abstract.ACodeCheck, metaclass=abstracts.Abstraction):
             raise exceptions.ExtensionsConfigurationError(
                 "Failed to parse extensions metadata "
                 f"({path}): {e}")
+
+    def _owners_error_matches(self, path) -> tuple[str, ...]:
+        _skip = (
+            not self._owners_expected(path)
+            or path.startswith(self.tracked_ownership))
+        if _skip:
+            return ()
+        for tracked in self.tracked_ownership:
+            if tracked.startswith(path):
+                return ()
+        return (f"Directory ({path}) has no owners in CODEOWNERS", )
+
+    def _owners_error_tracked(
+            self,
+            extension: str,
+            extension_type: str,
+            data: dict[str, set]) -> tuple[str, ...]:
+        errors: tuple[str, ...] = ()
+        if min_owners := self._owners_less_than_min(extension, data):
+            errors += (
+                f"Extension ({extension}) has less than minimum "
+                f"of {min_owners} owners ({len(data['owners'])}) "
+                "in CODEOWNERS", )
+        if extension_type == "core" and len(data["maintainers"]) < 1:
+            errors += (
+                f"Extension ({extension}) has less than minimum "
+                f"of 1 maintainer ({len(data['maintainers'])}) "
+                "in CODEOWNERS", )
+        return errors
+
+    def _owners_expected(self, path: str, default: int = 1) -> int:
+        return self.ownership_exceptions.get(
+            path, {}).get("owners", default)
+
+    def _owners_extension_match_line(
+            self,
+            line: str,
+            matcher: Optional[
+                Pattern[str]] = None) -> dict[str, dict[str, set]]:
+        if line.startswith('#'):
+            return {}
+        m = (matcher or self.codeowners_extensions_re).search(line)
+        if m is None:
+            return {}
+        path = m.group(1).strip().lstrip("/")
+        owners = set(
+            self.codeowner_re.findall(m.group(2).strip()))
+        return {
+            path: dict(
+                owners=owners,
+                maintainers=(
+                    owners
+                    & self.maintainers))}

{envoy.code.check-0.5.7 → envoy.code.check-0.5.9}/envoy/code/check/abstract/yamllint.py

@@ -1,11 +1,10 @@
 
 import io
 import pathlib
-import re
 from functools import cached_property, partial
 from typing import (
     AsyncIterator, Dict, Generator, Iterator, List, Optional,
-    Pattern, Set, Tuple)
+    Set, Tuple)
 
 from yamllint import linter  # type:ignore
 from yamllint.config import YamlLintConfig  # type:ignore
@@ -23,12 +22,6 @@ from envoy.code.check import abstract, typing
 
 
 YAMLLINT_CONFIG = '.yamllint'
-YAMLLINT_MATCH_RE = (
-    r"[\w/\.]*\.yml$",
-    r"[\w/\.]*\.yaml$", )
-YAMLLINT_NOMATCH_RE = (
-    r"[\w/\.]*\.template\.yaml$",
-    r"[\w/\.]*/server_xds\.cds\.with_unknown_field\.*\.yaml$")
 
 
 @abstracts.implementer(directory.IDirectoryContext)
@@ -103,13 +96,7 @@ class AYamllintCheck(abstract.AFileCodeCheck, metaclass=abstracts.Abstraction):
 
     @async_property
     async def checker_files(self) -> Set[str]:
-        """Files with a `.yaml` suffix, but that are not excluded."""
-        return set(
-            path
-            for path
-            in await self.directory.files
-            if (self.path_match_re.match(path)
-                and not self.path_match_exclude_re.match(path)))
+        return await self.directory.files
 
     @cached_property
     def config(self) -> YamlLintConfig:
@@ -119,16 +106,6 @@ class AYamllintCheck(abstract.AFileCodeCheck, metaclass=abstracts.Abstraction):
     def config_path(self) -> pathlib.Path:
         return self.directory.path.joinpath(YAMLLINT_CONFIG)
 
-    @cached_property
-    def path_match_re(self) -> Pattern[str]:
-        """Regex to match files to check."""
-        return re.compile("|".join(YAMLLINT_MATCH_RE))
-
-    @cached_property
-    def path_match_exclude_re(self) -> Pattern[str]:
-        """Regex to match files not to check."""
-        return re.compile("|".join(YAMLLINT_NOMATCH_RE))
-
     @async_property(cache=True)
     async def problem_files(self) -> "typing.ProblemDict":
         return dict(await AwaitableGenerator(self._problem_files))

{envoy.code.check-0.5.7 → envoy.code.check-0.5.9}/envoy/code/check/checker.py

@@ -42,7 +42,7 @@ class YapfCheck(abstract.AYapfCheck):
 
 
 @abstracts.implementer(abstract.AYamllintCheck)
-class YamllintCheck:
+class YamllintCheck(abstract.AYamllintCheck):
     pass
 
 

{envoy.code.check-0.5.7 → envoy.code.check-0.5.9}/envoy/code/check/interface.py

@@ -80,6 +80,11 @@ class IExtensionsCheck(metaclass=abstracts.Interface):
     async def metadata_errors(self) -> Dict[str, Tuple[str, ...]]:
         raise NotImplementedError
 
+    @property  # type:ignore
+    @abstracts.interfacemethod
+    async def owners_errors(self) -> Dict[str, Tuple[str, ...]]:
+        raise NotImplementedError
+
     @property  # type:ignore
     @abstracts.interfacemethod
     async def registration_errors(self) -> List[str]:

{envoy.code.check-0.5.7 → envoy.code.check-0.5.9}/envoy.code.check.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: envoy.code.check
-Version: 0.5.7
+Version: 0.5.9
 Summary: "Code checker used in Envoy proxy's CI"
 Home-page: https://github.com/envoyproxy/toolshed/tree/main/envoy.code.check
 Author: Ryan Northey

{envoy.code.check-0.5.7 → envoy.code.check-0.5.9}/envoy.code.check.egg-info/requires.txt

@@ -1,5 +1,5 @@
 abstracts>=0.0.12
-aio.core>=0.10.0
+aio.core>=0.10.1
 aio.run.checker>=0.5.7
 envoy.base.utils>=0.4.7
 flake8>=6

{envoy.code.check-0.5.7 → envoy.code.check-0.5.9}/setup.py

@@ -15,7 +15,7 @@ setup(**{
     },
     'install_requires': (
         'abstracts>=0.0.12',
-        'aio.core>=0.10.0',
+        'aio.core>=0.10.1',
         'aio.run.checker>=0.5.7',
         'envoy.base.utils>=0.4.7',
         'flake8>=6',
@@ -46,5 +46,5 @@ Code checker used in Envoy proxy's CI
     ),
     'python_requires': '>=3.10.0',
     'url': 'https://github.com/envoyproxy/toolshed/tree/main/envoy.code.check',
-    'version': '0.5.7',
+    'version': '0.5.9',
 })

{envoy.code.check-0.5.7 → envoy.code.check-0.5.9}/envoy/code/check/abstract/__init__.py

@@ -51,10 +51,10 @@ __all__ = (
     "AYapfCheck",
     "base",
     "checker",
+    "extensions",
     "flake8",
     "glint",
     "gofmt",
-    "extensions",
     "shellcheck",
     "changelog",
     "yamllint",